drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * (c) Copyright 2007-2013 Datera, Inc.
   5  *
   6  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  ******************************************************************************/
  18
  19 #include <crypto/hash.h>
  20 #include <linux/string.h>
  21 #include <linux/kthread.h>
  22 #include <linux/completion.h>
  23 #include <linux/module.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/idr.h>
  26 #include <linux/delay.h>
  27 #include <linux/sched/signal.h>
  28 #include <asm/unaligned.h>
  29 #include <linux/inet.h>
  30 #include <net/ipv6.h>
  31 #include <scsi/scsi_proto.h>
  32 #include <scsi/iscsi_proto.h>
  33 #include <scsi/scsi_tcq.h>
  34 #include <target/target_core_base.h>
  35 #include <target/target_core_fabric.h>
  36
  37 #include <target/iscsi/iscsi_target_core.h>
  38 #include "iscsi_target_parameters.h"
  39 #include "iscsi_target_seq_pdu_list.h"
  40 #include "iscsi_target_datain_values.h"
  41 #include "iscsi_target_erl0.h"
  42 #include "iscsi_target_erl1.h"
  43 #include "iscsi_target_erl2.h"
  44 #include "iscsi_target_login.h"
  45 #include "iscsi_target_tmr.h"
  46 #include "iscsi_target_tpg.h"
  47 #include "iscsi_target_util.h"
  48 #include "iscsi_target.h"
  49 #include "iscsi_target_device.h"
  50 #include <target/iscsi/iscsi_target_stat.h>
  51
  52 #include <target/iscsi/iscsi_transport.h>
  53
  54 static LIST_HEAD(g_tiqn_list);
  55 static LIST_HEAD(g_np_list);
  56 static DEFINE_SPINLOCK(tiqn_lock);
  57 static DEFINE_MUTEX(np_lock);
  58
  59 static struct idr tiqn_idr;
  60 DEFINE_IDA(sess_ida);
  61 struct mutex auth_id_lock;
  62
  63 struct iscsit_global *iscsit_global;
  64
  65 struct kmem_cache *lio_qr_cache;
  66 struct kmem_cache *lio_dr_cache;
  67 struct kmem_cache *lio_ooo_cache;
  68 struct kmem_cache *lio_r2t_cache;
  69
  70 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  71                         struct iscsi_scsi_req *, u32);
  72
  73 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  74 {
  75         struct iscsi_tiqn *tiqn = NULL;
  76
  77         spin_lock(&tiqn_lock);
  78         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  79                 if (!strcmp(tiqn->tiqn, buf)) {
  80
  81                         spin_lock(&tiqn->tiqn_state_lock);
  82                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  83                                 tiqn->tiqn_access_count++;
  84                                 spin_unlock(&tiqn->tiqn_state_lock);
  85                                 spin_unlock(&tiqn_lock);
  86                                 return tiqn;
  87                         }
  88                         spin_unlock(&tiqn->tiqn_state_lock);
  89                 }
  90         }
  91         spin_unlock(&tiqn_lock);
  92
  93         return NULL;
  94 }
  95
  96 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  97 {
  98         spin_lock(&tiqn->tiqn_state_lock);
  99         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
 100                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 101                 spin_unlock(&tiqn->tiqn_state_lock);
 102                 return 0;
 103         }
 104         spin_unlock(&tiqn->tiqn_state_lock);
 105
 106         return -1;
 107 }
 108
 109 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 110 {
 111         spin_lock(&tiqn->tiqn_state_lock);
 112         tiqn->tiqn_access_count--;
 113         spin_unlock(&tiqn->tiqn_state_lock);
 114 }
 115
 116 /*
 117  * Note that IQN formatting is expected to be done in userspace, and
 118  * no explict IQN format checks are done here.
 119  */
 120 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 121 {
 122         struct iscsi_tiqn *tiqn = NULL;
 123         int ret;
 124
 125         if (strlen(buf) >= ISCSI_IQN_LEN) {
 126                 pr_err("Target IQN exceeds %d bytes\n",
 127                                 ISCSI_IQN_LEN);
 128                 return ERR_PTR(-EINVAL);
 129         }
 130
 131         tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL);
 132         if (!tiqn)
 133                 return ERR_PTR(-ENOMEM);
 134
 135         sprintf(tiqn->tiqn, "%s", buf);
 136         INIT_LIST_HEAD(&tiqn->tiqn_list);
 137         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 138         spin_lock_init(&tiqn->tiqn_state_lock);
 139         spin_lock_init(&tiqn->tiqn_tpg_lock);
 140         spin_lock_init(&tiqn->sess_err_stats.lock);
 141         spin_lock_init(&tiqn->login_stats.lock);
 142         spin_lock_init(&tiqn->logout_stats.lock);
 143
 144         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 145
 146         idr_preload(GFP_KERNEL);
 147         spin_lock(&tiqn_lock);
 148
 149         ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
 150         if (ret < 0) {
 151                 pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
 152                 spin_unlock(&tiqn_lock);
 153                 idr_preload_end();
 154                 kfree(tiqn);
 155                 return ERR_PTR(ret);
 156         }
 157         tiqn->tiqn_index = ret;
 158         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 159
 160         spin_unlock(&tiqn_lock);
 161         idr_preload_end();
 162
 163         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 164
 165         return tiqn;
 166
 167 }
 168
 169 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 170 {
 171         /*
 172          * Wait for accesses to said struct iscsi_tiqn to end.
 173          */
 174         spin_lock(&tiqn->tiqn_state_lock);
 175         while (tiqn->tiqn_access_count != 0) {
 176                 spin_unlock(&tiqn->tiqn_state_lock);
 177                 msleep(10);
 178                 spin_lock(&tiqn->tiqn_state_lock);
 179         }
 180         spin_unlock(&tiqn->tiqn_state_lock);
 181 }
 182
 183 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 184 {
 185         /*
 186          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 187          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 188          * attempts to access this struct iscsi_tiqn will fail from both transport
 189          * fabric and control code paths.
 190          */
 191         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 192                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 193                 return;
 194         }
 195
 196         iscsit_wait_for_tiqn(tiqn);
 197
 198         spin_lock(&tiqn_lock);
 199         list_del(&tiqn->tiqn_list);
 200         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 201         spin_unlock(&tiqn_lock);
 202
 203         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 204                         tiqn->tiqn);
 205         kfree(tiqn);
 206 }
 207
 208 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 209 {
 210         int ret;
 211         /*
 212          * Determine if the network portal is accepting storage traffic.
 213          */
 214         spin_lock_bh(&np->np_thread_lock);
 215         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 216                 spin_unlock_bh(&np->np_thread_lock);
 217                 return -1;
 218         }
 219         spin_unlock_bh(&np->np_thread_lock);
 220         /*
 221          * Determine if the portal group is accepting storage traffic.
 222          */
 223         spin_lock_bh(&tpg->tpg_state_lock);
 224         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 225                 spin_unlock_bh(&tpg->tpg_state_lock);
 226                 return -1;
 227         }
 228         spin_unlock_bh(&tpg->tpg_state_lock);
 229
 230         /*
 231          * Here we serialize access across the TIQN+TPG Tuple.
 232          */
 233         ret = down_interruptible(&tpg->np_login_sem);
 234         if (ret != 0)
 235                 return -1;
 236
 237         spin_lock_bh(&tpg->tpg_state_lock);
 238         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 239                 spin_unlock_bh(&tpg->tpg_state_lock);
 240                 up(&tpg->np_login_sem);
 241                 return -1;
 242         }
 243         spin_unlock_bh(&tpg->tpg_state_lock);
 244
 245         return 0;
 246 }
 247
 248 void iscsit_login_kref_put(struct kref *kref)
 249 {
 250         struct iscsi_tpg_np *tpg_np = container_of(kref,
 251                                 struct iscsi_tpg_np, tpg_np_kref);
 252
 253         complete(&tpg_np->tpg_np_comp);
 254 }
 255
 256 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg,
 257                        struct iscsi_tpg_np *tpg_np)
 258 {
 259         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 260
 261         up(&tpg->np_login_sem);
 262
 263         if (tpg_np)
 264                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 265
 266         if (tiqn)
 267                 iscsit_put_tiqn_for_login(tiqn);
 268
 269         return 0;
 270 }
 271
 272 bool iscsit_check_np_match(
 273         struct sockaddr_storage *sockaddr,
 274         struct iscsi_np *np,
 275         int network_transport)
 276 {
 277         struct sockaddr_in *sock_in, *sock_in_e;
 278         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 279         bool ip_match = false;
 280         u16 port, port_e;
 281
 282         if (sockaddr->ss_family == AF_INET6) {
 283                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 284                 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 285
 286                 if (!memcmp(&sock_in6->sin6_addr.in6_u,
 287                             &sock_in6_e->sin6_addr.in6_u,
 288                             sizeof(struct in6_addr)))
 289                         ip_match = true;
 290
 291                 port = ntohs(sock_in6->sin6_port);
 292                 port_e = ntohs(sock_in6_e->sin6_port);
 293         } else {
 294                 sock_in = (struct sockaddr_in *)sockaddr;
 295                 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 296
 297                 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
 298                         ip_match = true;
 299
 300                 port = ntohs(sock_in->sin_port);
 301                 port_e = ntohs(sock_in_e->sin_port);
 302         }
 303
 304         if (ip_match && (port_e == port) &&
 305             (np->np_network_transport == network_transport))
 306                 return true;
 307
 308         return false;
 309 }
 310
 311 static struct iscsi_np *iscsit_get_np(
 312         struct sockaddr_storage *sockaddr,
 313         int network_transport)
 314 {
 315         struct iscsi_np *np;
 316         bool match;
 317
 318         lockdep_assert_held(&np_lock);
 319
 320         list_for_each_entry(np, &g_np_list, np_list) {
 321                 spin_lock_bh(&np->np_thread_lock);
 322                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 323                         spin_unlock_bh(&np->np_thread_lock);
 324                         continue;
 325                 }
 326
 327                 match = iscsit_check_np_match(sockaddr, np, network_transport);
 328                 if (match) {
 329                         /*
 330                          * Increment the np_exports reference count now to
 331                          * prevent iscsit_del_np() below from being called
 332                          * while iscsi_tpg_add_network_portal() is called.
 333                          */
 334                         np->np_exports++;
 335                         spin_unlock_bh(&np->np_thread_lock);
 336                         return np;
 337                 }
 338                 spin_unlock_bh(&np->np_thread_lock);
 339         }
 340
 341         return NULL;
 342 }
 343
 344 struct iscsi_np *iscsit_add_np(
 345         struct sockaddr_storage *sockaddr,
 346         int network_transport)
 347 {
 348         struct iscsi_np *np;
 349         int ret;
 350
 351         mutex_lock(&np_lock);
 352
 353         /*
 354          * Locate the existing struct iscsi_np if already active..
 355          */
 356         np = iscsit_get_np(sockaddr, network_transport);
 357         if (np) {
 358                 mutex_unlock(&np_lock);
 359                 return np;
 360         }
 361
 362         np = kzalloc(sizeof(*np), GFP_KERNEL);
 363         if (!np) {
 364                 mutex_unlock(&np_lock);
 365                 return ERR_PTR(-ENOMEM);
 366         }
 367
 368         np->np_flags |= NPF_IP_NETWORK;
 369         np->np_network_transport = network_transport;
 370         spin_lock_init(&np->np_thread_lock);
 371         init_completion(&np->np_restart_comp);
 372         INIT_LIST_HEAD(&np->np_list);
 373
 374         timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0);
 375
 376         ret = iscsi_target_setup_login_socket(np, sockaddr);
 377         if (ret != 0) {
 378                 kfree(np);
 379                 mutex_unlock(&np_lock);
 380                 return ERR_PTR(ret);
 381         }
 382
 383         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 384         if (IS_ERR(np->np_thread)) {
 385                 pr_err("Unable to create kthread: iscsi_np\n");
 386                 ret = PTR_ERR(np->np_thread);
 387                 kfree(np);
 388                 mutex_unlock(&np_lock);
 389                 return ERR_PTR(ret);
 390         }
 391         /*
 392          * Increment the np_exports reference count now to prevent
 393          * iscsit_del_np() below from being run while a new call to
 394          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 395          * active.  We don't need to hold np->np_thread_lock at this
 396          * point because iscsi_np has not been added to g_np_list yet.
 397          */
 398         np->np_exports = 1;
 399         np->np_thread_state = ISCSI_NP_THREAD_ACTIVE;
 400
 401         list_add_tail(&np->np_list, &g_np_list);
 402         mutex_unlock(&np_lock);
 403
 404         pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n",
 405                 &np->np_sockaddr, np->np_transport->name);
 406
 407         return np;
 408 }
 409
 410 int iscsit_reset_np_thread(
 411         struct iscsi_np *np,
 412         struct iscsi_tpg_np *tpg_np,
 413         struct iscsi_portal_group *tpg,
 414         bool shutdown)
 415 {
 416         spin_lock_bh(&np->np_thread_lock);
 417         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 418                 spin_unlock_bh(&np->np_thread_lock);
 419                 return 0;
 420         }
 421         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 422         atomic_inc(&np->np_reset_count);
 423
 424         if (np->np_thread) {
 425                 spin_unlock_bh(&np->np_thread_lock);
 426                 send_sig(SIGINT, np->np_thread, 1);
 427                 wait_for_completion(&np->np_restart_comp);
 428                 spin_lock_bh(&np->np_thread_lock);
 429         }
 430         spin_unlock_bh(&np->np_thread_lock);
 431
 432         if (tpg_np && shutdown) {
 433                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 434
 435                 wait_for_completion(&tpg_np->tpg_np_comp);
 436         }
 437
 438         return 0;
 439 }
 440
 441 static void iscsit_free_np(struct iscsi_np *np)
 442 {
 443         if (np->np_socket)
 444                 sock_release(np->np_socket);
 445 }
 446
 447 int iscsit_del_np(struct iscsi_np *np)
 448 {
 449         spin_lock_bh(&np->np_thread_lock);
 450         np->np_exports--;
 451         if (np->np_exports) {
 452                 np->enabled = true;
 453                 spin_unlock_bh(&np->np_thread_lock);
 454                 return 0;
 455         }
 456         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 457         spin_unlock_bh(&np->np_thread_lock);
 458
 459         if (np->np_thread) {
 460                 /*
 461                  * We need to send the signal to wakeup Linux/Net
 462                  * which may be sleeping in sock_accept()..
 463                  */
 464                 send_sig(SIGINT, np->np_thread, 1);
 465                 kthread_stop(np->np_thread);
 466                 np->np_thread = NULL;
 467         }
 468
 469         np->np_transport->iscsit_free_np(np);
 470
 471         mutex_lock(&np_lock);
 472         list_del(&np->np_list);
 473         mutex_unlock(&np_lock);
 474
 475         pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n",
 476                 &np->np_sockaddr, np->np_transport->name);
 477
 478         iscsit_put_transport(np->np_transport);
 479         kfree(np);
 480         return 0;
 481 }
 482
 483 static void iscsit_get_rx_pdu(struct iscsi_conn *);
 484
 485 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 486 {
 487         return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
 488 }
 489 EXPORT_SYMBOL(iscsit_queue_rsp);
 490
 491 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 492 {
 493         spin_lock_bh(&conn->cmd_lock);
 494         if (!list_empty(&cmd->i_conn_node) &&
 495             !(cmd->se_cmd.transport_state & CMD_T_FABRIC_STOP))
 496                 list_del_init(&cmd->i_conn_node);
 497         spin_unlock_bh(&conn->cmd_lock);
 498
 499         __iscsit_free_cmd(cmd, true);
 500 }
 501 EXPORT_SYMBOL(iscsit_aborted_task);
 502
 503 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
 504                                       u32, u32, const void *, void *);
 505 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *);
 506
 507 static int
 508 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 509                           const void *data_buf, u32 data_buf_len)
 510 {
 511         struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu;
 512         struct kvec *iov;
 513         u32 niov = 0, tx_size = ISCSI_HDR_LEN;
 514         int ret;
 515
 516         iov = &cmd->iov_misc[0];
 517         iov[niov].iov_base      = cmd->pdu;
 518         iov[niov++].iov_len     = ISCSI_HDR_LEN;
 519
 520         if (conn->conn_ops->HeaderDigest) {
 521                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 522
 523                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
 524                                           ISCSI_HDR_LEN, 0, NULL,
 525                                           header_digest);
 526
 527                 iov[0].iov_len += ISCSI_CRC_LEN;
 528                 tx_size += ISCSI_CRC_LEN;
 529                 pr_debug("Attaching CRC32C HeaderDigest"
 530                          " to opcode 0x%x 0x%08x\n",
 531                          hdr->opcode, *header_digest);
 532         }
 533
 534         if (data_buf_len) {
 535                 u32 padding = ((-data_buf_len) & 3);
 536
 537                 iov[niov].iov_base      = (void *)data_buf;
 538                 iov[niov++].iov_len     = data_buf_len;
 539                 tx_size += data_buf_len;
 540
 541                 if (padding != 0) {
 542                         iov[niov].iov_base = &cmd->pad_bytes;
 543                         iov[niov++].iov_len = padding;
 544                         tx_size += padding;
 545                         pr_debug("Attaching %u additional"
 546                                  " padding bytes.\n", padding);
 547                 }
 548
 549                 if (conn->conn_ops->DataDigest) {
 550                         iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
 551                                                   data_buf, data_buf_len,
 552                                                   padding, &cmd->pad_bytes,
 553                                                   &cmd->data_crc);
 554
 555                         iov[niov].iov_base = &cmd->data_crc;
 556                         iov[niov++].iov_len = ISCSI_CRC_LEN;
 557                         tx_size += ISCSI_CRC_LEN;
 558                         pr_debug("Attached DataDigest for %u"
 559                                  " bytes opcode 0x%x, CRC 0x%08x\n",
 560                                  data_buf_len, hdr->opcode, cmd->data_crc);
 561                 }
 562         }
 563
 564         cmd->iov_misc_count = niov;
 565         cmd->tx_size = tx_size;
 566
 567         ret = iscsit_send_tx_data(cmd, conn, 1);
 568         if (ret < 0) {
 569                 iscsit_tx_thread_wait_for_tcp(conn);
 570                 return ret;
 571         }
 572
 573         return 0;
 574 }
 575
 576 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
 577                             u32 data_offset, u32 data_length);
 578 static void iscsit_unmap_iovec(struct iscsi_cmd *);
 579 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *,
 580                                     u32, u32, u32, u8 *);
 581 static int
 582 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 583                        const struct iscsi_datain *datain)
 584 {
 585         struct kvec *iov;
 586         u32 iov_count = 0, tx_size = 0;
 587         int ret, iov_ret;
 588
 589         iov = &cmd->iov_data[0];
 590         iov[iov_count].iov_base = cmd->pdu;
 591         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
 592         tx_size += ISCSI_HDR_LEN;
 593
 594         if (conn->conn_ops->HeaderDigest) {
 595                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 596
 597                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
 598                                           ISCSI_HDR_LEN, 0, NULL,
 599                                           header_digest);
 600
 601                 iov[0].iov_len += ISCSI_CRC_LEN;
 602                 tx_size += ISCSI_CRC_LEN;
 603
 604                 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n",
 605                          *header_digest);
 606         }
 607
 608         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count],
 609                                    cmd->orig_iov_data_count - (iov_count + 2),
 610                                    datain->offset, datain->length);
 611         if (iov_ret < 0)
 612                 return -1;
 613
 614         iov_count += iov_ret;
 615         tx_size += datain->length;
 616
 617         cmd->padding = ((-datain->length) & 3);
 618         if (cmd->padding) {
 619                 iov[iov_count].iov_base         = cmd->pad_bytes;
 620                 iov[iov_count++].iov_len        = cmd->padding;
 621                 tx_size += cmd->padding;
 622
 623                 pr_debug("Attaching %u padding bytes\n", cmd->padding);
 624         }
 625
 626         if (conn->conn_ops->DataDigest) {
 627                 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
 628                                                          cmd, datain->offset,
 629                                                          datain->length,
 630                                                          cmd->padding,
 631                                                          cmd->pad_bytes);
 632
 633                 iov[iov_count].iov_base = &cmd->data_crc;
 634                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
 635                 tx_size += ISCSI_CRC_LEN;
 636
 637                 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n",
 638                          datain->length + cmd->padding, cmd->data_crc);
 639         }
 640
 641         cmd->iov_data_count = iov_count;
 642         cmd->tx_size = tx_size;
 643
 644         ret = iscsit_fe_sendpage_sg(cmd, conn);
 645
 646         iscsit_unmap_iovec(cmd);
 647
 648         if (ret < 0) {
 649                 iscsit_tx_thread_wait_for_tcp(conn);
 650                 return ret;
 651         }
 652
 653         return 0;
 654 }
 655
 656 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 657                            struct iscsi_datain_req *dr, const void *buf,
 658                            u32 buf_len)
 659 {
 660         if (dr)
 661                 return iscsit_xmit_datain_pdu(conn, cmd, buf);
 662         else
 663                 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 664 }
 665
 666 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn)
 667 {
 668         return TARGET_PROT_NORMAL;
 669 }
 670
 671 static struct iscsit_transport iscsi_target_transport = {
 672         .name                   = "iSCSI/TCP",
 673         .transport_type         = ISCSI_TCP,
 674         .rdma_shutdown          = false,
 675         .owner                  = NULL,
 676         .iscsit_setup_np        = iscsit_setup_np,
 677         .iscsit_accept_np       = iscsit_accept_np,
 678         .iscsit_free_np         = iscsit_free_np,
 679         .iscsit_get_login_rx    = iscsit_get_login_rx,
 680         .iscsit_put_login_tx    = iscsit_put_login_tx,
 681         .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
 682         .iscsit_immediate_queue = iscsit_immediate_queue,
 683         .iscsit_response_queue  = iscsit_response_queue,
 684         .iscsit_queue_data_in   = iscsit_queue_rsp,
 685         .iscsit_queue_status    = iscsit_queue_rsp,
 686         .iscsit_aborted_task    = iscsit_aborted_task,
 687         .iscsit_xmit_pdu        = iscsit_xmit_pdu,
 688         .iscsit_get_rx_pdu      = iscsit_get_rx_pdu,
 689         .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops,
 690 };
 691
 692 static int __init iscsi_target_init_module(void)
 693 {
 694         int ret = 0, size;
 695
 696         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 697         iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL);
 698         if (!iscsit_global)
 699                 return -1;
 700
 701         spin_lock_init(&iscsit_global->ts_bitmap_lock);
 702         mutex_init(&auth_id_lock);
 703         idr_init(&tiqn_idr);
 704
 705         ret = target_register_template(&iscsi_ops);
 706         if (ret)
 707                 goto out;
 708
 709         size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
 710         iscsit_global->ts_bitmap = vzalloc(size);
 711         if (!iscsit_global->ts_bitmap)
 712                 goto configfs_out;
 713
 714         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 715                         sizeof(struct iscsi_queue_req),
 716                         __alignof__(struct iscsi_queue_req), 0, NULL);
 717         if (!lio_qr_cache) {
 718                 pr_err("Unable to kmem_cache_create() for"
 719                                 " lio_qr_cache\n");
 720                 goto bitmap_out;
 721         }
 722
 723         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 724                         sizeof(struct iscsi_datain_req),
 725                         __alignof__(struct iscsi_datain_req), 0, NULL);
 726         if (!lio_dr_cache) {
 727                 pr_err("Unable to kmem_cache_create() for"
 728                                 " lio_dr_cache\n");
 729                 goto qr_out;
 730         }
 731
 732         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 733                         sizeof(struct iscsi_ooo_cmdsn),
 734                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 735         if (!lio_ooo_cache) {
 736                 pr_err("Unable to kmem_cache_create() for"
 737                                 " lio_ooo_cache\n");
 738                 goto dr_out;
 739         }
 740
 741         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 742                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 743                         0, NULL);
 744         if (!lio_r2t_cache) {
 745                 pr_err("Unable to kmem_cache_create() for"
 746                                 " lio_r2t_cache\n");
 747                 goto ooo_out;
 748         }
 749
 750         iscsit_register_transport(&iscsi_target_transport);
 751
 752         if (iscsit_load_discovery_tpg() < 0)
 753                 goto r2t_out;
 754
 755         return ret;
 756 r2t_out:
 757         iscsit_unregister_transport(&iscsi_target_transport);
 758         kmem_cache_destroy(lio_r2t_cache);
 759 ooo_out:
 760         kmem_cache_destroy(lio_ooo_cache);
 761 dr_out:
 762         kmem_cache_destroy(lio_dr_cache);
 763 qr_out:
 764         kmem_cache_destroy(lio_qr_cache);
 765 bitmap_out:
 766         vfree(iscsit_global->ts_bitmap);
 767 configfs_out:
 768         /* XXX: this probably wants it to be it's own unwind step.. */
 769         if (iscsit_global->discovery_tpg)
 770                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 771         target_unregister_template(&iscsi_ops);
 772 out:
 773         kfree(iscsit_global);
 774         return -ENOMEM;
 775 }
 776
 777 static void __exit iscsi_target_cleanup_module(void)
 778 {
 779         iscsit_release_discovery_tpg();
 780         iscsit_unregister_transport(&iscsi_target_transport);
 781         kmem_cache_destroy(lio_qr_cache);
 782         kmem_cache_destroy(lio_dr_cache);
 783         kmem_cache_destroy(lio_ooo_cache);
 784         kmem_cache_destroy(lio_r2t_cache);
 785
 786         /*
 787          * Shutdown discovery sessions and disable discovery TPG
 788          */
 789         if (iscsit_global->discovery_tpg)
 790                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 791
 792         target_unregister_template(&iscsi_ops);
 793
 794         vfree(iscsit_global->ts_bitmap);
 795         kfree(iscsit_global);
 796 }
 797
 798 int iscsit_add_reject(
 799         struct iscsi_conn *conn,
 800         u8 reason,
 801         unsigned char *buf)
 802 {
 803         struct iscsi_cmd *cmd;
 804
 805         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 806         if (!cmd)
 807                 return -1;
 808
 809         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 810         cmd->reject_reason = reason;
 811
 812         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 813         if (!cmd->buf_ptr) {
 814                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 815                 iscsit_free_cmd(cmd, false);
 816                 return -1;
 817         }
 818
 819         spin_lock_bh(&conn->cmd_lock);
 820         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 821         spin_unlock_bh(&conn->cmd_lock);
 822
 823         cmd->i_state = ISTATE_SEND_REJECT;
 824         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 825
 826         return -1;
 827 }
 828 EXPORT_SYMBOL(iscsit_add_reject);
 829
 830 static int iscsit_add_reject_from_cmd(
 831         struct iscsi_cmd *cmd,
 832         u8 reason,
 833         bool add_to_conn,
 834         unsigned char *buf)
 835 {
 836         struct iscsi_conn *conn;
 837         const bool do_put = cmd->se_cmd.se_tfo != NULL;
 838
 839         if (!cmd->conn) {
 840                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 841                                 cmd->init_task_tag);
 842                 return -1;
 843         }
 844         conn = cmd->conn;
 845
 846         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 847         cmd->reject_reason = reason;
 848
 849         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 850         if (!cmd->buf_ptr) {
 851                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 852                 iscsit_free_cmd(cmd, false);
 853                 return -1;
 854         }
 855
 856         if (add_to_conn) {
 857                 spin_lock_bh(&conn->cmd_lock);
 858                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 859                 spin_unlock_bh(&conn->cmd_lock);
 860         }
 861
 862         cmd->i_state = ISTATE_SEND_REJECT;
 863         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 864         /*
 865          * Perform the kref_put now if se_cmd has already been setup by
 866          * scsit_setup_scsi_cmd()
 867          */
 868         if (do_put) {
 869                 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
 870                 target_put_sess_cmd(&cmd->se_cmd);
 871         }
 872         return -1;
 873 }
 874
 875 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason,
 876                                  unsigned char *buf)
 877 {
 878         return iscsit_add_reject_from_cmd(cmd, reason, true, buf);
 879 }
 880
 881 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf)
 882 {
 883         return iscsit_add_reject_from_cmd(cmd, reason, false, buf);
 884 }
 885 EXPORT_SYMBOL(iscsit_reject_cmd);
 886
 887 /*
 888  * Map some portion of the allocated scatterlist to an iovec, suitable for
 889  * kernel sockets to copy data in/out.
 890  */
 891 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
 892                             u32 data_offset, u32 data_length)
 893 {
 894         u32 i = 0, orig_data_length = data_length;
 895         struct scatterlist *sg;
 896         unsigned int page_off;
 897
 898         /*
 899          * We know each entry in t_data_sg contains a page.
 900          */
 901         u32 ent = data_offset / PAGE_SIZE;
 902
 903         if (!data_length)
 904                 return 0;
 905
 906         if (ent >= cmd->se_cmd.t_data_nents) {
 907                 pr_err("Initial page entry out-of-bounds\n");
 908                 goto overflow;
 909         }
 910
 911         sg = &cmd->se_cmd.t_data_sg[ent];
 912         page_off = (data_offset % PAGE_SIZE);
 913
 914         cmd->first_data_sg = sg;
 915         cmd->first_data_sg_off = page_off;
 916
 917         while (data_length) {
 918                 u32 cur_len;
 919
 920                 if (WARN_ON_ONCE(!sg || i >= nvec))
 921                         goto overflow;
 922
 923                 cur_len = min_t(u32, data_length, sg->length - page_off);
 924
 925                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 926                 iov[i].iov_len = cur_len;
 927
 928                 data_length -= cur_len;
 929                 page_off = 0;
 930                 sg = sg_next(sg);
 931                 i++;
 932         }
 933
 934         cmd->kmapped_nents = i;
 935
 936         return i;
 937
 938 overflow:
 939         pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n",
 940                data_offset, orig_data_length, i, nvec);
 941         for_each_sg(cmd->se_cmd.t_data_sg, sg,
 942                     cmd->se_cmd.t_data_nents, i) {
 943                 pr_err("[%d] off %d len %d\n",
 944                        i, sg->offset, sg->length);
 945         }
 946         return -1;
 947 }
 948
 949 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 950 {
 951         u32 i;
 952         struct scatterlist *sg;
 953
 954         sg = cmd->first_data_sg;
 955
 956         for (i = 0; i < cmd->kmapped_nents; i++)
 957                 kunmap(sg_page(&sg[i]));
 958 }
 959
 960 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 961 {
 962         LIST_HEAD(ack_list);
 963         struct iscsi_cmd *cmd, *cmd_p;
 964
 965         conn->exp_statsn = exp_statsn;
 966
 967         if (conn->sess->sess_ops->RDMAExtensions)
 968                 return;
 969
 970         spin_lock_bh(&conn->cmd_lock);
 971         list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) {
 972                 spin_lock(&cmd->istate_lock);
 973                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 974                     iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
 975                         cmd->i_state = ISTATE_REMOVE;
 976                         spin_unlock(&cmd->istate_lock);
 977                         list_move_tail(&cmd->i_conn_node, &ack_list);
 978                         continue;
 979                 }
 980                 spin_unlock(&cmd->istate_lock);
 981         }
 982         spin_unlock_bh(&conn->cmd_lock);
 983
 984         list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) {
 985                 list_del_init(&cmd->i_conn_node);
 986                 iscsit_free_cmd(cmd, false);
 987         }
 988 }
 989
 990 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 991 {
 992         u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
 993
 994         iov_count += ISCSI_IOV_DATA_BUFFER;
 995         cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL);
 996         if (!cmd->iov_data)
 997                 return -ENOMEM;
 998
 999         cmd->orig_iov_data_count = iov_count;
1000         return 0;
1001 }
1002
1003 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1004                           unsigned char *buf)
1005 {
1006         int data_direction, payload_length;
1007         struct iscsi_scsi_req *hdr;
1008         int iscsi_task_attr;
1009         int sam_task_attr;
1010
1011         atomic_long_inc(&conn->sess->cmd_pdus);
1012
1013         hdr                     = (struct iscsi_scsi_req *) buf;
1014         payload_length          = ntoh24(hdr->dlength);
1015
1016         /* FIXME; Add checks for AdditionalHeaderSegment */
1017
1018         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
1019             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1020                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
1021                                 " not set. Bad iSCSI Initiator.\n");
1022                 return iscsit_add_reject_cmd(cmd,
1023                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1024         }
1025
1026         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
1027              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
1028                 /*
1029                  * From RFC-3720 Section 10.3.1:
1030                  *
1031                  * "Either or both of R and W MAY be 1 when either the
1032                  *  Expected Data Transfer Length and/or Bidirectional Read
1033                  *  Expected Data Transfer Length are 0"
1034                  *
1035                  * For this case, go ahead and clear the unnecssary bits
1036                  * to avoid any confusion with ->data_direction.
1037                  */
1038                 hdr->flags &= ~ISCSI_FLAG_CMD_READ;
1039                 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
1040
1041                 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
1042                         " set when Expected Data Transfer Length is 0 for"
1043                         " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]);
1044         }
1045
1046         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
1047             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
1048                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
1049                         " MUST be set if Expected Data Transfer Length is not 0."
1050                         " Bad iSCSI Initiator\n");
1051                 return iscsit_add_reject_cmd(cmd,
1052                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1053         }
1054
1055         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
1056             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
1057                 pr_err("Bidirectional operations not supported!\n");
1058                 return iscsit_add_reject_cmd(cmd,
1059                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1060         }
1061
1062         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1063                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
1064                                 " Scsi Command PDU.\n");
1065                 return iscsit_add_reject_cmd(cmd,
1066                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1067         }
1068
1069         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
1070                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
1071                         " protocol error.\n", payload_length);
1072                 return iscsit_add_reject_cmd(cmd,
1073                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1074         }
1075
1076         if ((be32_to_cpu(hdr->data_length) == payload_length) &&
1077             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
1078                 pr_err("Expected Data Transfer Length and Length of"
1079                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
1080                         " bit is not set protocol error\n");
1081                 return iscsit_add_reject_cmd(cmd,
1082                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1083         }
1084
1085         if (payload_length > be32_to_cpu(hdr->data_length)) {
1086                 pr_err("DataSegmentLength: %u is greater than"
1087                         " EDTL: %u, protocol error.\n", payload_length,
1088                                 hdr->data_length);
1089                 return iscsit_add_reject_cmd(cmd,
1090                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1091         }
1092
1093         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1094                 pr_err("DataSegmentLength: %u is greater than"
1095                         " MaxXmitDataSegmentLength: %u, protocol error.\n",
1096                         payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
1097                 return iscsit_add_reject_cmd(cmd,
1098                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1099         }
1100
1101         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
1102                 pr_err("DataSegmentLength: %u is greater than"
1103                         " FirstBurstLength: %u, protocol error.\n",
1104                         payload_length, conn->sess->sess_ops->FirstBurstLength);
1105                 return iscsit_add_reject_cmd(cmd,
1106                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1107         }
1108
1109         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
1110                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
1111                           DMA_NONE;
1112
1113         cmd->data_direction = data_direction;
1114         iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
1115         /*
1116          * Figure out the SAM Task Attribute for the incoming SCSI CDB
1117          */
1118         if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
1119             (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
1120                 sam_task_attr = TCM_SIMPLE_TAG;
1121         else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
1122                 sam_task_attr = TCM_ORDERED_TAG;
1123         else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
1124                 sam_task_attr = TCM_HEAD_TAG;
1125         else if (iscsi_task_attr == ISCSI_ATTR_ACA)
1126                 sam_task_attr = TCM_ACA_TAG;
1127         else {
1128                 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
1129                         " TCM_SIMPLE_TAG\n", iscsi_task_attr);
1130                 sam_task_attr = TCM_SIMPLE_TAG;
1131         }
1132
1133         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
1134         cmd->i_state            = ISTATE_NEW_CMD;
1135         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1136         cmd->immediate_data     = (payload_length) ? 1 : 0;
1137         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
1138                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
1139         if (cmd->unsolicited_data)
1140                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
1141
1142         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1143         if (hdr->flags & ISCSI_FLAG_CMD_READ)
1144                 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
1145         else
1146                 cmd->targ_xfer_tag = 0xFFFFFFFF;
1147         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1148         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1149         cmd->first_burst_len    = payload_length;
1150
1151         if (!conn->sess->sess_ops->RDMAExtensions &&
1152              cmd->data_direction == DMA_FROM_DEVICE) {
1153                 struct iscsi_datain_req *dr;
1154
1155                 dr = iscsit_allocate_datain_req();
1156                 if (!dr)
1157                         return iscsit_add_reject_cmd(cmd,
1158                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1159
1160                 iscsit_attach_datain_req(cmd, dr);
1161         }
1162
1163         /*
1164          * Initialize struct se_cmd descriptor from target_core_mod infrastructure
1165          */
1166         transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
1167                         conn->sess->se_sess, be32_to_cpu(hdr->data_length),
1168                         cmd->data_direction, sam_task_attr,
1169                         cmd->sense_buffer + 2);
1170
1171         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
1172                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
1173                 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
1174                 conn->cid);
1175
1176         target_get_sess_cmd(&cmd->se_cmd, true);
1177
1178         cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
1179                                                      scsilun_to_int(&hdr->lun));
1180         if (cmd->sense_reason)
1181                 goto attach_cmd;
1182
1183         /* only used for printks or comparing with ->ref_task_tag */
1184         cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
1185         cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb);
1186         if (cmd->sense_reason) {
1187                 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
1188                         return iscsit_add_reject_cmd(cmd,
1189                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1190                 }
1191
1192                 goto attach_cmd;
1193         }
1194
1195         if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
1196                 return iscsit_add_reject_cmd(cmd,
1197                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1198         }
1199
1200 attach_cmd:
1201         spin_lock_bh(&conn->cmd_lock);
1202         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1203         spin_unlock_bh(&conn->cmd_lock);
1204         /*
1205          * Check if we need to delay processing because of ALUA
1206          * Active/NonOptimized primary access state..
1207          */
1208         core_alua_check_nonop_delay(&cmd->se_cmd);
1209
1210         return 0;
1211 }
1212 EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
1213
1214 void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd)
1215 {
1216         iscsit_set_dataout_sequence_values(cmd);
1217
1218         spin_lock_bh(&cmd->dataout_timeout_lock);
1219         iscsit_start_dataout_timer(cmd, cmd->conn);
1220         spin_unlock_bh(&cmd->dataout_timeout_lock);
1221 }
1222 EXPORT_SYMBOL(iscsit_set_unsolicited_dataout);
1223
1224 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1225                             struct iscsi_scsi_req *hdr)
1226 {
1227         int cmdsn_ret = 0;
1228         /*
1229          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1230          * the Immediate Bit is not set, and no Immediate
1231          * Data is attached.
1232          *
1233          * A PDU/CmdSN carrying Immediate Data can only
1234          * be processed after the DataCRC has passed.
1235          * If the DataCRC fails, the CmdSN MUST NOT
1236          * be acknowledged. (See below)
1237          */
1238         if (!cmd->immediate_data) {
1239                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1240                                         (unsigned char *)hdr, hdr->cmdsn);
1241                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1242                         return -1;
1243                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1244                         target_put_sess_cmd(&cmd->se_cmd);
1245                         return 0;
1246                 }
1247         }
1248
1249         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1250
1251         /*
1252          * If no Immediate Data is attached, it's OK to return now.
1253          */
1254         if (!cmd->immediate_data) {
1255                 if (!cmd->sense_reason && cmd->unsolicited_data)
1256                         iscsit_set_unsolicited_dataout(cmd);
1257                 if (!cmd->sense_reason)
1258                         return 0;
1259
1260                 target_put_sess_cmd(&cmd->se_cmd);
1261                 return 0;
1262         }
1263
1264         /*
1265          * Early CHECK_CONDITIONs with ImmediateData never make it to command
1266          * execution.  These exceptions are processed in CmdSN order using
1267          * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
1268          */
1269         if (cmd->sense_reason)
1270                 return 1;
1271         /*
1272          * Call directly into transport_generic_new_cmd() to perform
1273          * the backend memory allocation.
1274          */
1275         cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
1276         if (cmd->sense_reason)
1277                 return 1;
1278
1279         return 0;
1280 }
1281 EXPORT_SYMBOL(iscsit_process_scsi_cmd);
1282
1283 static int
1284 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
1285                           bool dump_payload)
1286 {
1287         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1288         int rc;
1289
1290         /*
1291          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
1292          */
1293         if (dump_payload) {
1294                 u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done,
1295                                  cmd->first_burst_len);
1296
1297                 pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n",
1298                          cmd->se_cmd.data_length, cmd->write_data_done,
1299                          cmd->first_burst_len, length);
1300                 rc = iscsit_dump_data_payload(cmd->conn, length, 1);
1301                 pr_debug("Finished dumping immediate data\n");
1302                 if (rc < 0)
1303                         immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER;
1304         } else {
1305                 immed_ret = iscsit_handle_immediate_data(cmd, hdr,
1306                                                          cmd->first_burst_len);
1307         }
1308
1309         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1310                 /*
1311                  * A PDU/CmdSN carrying Immediate Data passed
1312                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1313                  * Immediate Bit is not set.
1314                  */
1315                 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd,
1316                                         (unsigned char *)hdr, hdr->cmdsn);
1317                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1318                         return -1;
1319
1320                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1321                         target_put_sess_cmd(&cmd->se_cmd);
1322
1323                         return 0;
1324                 } else if (cmd->unsolicited_data)
1325                         iscsit_set_unsolicited_dataout(cmd);
1326
1327         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1328                 /*
1329                  * Immediate Data failed DataCRC and ERL>=1,
1330                  * silently drop this PDU and let the initiator
1331                  * plug the CmdSN gap.
1332                  *
1333                  * FIXME: Send Unsolicited NOPIN with reserved
1334                  * TTT here to help the initiator figure out
1335                  * the missing CmdSN, although they should be
1336                  * intelligent enough to determine the missing
1337                  * CmdSN and issue a retry to plug the sequence.
1338                  */
1339                 cmd->i_state = ISTATE_REMOVE;
1340                 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
1341         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1342                 return -1;
1343
1344         return 0;
1345 }
1346
1347 static int
1348 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1349                            unsigned char *buf)
1350 {
1351         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1352         int rc, immed_data;
1353         bool dump_payload = false;
1354
1355         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1356         if (rc < 0)
1357                 return 0;
1358         /*
1359          * Allocation iovecs needed for struct socket operations for
1360          * traditional iSCSI block I/O.
1361          */
1362         if (iscsit_allocate_iovecs(cmd) < 0) {
1363                 return iscsit_reject_cmd(cmd,
1364                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1365         }
1366         immed_data = cmd->immediate_data;
1367
1368         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1369         if (rc < 0)
1370                 return rc;
1371         else if (rc > 0)
1372                 dump_payload = true;
1373
1374         if (!immed_data)
1375                 return 0;
1376
1377         return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1378 }
1379
1380 static u32 iscsit_do_crypto_hash_sg(
1381         struct ahash_request *hash,
1382         struct iscsi_cmd *cmd,
1383         u32 data_offset,
1384         u32 data_length,
1385         u32 padding,
1386         u8 *pad_bytes)
1387 {
1388         u32 data_crc;
1389         struct scatterlist *sg;
1390         unsigned int page_off;
1391
1392         crypto_ahash_init(hash);
1393
1394         sg = cmd->first_data_sg;
1395         page_off = cmd->first_data_sg_off;
1396
1397         while (data_length) {
1398                 u32 cur_len = min_t(u32, data_length, (sg->length - page_off));
1399
1400                 ahash_request_set_crypt(hash, sg, NULL, cur_len);
1401                 crypto_ahash_update(hash);
1402
1403                 data_length -= cur_len;
1404                 page_off = 0;
1405                 /* iscsit_map_iovec has already checked for invalid sg pointers */
1406                 sg = sg_next(sg);
1407         }
1408
1409         if (padding) {
1410                 struct scatterlist pad_sg;
1411
1412                 sg_init_one(&pad_sg, pad_bytes, padding);
1413                 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
1414                                         padding);
1415                 crypto_ahash_finup(hash);
1416         } else {
1417                 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
1418                 crypto_ahash_final(hash);
1419         }
1420
1421         return data_crc;
1422 }
1423
1424 static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,
1425         const void *buf, u32 payload_length, u32 padding,
1426         const void *pad_bytes, void *data_crc)
1427 {
1428         struct scatterlist sg[2];
1429
1430         sg_init_table(sg, ARRAY_SIZE(sg));
1431         sg_set_buf(sg, buf, payload_length);
1432         if (padding)
1433                 sg_set_buf(sg + 1, pad_bytes, padding);
1434
1435         ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
1436
1437         crypto_ahash_digest(hash);
1438 }
1439
1440 int
1441 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1442                            struct iscsi_cmd *cmd, u32 payload_length,
1443                            bool *success)
1444 {
1445         struct iscsi_data *hdr = buf;
1446         struct se_cmd *se_cmd;
1447         int rc;
1448
1449         /* iSCSI write */
1450         atomic_long_add(payload_length, &conn->sess->rx_data_octets);
1451
1452         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1453                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1454                 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
1455                 payload_length, conn->cid);
1456
1457         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1458                 pr_err("Command ITT: 0x%08x received DataOUT after"
1459                         " last DataOUT received, dumping payload\n",
1460                         cmd->init_task_tag);
1461                 return iscsit_dump_data_payload(conn, payload_length, 1);
1462         }
1463
1464         if (cmd->data_direction != DMA_TO_DEVICE) {
1465                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1466                         " NON-WRITE command.\n", cmd->init_task_tag);
1467                 return iscsit_dump_data_payload(conn, payload_length, 1);
1468         }
1469         se_cmd = &cmd->se_cmd;
1470         iscsit_mod_dataout_timer(cmd);
1471
1472         if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
1473                 pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n",
1474                        be32_to_cpu(hdr->offset), payload_length,
1475                        cmd->se_cmd.data_length);
1476                 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf);
1477         }
1478
1479         if (cmd->unsolicited_data) {
1480                 int dump_unsolicited_data = 0;
1481
1482                 if (conn->sess->sess_ops->InitialR2T) {
1483                         pr_err("Received unexpected unsolicited data"
1484                                 " while InitialR2T=Yes, protocol error.\n");
1485                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1486                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1487                         return -1;
1488                 }
1489                 /*
1490                  * Special case for dealing with Unsolicited DataOUT
1491                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1492                  * failures;
1493                  */
1494
1495                 /* Something's amiss if we're not in WRITE_PENDING state... */
1496                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1497                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
1498                         dump_unsolicited_data = 1;
1499
1500                 if (dump_unsolicited_data) {
1501                         /*
1502                          * Check if a delayed TASK_ABORTED status needs to
1503                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1504                          * received with the unsolicited data out.
1505                          */
1506                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1507                                 iscsit_stop_dataout_timer(cmd);
1508
1509                         return iscsit_dump_data_payload(conn, payload_length, 1);
1510                 }
1511         } else {
1512                 /*
1513                  * For the normal solicited data path:
1514                  *
1515                  * Check for a delayed TASK_ABORTED status and dump any
1516                  * incoming data out payload if one exists.  Also, when the
1517                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1518                  * data out sequence, we decrement outstanding_r2ts.  Once
1519                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1520                  * TASK_ABORTED status.
1521                  */
1522                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1523                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL &&
1524                             --cmd->outstanding_r2ts < 1)
1525                                 iscsit_stop_dataout_timer(cmd);
1526
1527                         return iscsit_dump_data_payload(conn, payload_length, 1);
1528                 }
1529         }
1530         /*
1531          * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1532          * within-command recovery checks before receiving the payload.
1533          */
1534         rc = iscsit_check_pre_dataout(cmd, buf);
1535         if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
1536                 return 0;
1537         else if (rc == DATAOUT_CANNOT_RECOVER)
1538                 return -1;
1539         *success = true;
1540         return 0;
1541 }
1542 EXPORT_SYMBOL(__iscsit_check_dataout_hdr);
1543
1544 int
1545 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1546                          struct iscsi_cmd **out_cmd)
1547 {
1548         struct iscsi_data *hdr = buf;
1549         struct iscsi_cmd *cmd;
1550         u32 payload_length = ntoh24(hdr->dlength);
1551         int rc;
1552         bool success = false;
1553
1554         if (!payload_length) {
1555                 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n");
1556                 return 0;
1557         }
1558
1559         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1560                 pr_err_ratelimited("DataSegmentLength: %u is greater than"
1561                         " MaxXmitDataSegmentLength: %u\n", payload_length,
1562                         conn->conn_ops->MaxXmitDataSegmentLength);
1563                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
1564         }
1565
1566         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length);
1567         if (!cmd)
1568                 return 0;
1569
1570         rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success);
1571
1572         if (success)
1573                 *out_cmd = cmd;
1574
1575         return rc;
1576 }
1577 EXPORT_SYMBOL(iscsit_check_dataout_hdr);
1578
1579 static int
1580 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1581                    struct iscsi_data *hdr)
1582 {
1583         struct kvec *iov;
1584         u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
1585         u32 payload_length;
1586         int iov_ret, data_crc_failed = 0;
1587
1588         payload_length = min_t(u32, cmd->se_cmd.data_length,
1589                                ntoh24(hdr->dlength));
1590         rx_size += payload_length;
1591         iov = &cmd->iov_data[0];
1592
1593         iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2,
1594                                    be32_to_cpu(hdr->offset), payload_length);
1595         if (iov_ret < 0)
1596                 return -1;
1597
1598         iov_count += iov_ret;
1599
1600         padding = ((-payload_length) & 3);
1601         if (padding != 0) {
1602                 iov[iov_count].iov_base = cmd->pad_bytes;
1603                 iov[iov_count++].iov_len = padding;
1604                 rx_size += padding;
1605                 pr_debug("Receiving %u padding bytes.\n", padding);
1606         }
1607
1608         if (conn->conn_ops->DataDigest) {
1609                 iov[iov_count].iov_base = &checksum;
1610                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1611                 rx_size += ISCSI_CRC_LEN;
1612         }
1613
1614         WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
1615         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1616
1617         iscsit_unmap_iovec(cmd);
1618
1619         if (rx_got != rx_size)
1620                 return -1;
1621
1622         if (conn->conn_ops->DataDigest) {
1623                 u32 data_crc;
1624
1625                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
1626                                                     be32_to_cpu(hdr->offset),
1627                                                     payload_length, padding,
1628                                                     cmd->pad_bytes);
1629
1630                 if (checksum != data_crc) {
1631                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1632                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1633                                 " does not match computed 0x%08x\n",
1634                                 hdr->itt, hdr->offset, payload_length,
1635                                 hdr->datasn, checksum, data_crc);
1636                         data_crc_failed = 1;
1637                 } else {
1638                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1639                                 " %u bytes of Data Out\n", checksum,
1640                                 payload_length);
1641                 }
1642         }
1643
1644         return data_crc_failed;
1645 }
1646
1647 int
1648 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
1649                              bool data_crc_failed)
1650 {
1651         struct iscsi_conn *conn = cmd->conn;
1652         int rc, ooo_cmdsn;
1653         /*
1654          * Increment post receive data and CRC values or perform
1655          * within-command recovery.
1656          */
1657         rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
1658         if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
1659                 return 0;
1660         else if (rc == DATAOUT_SEND_R2T) {
1661                 iscsit_set_dataout_sequence_values(cmd);
1662                 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
1663         } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
1664                 /*
1665                  * Handle extra special case for out of order
1666                  * Unsolicited Data Out.
1667                  */
1668                 spin_lock_bh(&cmd->istate_lock);
1669                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1670                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1671                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1672                 spin_unlock_bh(&cmd->istate_lock);
1673
1674                 iscsit_stop_dataout_timer(cmd);
1675                 if (ooo_cmdsn)
1676                         return 0;
1677                 target_execute_cmd(&cmd->se_cmd);
1678                 return 0;
1679         } else /* DATAOUT_CANNOT_RECOVER */
1680                 return -1;
1681
1682         return 0;
1683 }
1684 EXPORT_SYMBOL(iscsit_check_dataout_payload);
1685
1686 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1687 {
1688         struct iscsi_cmd *cmd = NULL;
1689         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1690         int rc;
1691         bool data_crc_failed = false;
1692
1693         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1694         if (rc < 0)
1695                 return 0;
1696         else if (!cmd)
1697                 return 0;
1698
1699         rc = iscsit_get_dataout(conn, cmd, hdr);
1700         if (rc < 0)
1701                 return rc;
1702         else if (rc > 0)
1703                 data_crc_failed = true;
1704
1705         return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
1706 }
1707
1708 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1709                          struct iscsi_nopout *hdr)
1710 {
1711         u32 payload_length = ntoh24(hdr->dlength);
1712
1713         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1714                 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n");
1715                 if (!cmd)
1716                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1717                                                  (unsigned char *)hdr);
1718
1719                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1720                                          (unsigned char *)hdr);
1721         }
1722
1723         if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1724                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1725                         " not set, protocol error.\n");
1726                 if (!cmd)
1727                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1728                                                  (unsigned char *)hdr);
1729
1730                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1731                                          (unsigned char *)hdr);
1732         }
1733
1734         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1735                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1736                         " greater than MaxXmitDataSegmentLength: %u, protocol"
1737                         " error.\n", payload_length,
1738                         conn->conn_ops->MaxXmitDataSegmentLength);
1739                 if (!cmd)
1740                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1741                                                  (unsigned char *)hdr);
1742
1743                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1744                                          (unsigned char *)hdr);
1745         }
1746
1747         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
1748                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1749                 hdr->itt == RESERVED_ITT ? "Response" : "Request",
1750                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1751                 payload_length);
1752         /*
1753          * This is not a response to a Unsolicited NopIN, which means
1754          * it can either be a NOPOUT ping request (with a valid ITT),
1755          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1756          * Either way, make sure we allocate an struct iscsi_cmd, as both
1757          * can contain ping data.
1758          */
1759         if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1760                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1761                 cmd->i_state            = ISTATE_SEND_NOPIN;
1762                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1763                                                 1 : 0);
1764                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1765                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1766                 cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1767                 cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1768                 cmd->data_direction     = DMA_NONE;
1769         }
1770
1771         return 0;
1772 }
1773 EXPORT_SYMBOL(iscsit_setup_nop_out);
1774
1775 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1776                            struct iscsi_nopout *hdr)
1777 {
1778         struct iscsi_cmd *cmd_p = NULL;
1779         int cmdsn_ret = 0;
1780         /*
1781          * Initiator is expecting a NopIN ping reply..
1782          */
1783         if (hdr->itt != RESERVED_ITT) {
1784                 if (!cmd)
1785                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1786                                                 (unsigned char *)hdr);
1787
1788                 spin_lock_bh(&conn->cmd_lock);
1789                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1790                 spin_unlock_bh(&conn->cmd_lock);
1791
1792                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1793
1794                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1795                         iscsit_add_cmd_to_response_queue(cmd, conn,
1796                                                          cmd->i_state);
1797                         return 0;
1798                 }
1799
1800                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1801                                 (unsigned char *)hdr, hdr->cmdsn);
1802                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1803                         return 0;
1804                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1805                         return -1;
1806
1807                 return 0;
1808         }
1809         /*
1810          * This was a response to a unsolicited NOPIN ping.
1811          */
1812         if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1813                 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
1814                 if (!cmd_p)
1815                         return -EINVAL;
1816
1817                 iscsit_stop_nopin_response_timer(conn);
1818
1819                 cmd_p->i_state = ISTATE_REMOVE;
1820                 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
1821
1822                 iscsit_start_nopin_timer(conn);
1823                 return 0;
1824         }
1825         /*
1826          * Otherwise, initiator is not expecting a NOPIN is response.
1827          * Just ignore for now.
1828          */
1829
1830         if (cmd)
1831                 iscsit_free_cmd(cmd, false);
1832
1833         return 0;
1834 }
1835 EXPORT_SYMBOL(iscsit_process_nop_out);
1836
1837 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1838                                  unsigned char *buf)
1839 {
1840         unsigned char *ping_data = NULL;
1841         struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1842         struct kvec *iov = NULL;
1843         u32 payload_length = ntoh24(hdr->dlength);
1844         int ret;
1845
1846         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1847         if (ret < 0)
1848                 return 0;
1849         /*
1850          * Handle NOP-OUT payload for traditional iSCSI sockets
1851          */
1852         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1853                 u32 checksum, data_crc, padding = 0;
1854                 int niov = 0, rx_got, rx_size = payload_length;
1855
1856                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1857                 if (!ping_data) {
1858                         ret = -1;
1859                         goto out;
1860                 }
1861
1862                 iov = &cmd->iov_misc[0];
1863                 iov[niov].iov_base      = ping_data;
1864                 iov[niov++].iov_len     = payload_length;
1865
1866                 padding = ((-payload_length) & 3);
1867                 if (padding != 0) {
1868                         pr_debug("Receiving %u additional bytes"
1869                                 " for padding.\n", padding);
1870                         iov[niov].iov_base      = &cmd->pad_bytes;
1871                         iov[niov++].iov_len     = padding;
1872                         rx_size += padding;
1873                 }
1874                 if (conn->conn_ops->DataDigest) {
1875                         iov[niov].iov_base      = &checksum;
1876                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1877                         rx_size += ISCSI_CRC_LEN;
1878                 }
1879
1880                 WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc));
1881                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1882                 if (rx_got != rx_size) {
1883                         ret = -1;
1884                         goto out;
1885                 }
1886
1887                 if (conn->conn_ops->DataDigest) {
1888                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,
1889                                                   payload_length, padding,
1890                                                   cmd->pad_bytes, &data_crc);
1891
1892                         if (checksum != data_crc) {
1893                                 pr_err("Ping data CRC32C DataDigest"
1894                                 " 0x%08x does not match computed 0x%08x\n",
1895                                         checksum, data_crc);
1896                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1897                                         pr_err("Unable to recover from"
1898                                         " NOPOUT Ping DataCRC failure while in"
1899                                                 " ERL=0.\n");
1900                                         ret = -1;
1901                                         goto out;
1902                                 } else {
1903                                         /*
1904                                          * Silently drop this PDU and let the
1905                                          * initiator plug the CmdSN gap.
1906                                          */
1907                                         pr_debug("Dropping NOPOUT"
1908                                         " Command CmdSN: 0x%08x due to"
1909                                         " DataCRC error.\n", hdr->cmdsn);
1910                                         ret = 0;
1911                                         goto out;
1912                                 }
1913                         } else {
1914                                 pr_debug("Got CRC32C DataDigest"
1915                                 " 0x%08x for %u bytes of ping data.\n",
1916                                         checksum, payload_length);
1917                         }
1918                 }
1919
1920                 ping_data[payload_length] = '\0';
1921                 /*
1922                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1923                  */
1924                 cmd->buf_ptr = ping_data;
1925                 cmd->buf_ptr_size = payload_length;
1926
1927                 pr_debug("Got %u bytes of NOPOUT ping"
1928                         " data.\n", payload_length);
1929                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1930         }
1931
1932         return iscsit_process_nop_out(conn, cmd, hdr);
1933 out:
1934         if (cmd)
1935                 iscsit_free_cmd(cmd, false);
1936
1937         kfree(ping_data);
1938         return ret;
1939 }
1940
1941 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf)
1942 {
1943         switch (iscsi_tmf) {
1944         case ISCSI_TM_FUNC_ABORT_TASK:
1945                 return TMR_ABORT_TASK;
1946         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1947                 return TMR_ABORT_TASK_SET;
1948         case ISCSI_TM_FUNC_CLEAR_ACA:
1949                 return TMR_CLEAR_ACA;
1950         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1951                 return TMR_CLEAR_TASK_SET;
1952         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1953                 return TMR_LUN_RESET;
1954         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1955                 return TMR_TARGET_WARM_RESET;
1956         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1957                 return TMR_TARGET_COLD_RESET;
1958         default:
1959                 return TMR_UNKNOWN;
1960         }
1961 }
1962
1963 int
1964 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1965                            unsigned char *buf)
1966 {
1967         struct se_tmr_req *se_tmr;
1968         struct iscsi_tmr_req *tmr_req;
1969         struct iscsi_tm *hdr;
1970         int out_of_order_cmdsn = 0, ret;
1971         u8 function, tcm_function = TMR_UNKNOWN;
1972
1973         hdr                     = (struct iscsi_tm *) buf;
1974         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1975         function = hdr->flags;
1976
1977         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1978                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1979                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1980                 hdr->rtt, hdr->refcmdsn, conn->cid);
1981
1982         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1983             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1984              hdr->rtt != RESERVED_ITT)) {
1985                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1986                 hdr->rtt = RESERVED_ITT;
1987         }
1988
1989         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1990                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1991                 pr_err("Task Management Request TASK_REASSIGN not"
1992                         " issued as immediate command, bad iSCSI Initiator"
1993                                 "implementation\n");
1994                 return iscsit_add_reject_cmd(cmd,
1995                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1996         }
1997         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1998             be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
1999                 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
2000
2001         cmd->data_direction = DMA_NONE;
2002         cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL);
2003         if (!cmd->tmr_req) {
2004                 return iscsit_add_reject_cmd(cmd,
2005                                              ISCSI_REASON_BOOKMARK_NO_RESOURCES,
2006                                              buf);
2007         }
2008
2009         transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
2010                               conn->sess->se_sess, 0, DMA_NONE,
2011                               TCM_SIMPLE_TAG, cmd->sense_buffer + 2);
2012
2013         target_get_sess_cmd(&cmd->se_cmd, true);
2014
2015         /*
2016          * TASK_REASSIGN for ERL=2 / connection stays inside of
2017          * LIO-Target $FABRIC_MOD
2018          */
2019         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2020                 tcm_function = iscsit_convert_tmf(function);
2021                 if (tcm_function == TMR_UNKNOWN) {
2022                         pr_err("Unknown iSCSI TMR Function:"
2023                                " 0x%02x\n", function);
2024                         return iscsit_add_reject_cmd(cmd,
2025                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2026                 }
2027         }
2028         ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function,
2029                                  GFP_KERNEL);
2030         if (ret < 0)
2031                 return iscsit_add_reject_cmd(cmd,
2032                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2033
2034         cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
2035
2036         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
2037         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
2038         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2039         cmd->init_task_tag      = hdr->itt;
2040         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2041         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2042         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2043         se_tmr                  = cmd->se_cmd.se_tmr_req;
2044         tmr_req                 = cmd->tmr_req;
2045         /*
2046          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
2047          */
2048         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2049                 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
2050                                                scsilun_to_int(&hdr->lun));
2051                 if (ret < 0) {
2052                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
2053                         goto attach;
2054                 }
2055         }
2056
2057         switch (function) {
2058         case ISCSI_TM_FUNC_ABORT_TASK:
2059                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
2060                 if (se_tmr->response)
2061                         goto attach;
2062                 break;
2063         case ISCSI_TM_FUNC_ABORT_TASK_SET:
2064         case ISCSI_TM_FUNC_CLEAR_ACA:
2065         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
2066         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
2067                 break;
2068         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
2069                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
2070                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2071                         goto attach;
2072                 }
2073                 break;
2074         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
2075                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
2076                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2077                         goto attach;
2078                 }
2079                 break;
2080         case ISCSI_TM_FUNC_TASK_REASSIGN:
2081                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
2082                 /*
2083                  * Perform sanity checks on the ExpDataSN only if the
2084                  * TASK_REASSIGN was successful.
2085                  */
2086                 if (se_tmr->response)
2087                         break;
2088
2089                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
2090                         return iscsit_add_reject_cmd(cmd,
2091                                         ISCSI_REASON_BOOKMARK_INVALID, buf);
2092                 break;
2093         default:
2094                 pr_err("Unknown TMR function: 0x%02x, protocol"
2095                         " error.\n", function);
2096                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
2097                 goto attach;
2098         }
2099
2100         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
2101             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
2102                 se_tmr->call_transport = 1;
2103 attach:
2104         spin_lock_bh(&conn->cmd_lock);
2105         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2106         spin_unlock_bh(&conn->cmd_lock);
2107
2108         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2109                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2110                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) {
2111                         out_of_order_cmdsn = 1;
2112                 } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2113                         target_put_sess_cmd(&cmd->se_cmd);
2114                         return 0;
2115                 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2116                         return -1;
2117                 }
2118         }
2119         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2120
2121         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2122                 return 0;
2123         /*
2124          * Found the referenced task, send to transport for processing.
2125          */
2126         if (se_tmr->call_transport)
2127                 return transport_generic_handle_tmr(&cmd->se_cmd);
2128
2129         /*
2130          * Could not find the referenced LUN, task, or Task Management
2131          * command not authorized or supported.  Change state and
2132          * let the tx_thread send the response.
2133          *
2134          * For connection recovery, this is also the default action for
2135          * TMR TASK_REASSIGN.
2136          */
2137         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2138         target_put_sess_cmd(&cmd->se_cmd);
2139         return 0;
2140 }
2141 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
2142
2143 /* #warning FIXME: Support Text Command parameters besides SendTargets */
2144 int
2145 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2146                       struct iscsi_text *hdr)
2147 {
2148         u32 payload_length = ntoh24(hdr->dlength);
2149
2150         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
2151                 pr_err("Unable to accept text parameter length: %u"
2152                         "greater than MaxXmitDataSegmentLength %u.\n",
2153                        payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
2154                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2155                                          (unsigned char *)hdr);
2156         }
2157
2158         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) ||
2159              (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) {
2160                 pr_err("Multi sequence text commands currently not supported\n");
2161                 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED,
2162                                         (unsigned char *)hdr);
2163         }
2164
2165         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
2166                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
2167                 hdr->exp_statsn, payload_length);
2168
2169         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
2170         cmd->i_state            = ISTATE_SEND_TEXTRSP;
2171         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2172         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2173         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2174         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2175         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2176         cmd->data_direction     = DMA_NONE;
2177         kfree(cmd->text_in_ptr);
2178         cmd->text_in_ptr        = NULL;
2179
2180         return 0;
2181 }
2182 EXPORT_SYMBOL(iscsit_setup_text_cmd);
2183
2184 int
2185 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2186                         struct iscsi_text *hdr)
2187 {
2188         unsigned char *text_in = cmd->text_in_ptr, *text_ptr;
2189         int cmdsn_ret;
2190
2191         if (!text_in) {
2192                 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt);
2193                 if (cmd->targ_xfer_tag == 0xFFFFFFFF) {
2194                         pr_err("Unable to locate text_in buffer for sendtargets"
2195                                " discovery\n");
2196                         goto reject;
2197                 }
2198                 goto empty_sendtargets;
2199         }
2200         if (strncmp("SendTargets", text_in, 11) != 0) {
2201                 pr_err("Received Text Data that is not"
2202                         " SendTargets, cannot continue.\n");
2203                 goto reject;
2204         }
2205         text_ptr = strchr(text_in, '=');
2206         if (!text_ptr) {
2207                 pr_err("No \"=\" separator found in Text Data,"
2208                         "  cannot continue.\n");
2209                 goto reject;
2210         }
2211         if (!strncmp("=All", text_ptr, 4)) {
2212                 cmd->cmd_flags |= ICF_SENDTARGETS_ALL;
2213         } else if (!strncmp("=iqn.", text_ptr, 5) ||
2214                    !strncmp("=eui.", text_ptr, 5)) {
2215                 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE;
2216         } else {
2217                 pr_err("Unable to locate valid SendTargets=%s value\n", text_ptr);
2218                 goto reject;
2219         }
2220
2221         spin_lock_bh(&conn->cmd_lock);
2222         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2223         spin_unlock_bh(&conn->cmd_lock);
2224
2225 empty_sendtargets:
2226         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2227
2228         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2229                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
2230                                 (unsigned char *)hdr, hdr->cmdsn);
2231                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2232                         return -1;
2233
2234                 return 0;
2235         }
2236
2237         return iscsit_execute_cmd(cmd, 0);
2238
2239 reject:
2240         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2241                                  (unsigned char *)hdr);
2242 }
2243 EXPORT_SYMBOL(iscsit_process_text_cmd);
2244
2245 static int
2246 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2247                        unsigned char *buf)
2248 {
2249         struct iscsi_text *hdr = (struct iscsi_text *)buf;
2250         char *text_in = NULL;
2251         u32 payload_length = ntoh24(hdr->dlength);
2252         int rx_size, rc;
2253
2254         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
2255         if (rc < 0)
2256                 return 0;
2257
2258         rx_size = payload_length;
2259         if (payload_length) {
2260                 u32 checksum = 0, data_crc = 0;
2261                 u32 padding = 0;
2262                 int niov = 0, rx_got;
2263                 struct kvec iov[2];
2264
2265                 rx_size = ALIGN(payload_length, 4);
2266                 text_in = kzalloc(rx_size, GFP_KERNEL);
2267                 if (!text_in)
2268                         goto reject;
2269
2270                 cmd->text_in_ptr = text_in;
2271
2272                 memset(iov, 0, sizeof(iov));
2273                 iov[niov].iov_base      = text_in;
2274                 iov[niov++].iov_len     = rx_size;
2275
2276                 padding = rx_size - payload_length;
2277                 if (padding)
2278                         pr_debug("Receiving %u additional bytes"
2279                                         " for padding.\n", padding);
2280                 if (conn->conn_ops->DataDigest) {
2281                         iov[niov].iov_base      = &checksum;
2282                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
2283                         rx_size += ISCSI_CRC_LEN;
2284                 }
2285
2286                 WARN_ON_ONCE(niov > ARRAY_SIZE(iov));
2287                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
2288                 if (rx_got != rx_size)
2289                         goto reject;
2290
2291                 if (conn->conn_ops->DataDigest) {
2292                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
2293                                                   text_in, rx_size, 0, NULL,
2294                                                   &data_crc);
2295
2296                         if (checksum != data_crc) {
2297                                 pr_err("Text data CRC32C DataDigest"
2298                                         " 0x%08x does not match computed"
2299                                         " 0x%08x\n", checksum, data_crc);
2300                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2301                                         pr_err("Unable to recover from"
2302                                         " Text Data digest failure while in"
2303                                                 " ERL=0.\n");
2304                                         goto reject;
2305                                 } else {
2306                                         /*
2307                                          * Silently drop this PDU and let the
2308                                          * initiator plug the CmdSN gap.
2309                                          */
2310                                         pr_debug("Dropping Text"
2311                                         " Command CmdSN: 0x%08x due to"
2312                                         " DataCRC error.\n", hdr->cmdsn);
2313                                         kfree(text_in);
2314                                         return 0;
2315                                 }
2316                         } else {
2317                                 pr_debug("Got CRC32C DataDigest"
2318                                         " 0x%08x for %u bytes of text data.\n",
2319                                                 checksum, payload_length);
2320                         }
2321                 }
2322                 text_in[payload_length - 1] = '\0';
2323                 pr_debug("Successfully read %d bytes of text"
2324                                 " data.\n", payload_length);
2325         }
2326
2327         return iscsit_process_text_cmd(conn, cmd, hdr);
2328
2329 reject:
2330         kfree(cmd->text_in_ptr);
2331         cmd->text_in_ptr = NULL;
2332         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
2333 }
2334
2335 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2336 {
2337         struct iscsi_conn *conn_p;
2338         struct iscsi_session *sess = conn->sess;
2339
2340         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2341                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2342
2343         atomic_set(&sess->session_logout, 1);
2344         atomic_set(&conn->conn_logout_remove, 1);
2345         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2346
2347         iscsit_inc_conn_usage_count(conn);
2348         iscsit_inc_session_usage_count(sess);
2349
2350         spin_lock_bh(&sess->conn_lock);
2351         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2352                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2353                         continue;
2354
2355                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2356                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2357         }
2358         spin_unlock_bh(&sess->conn_lock);
2359
2360         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2361
2362         return 0;
2363 }
2364
2365 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2366 {
2367         struct iscsi_conn *l_conn;
2368         struct iscsi_session *sess = conn->sess;
2369
2370         pr_debug("Received logout request CLOSECONNECTION for CID:"
2371                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2372
2373         /*
2374          * A Logout Request with a CLOSECONNECTION reason code for a CID
2375          * can arrive on a connection with a differing CID.
2376          */
2377         if (conn->cid == cmd->logout_cid) {
2378                 spin_lock_bh(&conn->state_lock);
2379                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2380                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2381
2382                 atomic_set(&conn->conn_logout_remove, 1);
2383                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2384                 iscsit_inc_conn_usage_count(conn);
2385
2386                 spin_unlock_bh(&conn->state_lock);
2387         } else {
2388                 /*
2389                  * Handle all different cid CLOSECONNECTION requests in
2390                  * iscsit_logout_post_handler_diffcid() as to give enough
2391                  * time for any non immediate command's CmdSN to be
2392                  * acknowledged on the connection in question.
2393                  *
2394                  * Here we simply make sure the CID is still around.
2395                  */
2396                 l_conn = iscsit_get_conn_from_cid(sess,
2397                                 cmd->logout_cid);
2398                 if (!l_conn) {
2399                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2400                         iscsit_add_cmd_to_response_queue(cmd, conn,
2401                                         cmd->i_state);
2402                         return 0;
2403                 }
2404
2405                 iscsit_dec_conn_usage_count(l_conn);
2406         }
2407
2408         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2409
2410         return 0;
2411 }
2412
2413 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2414 {
2415         struct iscsi_session *sess = conn->sess;
2416
2417         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2418                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2419
2420         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2421                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2422                         " while ERL!=2.\n");
2423                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2424                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2425                 return 0;
2426         }
2427
2428         if (conn->cid == cmd->logout_cid) {
2429                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2430                         " with CID: %hu on CID: %hu, implementation error.\n",
2431                                 cmd->logout_cid, conn->cid);
2432                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2433                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2434                 return 0;
2435         }
2436
2437         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2438
2439         return 0;
2440 }
2441
2442 int
2443 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2444                         unsigned char *buf)
2445 {
2446         int cmdsn_ret, logout_remove = 0;
2447         u8 reason_code = 0;
2448         struct iscsi_logout *hdr;
2449         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2450
2451         hdr                     = (struct iscsi_logout *) buf;
2452         reason_code             = (hdr->flags & 0x7f);
2453
2454         if (tiqn) {
2455                 spin_lock(&tiqn->logout_stats.lock);
2456                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2457                         tiqn->logout_stats.normal_logouts++;
2458                 else
2459                         tiqn->logout_stats.abnormal_logouts++;
2460                 spin_unlock(&tiqn->logout_stats.lock);
2461         }
2462
2463         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2464                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2465                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2466                 hdr->cid, conn->cid);
2467
2468         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2469                 pr_err("Received logout request on connection that"
2470                         " is not in logged in state, ignoring request.\n");
2471                 iscsit_free_cmd(cmd, false);
2472                 return 0;
2473         }
2474
2475         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2476         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2477         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2478         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2479         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2480         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2481         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2482         cmd->logout_cid         = be16_to_cpu(hdr->cid);
2483         cmd->logout_reason      = reason_code;
2484         cmd->data_direction     = DMA_NONE;
2485
2486         /*
2487          * We need to sleep in these cases (by returning 1) until the Logout
2488          * Response gets sent in the tx thread.
2489          */
2490         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2491            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2492             be16_to_cpu(hdr->cid) == conn->cid))
2493                 logout_remove = 1;
2494
2495         spin_lock_bh(&conn->cmd_lock);
2496         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2497         spin_unlock_bh(&conn->cmd_lock);
2498
2499         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2500                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2501
2502         /*
2503          * Immediate commands are executed, well, immediately.
2504          * Non-Immediate Logout Commands are executed in CmdSN order.
2505          */
2506         if (cmd->immediate_cmd) {
2507                 int ret = iscsit_execute_cmd(cmd, 0);
2508
2509                 if (ret < 0)
2510                         return ret;
2511         } else {
2512                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2513                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
2514                         logout_remove = 0;
2515                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2516                         return -1;
2517         }
2518
2519         return logout_remove;
2520 }
2521 EXPORT_SYMBOL(iscsit_handle_logout_cmd);
2522
2523 int iscsit_handle_snack(
2524         struct iscsi_conn *conn,
2525         unsigned char *buf)
2526 {
2527         struct iscsi_snack *hdr;
2528
2529         hdr                     = (struct iscsi_snack *) buf;
2530         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2531
2532         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2533                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2534                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2535                         hdr->begrun, hdr->runlength, conn->cid);
2536
2537         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2538                 pr_err("Initiator sent SNACK request while in"
2539                         " ErrorRecoveryLevel=0.\n");
2540                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2541                                          buf);
2542         }
2543         /*
2544          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2545          * call from inside iscsi_send_recovery_datain_or_r2t().
2546          */
2547         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2548         case 0:
2549                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2550                         hdr->itt,
2551                         be32_to_cpu(hdr->ttt),
2552                         be32_to_cpu(hdr->begrun),
2553                         be32_to_cpu(hdr->runlength));
2554         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2555                 return iscsit_handle_status_snack(conn, hdr->itt,
2556                         be32_to_cpu(hdr->ttt),
2557                         be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
2558         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2559                 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
2560                         be32_to_cpu(hdr->begrun),
2561                         be32_to_cpu(hdr->runlength));
2562         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2563                 /* FIXME: Support R-Data SNACK */
2564                 pr_err("R-Data SNACK Not Supported.\n");
2565                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2566                                          buf);
2567         default:
2568                 pr_err("Unknown SNACK type 0x%02x, protocol"
2569                         " error.\n", hdr->flags & 0x0f);
2570                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2571                                          buf);
2572         }
2573
2574         return 0;
2575 }
2576 EXPORT_SYMBOL(iscsit_handle_snack);
2577
2578 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2579 {
2580         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2581             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2582                 wait_for_completion_interruptible_timeout(
2583                                         &conn->rx_half_close_comp,
2584                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2585         }
2586 }
2587
2588 static int iscsit_handle_immediate_data(
2589         struct iscsi_cmd *cmd,
2590         struct iscsi_scsi_req *hdr,
2591         u32 length)
2592 {
2593         int iov_ret, rx_got = 0, rx_size = 0;
2594         u32 checksum, iov_count = 0, padding = 0;
2595         struct iscsi_conn *conn = cmd->conn;
2596         struct kvec *iov;
2597         void *overflow_buf = NULL;
2598
2599         BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length);
2600         rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length);
2601         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data,
2602                                    cmd->orig_iov_data_count - 2,
2603                                    cmd->write_data_done, rx_size);
2604         if (iov_ret < 0)
2605                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2606
2607         iov_count = iov_ret;
2608         iov = &cmd->iov_data[0];
2609         if (rx_size < length) {
2610                 /*
2611                  * Special case: length of immediate data exceeds the data
2612                  * buffer size derived from the CDB.
2613                  */
2614                 overflow_buf = kmalloc(length - rx_size, GFP_KERNEL);
2615                 if (!overflow_buf) {
2616                         iscsit_unmap_iovec(cmd);
2617                         return IMMEDIATE_DATA_CANNOT_RECOVER;
2618                 }
2619                 cmd->overflow_buf = overflow_buf;
2620                 iov[iov_count].iov_base = overflow_buf;
2621                 iov[iov_count].iov_len = length - rx_size;
2622                 iov_count++;
2623                 rx_size = length;
2624         }
2625
2626         padding = ((-length) & 3);
2627         if (padding != 0) {
2628                 iov[iov_count].iov_base = cmd->pad_bytes;
2629                 iov[iov_count++].iov_len = padding;
2630                 rx_size += padding;
2631         }
2632
2633         if (conn->conn_ops->DataDigest) {
2634                 iov[iov_count].iov_base         = &checksum;
2635                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2636                 rx_size += ISCSI_CRC_LEN;
2637         }
2638
2639         WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
2640         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2641
2642         iscsit_unmap_iovec(cmd);
2643
2644         if (rx_got != rx_size) {
2645                 iscsit_rx_thread_wait_for_tcp(conn);
2646                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2647         }
2648
2649         if (conn->conn_ops->DataDigest) {
2650                 u32 data_crc;
2651
2652                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
2653                                                     cmd->write_data_done, length, padding,
2654                                                     cmd->pad_bytes);
2655
2656                 if (checksum != data_crc) {
2657                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2658                                 " does not match computed 0x%08x\n", checksum,
2659                                 data_crc);
2660
2661                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2662                                 pr_err("Unable to recover from"
2663                                         " Immediate Data digest failure while"
2664                                         " in ERL=0.\n");
2665                                 iscsit_reject_cmd(cmd,
2666                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2667                                                 (unsigned char *)hdr);
2668                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2669                         } else {
2670                                 iscsit_reject_cmd(cmd,
2671                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2672                                                 (unsigned char *)hdr);
2673                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2674                         }
2675                 } else {
2676                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2677                                 " %u bytes of Immediate Data\n", checksum,
2678                                 length);
2679                 }
2680         }
2681
2682         cmd->write_data_done += length;
2683
2684         if (cmd->write_data_done == cmd->se_cmd.data_length) {
2685                 spin_lock_bh(&cmd->istate_lock);
2686                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2687                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2688                 spin_unlock_bh(&cmd->istate_lock);
2689         }
2690
2691         return IMMEDIATE_DATA_NORMAL_OPERATION;
2692 }
2693
2694 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2695         with active network interface */
2696 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2697 {
2698         struct iscsi_cmd *cmd;
2699         struct iscsi_conn *conn_p;
2700         bool found = false;
2701
2702         lockdep_assert_held(&conn->sess->conn_lock);
2703
2704         /*
2705          * Only send a Asynchronous Message on connections whos network
2706          * interface is still functional.
2707          */
2708         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2709                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2710                         iscsit_inc_conn_usage_count(conn_p);
2711                         found = true;
2712                         break;
2713                 }
2714         }
2715
2716         if (!found)
2717                 return;
2718
2719         cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING);
2720         if (!cmd) {
2721                 iscsit_dec_conn_usage_count(conn_p);
2722                 return;
2723         }
2724
2725         cmd->logout_cid = conn->cid;
2726         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2727         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2728
2729         spin_lock_bh(&conn_p->cmd_lock);
2730         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2731         spin_unlock_bh(&conn_p->cmd_lock);
2732
2733         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2734         iscsit_dec_conn_usage_count(conn_p);
2735 }
2736
2737 static int iscsit_send_conn_drop_async_message(
2738         struct iscsi_cmd *cmd,
2739         struct iscsi_conn *conn)
2740 {
2741         struct iscsi_async *hdr;
2742
2743         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2744
2745         hdr                     = (struct iscsi_async *) cmd->pdu;
2746         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2747         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2748         cmd->init_task_tag      = RESERVED_ITT;
2749         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2750         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2751         cmd->stat_sn            = conn->stat_sn++;
2752         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2753         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2754         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2755         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2756         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2757         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2758         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2759
2760         pr_debug("Sending Connection Dropped Async Message StatSN:"
2761                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2762                         cmd->logout_cid, conn->cid);
2763
2764         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
2765 }
2766
2767 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2768 {
2769         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2770             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2771                 wait_for_completion_interruptible_timeout(
2772                                         &conn->tx_half_close_comp,
2773                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2774         }
2775 }
2776
2777 void
2778 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2779                         struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
2780                         bool set_statsn)
2781 {
2782         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2783         hdr->flags              = datain->flags;
2784         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2785                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2786                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2787                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2788                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2789                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2790                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2791                 }
2792         }
2793         hton24(hdr->dlength, datain->length);
2794         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2795                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2796                                 (struct scsi_lun *)&hdr->lun);
2797         else
2798                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2799
2800         hdr->itt                = cmd->init_task_tag;
2801
2802         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2803                 hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2804         else
2805                 hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
2806         if (set_statsn)
2807                 hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2808         else
2809                 hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
2810
2811         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2812         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2813         hdr->datasn             = cpu_to_be32(datain->data_sn);
2814         hdr->offset             = cpu_to_be32(datain->offset);
2815
2816         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2817                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2818                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2819                 ntohl(hdr->offset), datain->length, conn->cid);
2820 }
2821 EXPORT_SYMBOL(iscsit_build_datain_pdu);
2822
2823 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2824 {
2825         struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
2826         struct iscsi_datain datain;
2827         struct iscsi_datain_req *dr;
2828         int eodr = 0, ret;
2829         bool set_statsn = false;
2830
2831         memset(&datain, 0, sizeof(struct iscsi_datain));
2832         dr = iscsit_get_datain_values(cmd, &datain);
2833         if (!dr) {
2834                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2835                                 cmd->init_task_tag);
2836                 return -1;
2837         }
2838         /*
2839          * Be paranoid and double check the logic for now.
2840          */
2841         if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
2842                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2843                         " datain.length: %u exceeds cmd->data_length: %u\n",
2844                         cmd->init_task_tag, datain.offset, datain.length,
2845                         cmd->se_cmd.data_length);
2846                 return -1;
2847         }
2848
2849         atomic_long_add(datain.length, &conn->sess->tx_data_octets);
2850         /*
2851          * Special case for successfully execution w/ both DATAIN
2852          * and Sense Data.
2853          */
2854         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2855             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2856                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2857         else {
2858                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2859                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2860                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2861                         cmd->stat_sn = conn->stat_sn++;
2862                         set_statsn = true;
2863                 } else if (dr->dr_complete ==
2864                            DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2865                         set_statsn = true;
2866         }
2867
2868         iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
2869
2870         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0);
2871         if (ret < 0)
2872                 return ret;
2873
2874         if (dr->dr_complete) {
2875                 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2876                                 2 : 1;
2877                 iscsit_free_datain_req(cmd, dr);
2878         }
2879
2880         return eodr;
2881 }
2882
2883 int
2884 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2885                         struct iscsi_logout_rsp *hdr)
2886 {
2887         struct iscsi_conn *logout_conn = NULL;
2888         struct iscsi_conn_recovery *cr = NULL;
2889         struct iscsi_session *sess = conn->sess;
2890         /*
2891          * The actual shutting down of Sessions and/or Connections
2892          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2893          * is done in scsi_logout_post_handler().
2894          */
2895         switch (cmd->logout_reason) {
2896         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2897                 pr_debug("iSCSI session logout successful, setting"
2898                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2899                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2900                 break;
2901         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2902                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2903                         break;
2904                 /*
2905                  * For CLOSECONNECTION logout requests carrying
2906                  * a matching logout CID -> local CID, the reference
2907                  * for the local CID will have been incremented in
2908                  * iscsi_logout_closeconnection().
2909                  *
2910                  * For CLOSECONNECTION logout requests carrying
2911                  * a different CID than the connection it arrived
2912                  * on, the connection responding to cmd->logout_cid
2913                  * is stopped in iscsit_logout_post_handler_diffcid().
2914                  */
2915
2916                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2917                         " successful.\n", cmd->logout_cid, conn->cid);
2918                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2919                 break;
2920         case ISCSI_LOGOUT_REASON_RECOVERY:
2921                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2922                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2923                         break;
2924                 /*
2925                  * If the connection is still active from our point of view
2926                  * force connection recovery to occur.
2927                  */
2928                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2929                                 cmd->logout_cid);
2930                 if (logout_conn) {
2931                         iscsit_connection_reinstatement_rcfr(logout_conn);
2932                         iscsit_dec_conn_usage_count(logout_conn);
2933                 }
2934
2935                 cr = iscsit_get_inactive_connection_recovery_entry(
2936                                 conn->sess, cmd->logout_cid);
2937                 if (!cr) {
2938                         pr_err("Unable to locate CID: %hu for"
2939                         " REMOVECONNFORRECOVERY Logout Request.\n",
2940                                 cmd->logout_cid);
2941                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2942                         break;
2943                 }
2944
2945                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2946
2947                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2948                         " for recovery for CID: %hu on CID: %hu successful.\n",
2949                                 cmd->logout_cid, conn->cid);
2950                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2951                 break;
2952         default:
2953                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2954                                 cmd->logout_reason);
2955                 return -1;
2956         }
2957
2958         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2959         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2960         hdr->response           = cmd->logout_response;
2961         hdr->itt                = cmd->init_task_tag;
2962         cmd->stat_sn            = conn->stat_sn++;
2963         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2964
2965         iscsit_increment_maxcmdsn(cmd, conn->sess);
2966         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2967         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2968
2969         pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
2970                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2971                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2972                 cmd->logout_cid, conn->cid);
2973
2974         return 0;
2975 }
2976 EXPORT_SYMBOL(iscsit_build_logout_rsp);
2977
2978 static int
2979 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2980 {
2981         int rc;
2982
2983         rc = iscsit_build_logout_rsp(cmd, conn,
2984                         (struct iscsi_logout_rsp *)&cmd->pdu[0]);
2985         if (rc < 0)
2986                 return rc;
2987
2988         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
2989 }
2990
2991 void
2992 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2993                        struct iscsi_nopin *hdr, bool nopout_response)
2994 {
2995         hdr->opcode             = ISCSI_OP_NOOP_IN;
2996         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2997         hton24(hdr->dlength, cmd->buf_ptr_size);
2998         if (nopout_response)
2999                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
3000         hdr->itt                = cmd->init_task_tag;
3001         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3002         cmd->stat_sn            = (nopout_response) ? conn->stat_sn++ :
3003                                   conn->stat_sn;
3004         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3005
3006         if (nopout_response)
3007                 iscsit_increment_maxcmdsn(cmd, conn->sess);
3008
3009         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3010         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3011
3012         pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
3013                 " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
3014                 "Solicited" : "Unsolicited", cmd->init_task_tag,
3015                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
3016 }
3017 EXPORT_SYMBOL(iscsit_build_nopin_rsp);
3018
3019 /*
3020  *      Unsolicited NOPIN, either requesting a response or not.
3021  */
3022 static int iscsit_send_unsolicited_nopin(
3023         struct iscsi_cmd *cmd,
3024         struct iscsi_conn *conn,
3025         int want_response)
3026 {
3027         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3028         int ret;
3029
3030         iscsit_build_nopin_rsp(cmd, conn, hdr, false);
3031
3032         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
3033                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
3034
3035         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3036         if (ret < 0)
3037                 return ret;
3038
3039         spin_lock_bh(&cmd->istate_lock);
3040         cmd->i_state = want_response ?
3041                 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
3042         spin_unlock_bh(&cmd->istate_lock);
3043
3044         return 0;
3045 }
3046
3047 static int
3048 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3049 {
3050         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3051
3052         iscsit_build_nopin_rsp(cmd, conn, hdr, true);
3053
3054         /*
3055          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
3056          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
3057          */
3058         pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size);
3059
3060         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3061                                                      cmd->buf_ptr,
3062                                                      cmd->buf_ptr_size);
3063 }
3064
3065 static int iscsit_send_r2t(
3066         struct iscsi_cmd *cmd,
3067         struct iscsi_conn *conn)
3068 {
3069         struct iscsi_r2t *r2t;
3070         struct iscsi_r2t_rsp *hdr;
3071         int ret;
3072
3073         r2t = iscsit_get_r2t_from_list(cmd);
3074         if (!r2t)
3075                 return -1;
3076
3077         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
3078         memset(hdr, 0, ISCSI_HDR_LEN);
3079         hdr->opcode             = ISCSI_OP_R2T;
3080         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3081         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
3082                         (struct scsi_lun *)&hdr->lun);
3083         hdr->itt                = cmd->init_task_tag;
3084         if (conn->conn_transport->iscsit_get_r2t_ttt)
3085                 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t);
3086         else
3087                 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess);
3088         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
3089         hdr->statsn             = cpu_to_be32(conn->stat_sn);
3090         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3091         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3092         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
3093         hdr->data_offset        = cpu_to_be32(r2t->offset);
3094         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
3095
3096         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
3097                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
3098                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
3099                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
3100                         r2t->offset, r2t->xfer_len, conn->cid);
3101
3102         spin_lock_bh(&cmd->r2t_lock);
3103         r2t->sent_r2t = 1;
3104         spin_unlock_bh(&cmd->r2t_lock);
3105
3106         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3107         if (ret < 0) {
3108                 return ret;
3109         }
3110
3111         spin_lock_bh(&cmd->dataout_timeout_lock);
3112         iscsit_start_dataout_timer(cmd, conn);
3113         spin_unlock_bh(&cmd->dataout_timeout_lock);
3114
3115         return 0;
3116 }
3117
3118 /*
3119  *      @recovery: If called from iscsi_task_reassign_complete_write() for
3120  *              connection recovery.
3121  */
3122 int iscsit_build_r2ts_for_cmd(
3123         struct iscsi_conn *conn,
3124         struct iscsi_cmd *cmd,
3125         bool recovery)
3126 {
3127         int first_r2t = 1;
3128         u32 offset = 0, xfer_len = 0;
3129
3130         spin_lock_bh(&cmd->r2t_lock);
3131         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
3132                 spin_unlock_bh(&cmd->r2t_lock);
3133                 return 0;
3134         }
3135
3136         if (conn->sess->sess_ops->DataSequenceInOrder &&
3137             !recovery)
3138                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
3139
3140         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
3141                 if (conn->sess->sess_ops->DataSequenceInOrder) {
3142                         offset = cmd->r2t_offset;
3143
3144                         if (first_r2t && recovery) {
3145                                 int new_data_end = offset +
3146                                         conn->sess->sess_ops->MaxBurstLength -
3147                                         cmd->next_burst_len;
3148
3149                                 if (new_data_end > cmd->se_cmd.data_length)
3150                                         xfer_len = cmd->se_cmd.data_length - offset;
3151                                 else
3152                                         xfer_len =
3153                                                 conn->sess->sess_ops->MaxBurstLength -
3154                                                 cmd->next_burst_len;
3155                         } else {
3156                                 int new_data_end = offset +
3157                                         conn->sess->sess_ops->MaxBurstLength;
3158
3159                                 if (new_data_end > cmd->se_cmd.data_length)
3160                                         xfer_len = cmd->se_cmd.data_length - offset;
3161                                 else
3162                                         xfer_len = conn->sess->sess_ops->MaxBurstLength;
3163                         }
3164
3165                         if ((s32)xfer_len < 0) {
3166                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3167                                 break;
3168                         }
3169
3170                         cmd->r2t_offset += xfer_len;
3171
3172                         if (cmd->r2t_offset == cmd->se_cmd.data_length)
3173                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3174                 } else {
3175                         struct iscsi_seq *seq;
3176
3177                         seq = iscsit_get_seq_holder_for_r2t(cmd);
3178                         if (!seq) {
3179                                 spin_unlock_bh(&cmd->r2t_lock);
3180                                 return -1;
3181                         }
3182
3183                         offset = seq->offset;
3184                         xfer_len = seq->xfer_len;
3185
3186                         if (cmd->seq_send_order == cmd->seq_count)
3187                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3188                 }
3189                 cmd->outstanding_r2ts++;
3190                 first_r2t = 0;
3191
3192                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3193                         spin_unlock_bh(&cmd->r2t_lock);
3194                         return -1;
3195                 }
3196
3197                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3198                         break;
3199         }
3200         spin_unlock_bh(&cmd->r2t_lock);
3201
3202         return 0;
3203 }
3204 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd);
3205
3206 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3207                         bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
3208 {
3209         if (inc_stat_sn)
3210                 cmd->stat_sn = conn->stat_sn++;
3211
3212         atomic_long_inc(&conn->sess->rsp_pdus);
3213
3214         memset(hdr, 0, ISCSI_HDR_LEN);
3215         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3216         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3217         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3218                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3219                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3220         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3221                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3222                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3223         }
3224         hdr->response           = cmd->iscsi_response;
3225         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3226         hdr->itt                = cmd->init_task_tag;
3227         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3228
3229         iscsit_increment_maxcmdsn(cmd, conn->sess);
3230         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3231         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3232
3233         pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3234                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3235                 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
3236                 cmd->se_cmd.scsi_status, conn->cid);
3237 }
3238 EXPORT_SYMBOL(iscsit_build_rsp_pdu);
3239
3240 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3241 {
3242         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
3243         bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
3244         void *data_buf = NULL;
3245         u32 padding = 0, data_buf_len = 0;
3246
3247         iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
3248
3249         /*
3250          * Attach SENSE DATA payload to iSCSI Response PDU
3251          */
3252         if (cmd->se_cmd.sense_buffer &&
3253            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3254             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3255                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
3256                 cmd->se_cmd.scsi_sense_length += sizeof (__be16);
3257
3258                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3259                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
3260                 data_buf = cmd->sense_buffer;
3261                 data_buf_len = cmd->se_cmd.scsi_sense_length + padding;
3262
3263                 if (padding) {
3264                         memset(cmd->sense_buffer +
3265                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3266                         pr_debug("Adding %u bytes of padding to"
3267                                 " SENSE.\n", padding);
3268                 }
3269
3270                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3271                                 " Response PDU\n",
3272                                 cmd->se_cmd.scsi_sense_length);
3273         }
3274
3275         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf,
3276                                                      data_buf_len);
3277 }
3278
3279 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3280 {
3281         switch (se_tmr->response) {
3282         case TMR_FUNCTION_COMPLETE:
3283                 return ISCSI_TMF_RSP_COMPLETE;
3284         case TMR_TASK_DOES_NOT_EXIST:
3285                 return ISCSI_TMF_RSP_NO_TASK;
3286         case TMR_LUN_DOES_NOT_EXIST:
3287                 return ISCSI_TMF_RSP_NO_LUN;
3288         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3289                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3290         case TMR_FUNCTION_REJECTED:
3291         default:
3292                 return ISCSI_TMF_RSP_REJECTED;
3293         }
3294 }
3295
3296 void
3297 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3298                           struct iscsi_tm_rsp *hdr)
3299 {
3300         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3301
3302         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3303         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3304         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3305         hdr->itt                = cmd->init_task_tag;
3306         cmd->stat_sn            = conn->stat_sn++;
3307         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3308
3309         iscsit_increment_maxcmdsn(cmd, conn->sess);
3310         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3311         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3312
3313         pr_debug("Built Task Management Response ITT: 0x%08x,"
3314                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3315                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3316 }
3317 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
3318
3319 static int
3320 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3321 {
3322         struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
3323
3324         iscsit_build_task_mgt_rsp(cmd, conn, hdr);
3325
3326         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3327 }
3328
3329 #define SENDTARGETS_BUF_LIMIT 32768U
3330
3331 static int
3332 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
3333                                   enum iscsit_transport_type network_transport,
3334                                   int skip_bytes, bool *completed)
3335 {
3336         char *payload = NULL;
3337         struct iscsi_conn *conn = cmd->conn;
3338         struct iscsi_portal_group *tpg;
3339         struct iscsi_tiqn *tiqn;
3340         struct iscsi_tpg_np *tpg_np;
3341         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3342         int target_name_printed;
3343         unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
3344         unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL;
3345         bool active;
3346
3347         buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength,
3348                          SENDTARGETS_BUF_LIMIT);
3349
3350         payload = kzalloc(buffer_len, GFP_KERNEL);
3351         if (!payload)
3352                 return -ENOMEM;
3353
3354         /*
3355          * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE
3356          * explicit case..
3357          */
3358         if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) {
3359                 text_ptr = strchr(text_in, '=');
3360                 if (!text_ptr) {
3361                         pr_err("Unable to locate '=' string in text_in:"
3362                                " %s\n", text_in);
3363                         kfree(payload);
3364                         return -EINVAL;
3365                 }
3366                 /*
3367                  * Skip over '=' character..
3368                  */
3369                 text_ptr += 1;
3370         }
3371
3372         spin_lock(&tiqn_lock);
3373         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3374                 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) &&
3375                      strcmp(tiqn->tiqn, text_ptr)) {
3376                         continue;
3377                 }
3378
3379                 target_name_printed = 0;
3380
3381                 spin_lock(&tiqn->tiqn_tpg_lock);
3382                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3383
3384                         /* If demo_mode_discovery=0 and generate_node_acls=0
3385                          * (demo mode dislabed) do not return
3386                          * TargetName+TargetAddress unless a NodeACL exists.
3387                          */
3388
3389                         if ((tpg->tpg_attrib.generate_node_acls == 0) &&
3390                             (tpg->tpg_attrib.demo_mode_discovery == 0) &&
3391                             (!target_tpg_has_node_acl(&tpg->tpg_se_tpg,
3392                                 cmd->conn->sess->sess_ops->InitiatorName))) {
3393                                 continue;
3394                         }
3395
3396                         spin_lock(&tpg->tpg_state_lock);
3397                         active = (tpg->tpg_state == TPG_STATE_ACTIVE);
3398                         spin_unlock(&tpg->tpg_state_lock);
3399
3400                         if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets)
3401                                 continue;
3402
3403                         spin_lock(&tpg->tpg_np_lock);
3404                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3405                                                 tpg_np_list) {
3406                                 struct iscsi_np *np = tpg_np->tpg_np;
3407                                 struct sockaddr_storage *sockaddr;
3408
3409                                 if (np->np_network_transport != network_transport)
3410                                         continue;
3411
3412                                 if (!target_name_printed) {
3413                                         len = sprintf(buf, "TargetName=%s",
3414                                                       tiqn->tiqn);
3415                                         len += 1;
3416
3417                                         if ((len + payload_len) > buffer_len) {
3418                                                 spin_unlock(&tpg->tpg_np_lock);
3419                                                 spin_unlock(&tiqn->tiqn_tpg_lock);
3420                                                 end_of_buf = 1;
3421                                                 goto eob;
3422                                         }
3423
3424                                         if (skip_bytes && len <= skip_bytes) {
3425                                                 skip_bytes -= len;
3426                                         } else {
3427                                                 memcpy(payload + payload_len, buf, len);
3428                                                 payload_len += len;
3429                                                 target_name_printed = 1;
3430                                                 if (len > skip_bytes)
3431                                                         skip_bytes = 0;
3432                                         }
3433                                 }
3434
3435                                 if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr))
3436                                         sockaddr = &conn->local_sockaddr;
3437                                 else
3438                                         sockaddr = &np->np_sockaddr;
3439
3440                                 len = sprintf(buf, "TargetAddress="
3441                                               "%pISpc,%hu",
3442                                               sockaddr,
3443                                               tpg->tpgt);
3444                                 len += 1;
3445
3446                                 if ((len + payload_len) > buffer_len) {
3447                                         spin_unlock(&tpg->tpg_np_lock);
3448                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3449                                         end_of_buf = 1;
3450                                         goto eob;
3451                                 }
3452
3453                                 if (skip_bytes && len <= skip_bytes) {
3454                                         skip_bytes -= len;
3455                                 } else {
3456                                         memcpy(payload + payload_len, buf, len);
3457                                         payload_len += len;
3458                                         if (len > skip_bytes)
3459                                                 skip_bytes = 0;
3460                                 }
3461                         }
3462                         spin_unlock(&tpg->tpg_np_lock);
3463                 }
3464                 spin_unlock(&tiqn->tiqn_tpg_lock);
3465 eob:
3466                 if (end_of_buf) {
3467                         *completed = false;
3468                         break;
3469                 }
3470
3471                 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE)
3472                         break;
3473         }
3474         spin_unlock(&tiqn_lock);
3475
3476         cmd->buf_ptr = payload;
3477
3478         return payload_len;
3479 }
3480
3481 int
3482 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3483                       struct iscsi_text_rsp *hdr,
3484                       enum iscsit_transport_type network_transport)
3485 {
3486         int text_length, padding;
3487         bool completed = true;
3488
3489         text_length = iscsit_build_sendtargets_response(cmd, network_transport,
3490                                                         cmd->read_data_done,
3491                                                         &completed);
3492         if (text_length < 0)
3493                 return text_length;
3494
3495         if (completed) {
3496                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
3497         } else {
3498                 hdr->flags = ISCSI_FLAG_TEXT_CONTINUE;
3499                 cmd->read_data_done += text_length;
3500                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
3501                         cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
3502         }
3503         hdr->opcode = ISCSI_OP_TEXT_RSP;
3504         padding = ((-text_length) & 3);
3505         hton24(hdr->dlength, text_length);
3506         hdr->itt = cmd->init_task_tag;
3507         hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
3508         cmd->stat_sn = conn->stat_sn++;
3509         hdr->statsn = cpu_to_be32(cmd->stat_sn);
3510
3511         iscsit_increment_maxcmdsn(cmd, conn->sess);
3512         /*
3513          * Reset maxcmdsn_inc in multi-part text payload exchanges to
3514          * correctly increment MaxCmdSN for each response answering a
3515          * non immediate text request with a valid CmdSN.
3516          */
3517         cmd->maxcmdsn_inc = 0;
3518         hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3519         hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3520
3521         pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x,"
3522                 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag,
3523                 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid,
3524                 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL),
3525                 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE));
3526
3527         return text_length + padding;
3528 }
3529 EXPORT_SYMBOL(iscsit_build_text_rsp);
3530
3531 static int iscsit_send_text_rsp(
3532         struct iscsi_cmd *cmd,
3533         struct iscsi_conn *conn)
3534 {
3535         struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu;
3536         int text_length;
3537
3538         text_length = iscsit_build_text_rsp(cmd, conn, hdr,
3539                                 conn->conn_transport->transport_type);
3540         if (text_length < 0)
3541                 return text_length;
3542
3543         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3544                                                      cmd->buf_ptr,
3545                                                      text_length);
3546 }
3547
3548 void
3549 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3550                     struct iscsi_reject *hdr)
3551 {
3552         hdr->opcode             = ISCSI_OP_REJECT;
3553         hdr->reason             = cmd->reject_reason;
3554         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3555         hton24(hdr->dlength, ISCSI_HDR_LEN);
3556         hdr->ffffffff           = cpu_to_be32(0xffffffff);
3557         cmd->stat_sn            = conn->stat_sn++;
3558         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3559         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3560         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3561
3562 }
3563 EXPORT_SYMBOL(iscsit_build_reject);
3564
3565 static int iscsit_send_reject(
3566         struct iscsi_cmd *cmd,
3567         struct iscsi_conn *conn)
3568 {
3569         struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0];
3570
3571         iscsit_build_reject(cmd, conn, hdr);
3572
3573         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3574                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3575
3576         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3577                                                      cmd->buf_ptr,
3578                                                      ISCSI_HDR_LEN);
3579 }
3580
3581 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3582 {
3583         int ord, cpu;
3584         /*
3585          * bitmap_id is assigned from iscsit_global->ts_bitmap from
3586          * within iscsit_start_kthreads()
3587          *
3588          * Here we use bitmap_id to determine which CPU that this
3589          * iSCSI connection's RX/TX threads will be scheduled to
3590          * execute upon.
3591          */
3592         ord = conn->bitmap_id % cpumask_weight(cpu_online_mask);
3593         for_each_online_cpu(cpu) {
3594                 if (ord-- == 0) {
3595                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3596                         return;
3597                 }
3598         }
3599         /*
3600          * This should never be reached..
3601          */
3602         dump_stack();
3603         cpumask_setall(conn->conn_cpumask);
3604 }
3605
3606 int
3607 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3608 {
3609         int ret;
3610
3611         switch (state) {
3612         case ISTATE_SEND_R2T:
3613                 ret = iscsit_send_r2t(cmd, conn);
3614                 if (ret < 0)
3615                         goto err;
3616                 break;
3617         case ISTATE_REMOVE:
3618                 spin_lock_bh(&conn->cmd_lock);
3619                 list_del_init(&cmd->i_conn_node);
3620                 spin_unlock_bh(&conn->cmd_lock);
3621
3622                 iscsit_free_cmd(cmd, false);
3623                 break;
3624         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3625                 iscsit_mod_nopin_response_timer(conn);
3626                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
3627                 if (ret < 0)
3628                         goto err;
3629                 break;
3630         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3631                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
3632                 if (ret < 0)
3633                         goto err;
3634                 break;
3635         default:
3636                 pr_err("Unknown Opcode: 0x%02x ITT:"
3637                        " 0x%08x, i_state: %d on CID: %hu\n",
3638                        cmd->iscsi_opcode, cmd->init_task_tag, state,
3639                        conn->cid);
3640                 goto err;
3641         }
3642
3643         return 0;
3644
3645 err:
3646         return -1;
3647 }
3648 EXPORT_SYMBOL(iscsit_immediate_queue);
3649
3650 static int
3651 iscsit_handle_immediate_queue(struct iscsi_conn *conn)
3652 {
3653         struct iscsit_transport *t = conn->conn_transport;
3654         struct iscsi_queue_req *qr;
3655         struct iscsi_cmd *cmd;
3656         u8 state;
3657         int ret;
3658
3659         while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3660                 atomic_set(&conn->check_immediate_queue, 0);
3661                 cmd = qr->cmd;
3662                 state = qr->state;
3663                 kmem_cache_free(lio_qr_cache, qr);
3664
3665                 ret = t->iscsit_immediate_queue(conn, cmd, state);
3666                 if (ret < 0)
3667                         return ret;
3668         }
3669
3670         return 0;
3671 }
3672
3673 int
3674 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3675 {
3676         int ret;
3677
3678 check_rsp_state:
3679         switch (state) {
3680         case ISTATE_SEND_DATAIN:
3681                 ret = iscsit_send_datain(cmd, conn);
3682                 if (ret < 0)
3683                         goto err;
3684                 else if (!ret)
3685                         /* more drs */
3686                         goto check_rsp_state;
3687                 else if (ret == 1) {
3688                         /* all done */
3689                         spin_lock_bh(&cmd->istate_lock);
3690                         cmd->i_state = ISTATE_SENT_STATUS;
3691                         spin_unlock_bh(&cmd->istate_lock);
3692
3693                         if (atomic_read(&conn->check_immediate_queue))
3694                                 return 1;
3695
3696                         return 0;
3697                 } else if (ret == 2) {
3698                         /* Still must send status,
3699                            SCF_TRANSPORT_TASK_SENSE was set */
3700                         spin_lock_bh(&cmd->istate_lock);
3701                         cmd->i_state = ISTATE_SEND_STATUS;
3702                         spin_unlock_bh(&cmd->istate_lock);
3703                         state = ISTATE_SEND_STATUS;
3704                         goto check_rsp_state;
3705                 }
3706
3707                 break;
3708         case ISTATE_SEND_STATUS:
3709         case ISTATE_SEND_STATUS_RECOVERY:
3710                 ret = iscsit_send_response(cmd, conn);
3711                 break;
3712         case ISTATE_SEND_LOGOUTRSP:
3713                 ret = iscsit_send_logout(cmd, conn);
3714                 break;
3715         case ISTATE_SEND_ASYNCMSG:
3716                 ret = iscsit_send_conn_drop_async_message(
3717                         cmd, conn);
3718                 break;
3719         case ISTATE_SEND_NOPIN:
3720                 ret = iscsit_send_nopin(cmd, conn);
3721                 break;
3722         case ISTATE_SEND_REJECT:
3723                 ret = iscsit_send_reject(cmd, conn);
3724                 break;
3725         case ISTATE_SEND_TASKMGTRSP:
3726                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3727                 if (ret != 0)
3728                         break;
3729                 ret = iscsit_tmr_post_handler(cmd, conn);
3730                 if (ret != 0)
3731                         iscsit_fall_back_to_erl0(conn->sess);
3732                 break;
3733         case ISTATE_SEND_TEXTRSP:
3734                 ret = iscsit_send_text_rsp(cmd, conn);
3735                 break;
3736         default:
3737                 pr_err("Unknown Opcode: 0x%02x ITT:"
3738                        " 0x%08x, i_state: %d on CID: %hu\n",
3739                        cmd->iscsi_opcode, cmd->init_task_tag,
3740                        state, conn->cid);
3741                 goto err;
3742         }
3743         if (ret < 0)
3744                 goto err;
3745
3746         switch (state) {
3747         case ISTATE_SEND_LOGOUTRSP:
3748                 if (!iscsit_logout_post_handler(cmd, conn))
3749                         return -ECONNRESET;
3750                 /* fall through */
3751         case ISTATE_SEND_STATUS:
3752         case ISTATE_SEND_ASYNCMSG:
3753         case ISTATE_SEND_NOPIN:
3754         case ISTATE_SEND_STATUS_RECOVERY:
3755         case ISTATE_SEND_TEXTRSP:
3756         case ISTATE_SEND_TASKMGTRSP:
3757         case ISTATE_SEND_REJECT:
3758                 spin_lock_bh(&cmd->istate_lock);
3759                 cmd->i_state = ISTATE_SENT_STATUS;
3760                 spin_unlock_bh(&cmd->istate_lock);
3761                 break;
3762         default:
3763                 pr_err("Unknown Opcode: 0x%02x ITT:"
3764                        " 0x%08x, i_state: %d on CID: %hu\n",
3765                        cmd->iscsi_opcode, cmd->init_task_tag,
3766                        cmd->i_state, conn->cid);
3767                 goto err;
3768         }
3769
3770         if (atomic_read(&conn->check_immediate_queue))
3771                 return 1;
3772
3773         return 0;
3774
3775 err:
3776         return -1;
3777 }
3778 EXPORT_SYMBOL(iscsit_response_queue);
3779
3780 static int iscsit_handle_response_queue(struct iscsi_conn *conn)
3781 {
3782         struct iscsit_transport *t = conn->conn_transport;
3783         struct iscsi_queue_req *qr;
3784         struct iscsi_cmd *cmd;
3785         u8 state;
3786         int ret;
3787
3788         while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3789                 cmd = qr->cmd;
3790                 state = qr->state;
3791                 kmem_cache_free(lio_qr_cache, qr);
3792
3793                 ret = t->iscsit_response_queue(conn, cmd, state);
3794                 if (ret == 1 || ret < 0)
3795                         return ret;
3796         }
3797
3798         return 0;
3799 }
3800
3801 int iscsi_target_tx_thread(void *arg)
3802 {
3803         int ret = 0;
3804         struct iscsi_conn *conn = arg;
3805         bool conn_freed = false;
3806
3807         /*
3808          * Allow ourselves to be interrupted by SIGINT so that a
3809          * connection recovery / failure event can be triggered externally.
3810          */
3811         allow_signal(SIGINT);
3812
3813         while (!kthread_should_stop()) {
3814                 /*
3815                  * Ensure that both TX and RX per connection kthreads
3816                  * are scheduled to run on the same CPU.
3817                  */
3818                 iscsit_thread_check_cpumask(conn, current, 1);
3819
3820                 wait_event_interruptible(conn->queues_wq,
3821                                          !iscsit_conn_all_queues_empty(conn));
3822
3823                 if (signal_pending(current))
3824                         goto transport_err;
3825
3826 get_immediate:
3827                 ret = iscsit_handle_immediate_queue(conn);
3828                 if (ret < 0)
3829                         goto transport_err;
3830
3831                 ret = iscsit_handle_response_queue(conn);
3832                 if (ret == 1) {
3833                         goto get_immediate;
3834                 } else if (ret == -ECONNRESET) {
3835                         conn_freed = true;
3836                         goto out;
3837                 } else if (ret < 0) {
3838                         goto transport_err;
3839                 }
3840         }
3841
3842 transport_err:
3843         /*
3844          * Avoid the normal connection failure code-path if this connection
3845          * is still within LOGIN mode, and iscsi_np process context is
3846          * responsible for cleaning up the early connection failure.
3847          */
3848         if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN)
3849                 iscsit_take_action_for_connection_exit(conn, &conn_freed);
3850 out:
3851         if (!conn_freed) {
3852                 while (!kthread_should_stop()) {
3853                         msleep(100);
3854                 }
3855         }
3856         return 0;
3857 }
3858
3859 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
3860 {
3861         struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
3862         struct iscsi_cmd *cmd;
3863         int ret = 0;
3864
3865         switch (hdr->opcode & ISCSI_OPCODE_MASK) {
3866         case ISCSI_OP_SCSI_CMD:
3867                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3868                 if (!cmd)
3869                         goto reject;
3870
3871                 ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
3872                 break;
3873         case ISCSI_OP_SCSI_DATA_OUT:
3874                 ret = iscsit_handle_data_out(conn, buf);
3875                 break;
3876         case ISCSI_OP_NOOP_OUT:
3877                 cmd = NULL;
3878                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
3879                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3880                         if (!cmd)
3881                                 goto reject;
3882                 }
3883                 ret = iscsit_handle_nop_out(conn, cmd, buf);
3884                 break;
3885         case ISCSI_OP_SCSI_TMFUNC:
3886                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3887                 if (!cmd)
3888                         goto reject;
3889
3890                 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
3891                 break;
3892         case ISCSI_OP_TEXT:
3893                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
3894                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
3895                         if (!cmd)
3896                                 goto reject;
3897                 } else {
3898                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3899                         if (!cmd)
3900                                 goto reject;
3901                 }
3902
3903                 ret = iscsit_handle_text_cmd(conn, cmd, buf);
3904                 break;
3905         case ISCSI_OP_LOGOUT:
3906                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3907                 if (!cmd)
3908                         goto reject;
3909
3910                 ret = iscsit_handle_logout_cmd(conn, cmd, buf);
3911                 if (ret > 0)
3912                         wait_for_completion_timeout(&conn->conn_logout_comp,
3913                                         SECONDS_FOR_LOGOUT_COMP * HZ);
3914                 break;
3915         case ISCSI_OP_SNACK:
3916                 ret = iscsit_handle_snack(conn, buf);
3917                 break;
3918         default:
3919                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
3920                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3921                         pr_err("Cannot recover from unknown"
3922                         " opcode while ERL=0, closing iSCSI connection.\n");
3923                         return -1;
3924                 }
3925                 pr_err("Unable to recover from unknown opcode while OFMarker=No,"
3926                        " closing iSCSI connection.\n");
3927                 ret = -1;
3928                 break;
3929         }
3930
3931         return ret;
3932 reject:
3933         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
3934 }
3935
3936 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn)
3937 {
3938         bool ret;
3939
3940         spin_lock_bh(&conn->state_lock);
3941         ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN);
3942         spin_unlock_bh(&conn->state_lock);
3943
3944         return ret;
3945 }
3946
3947 static void iscsit_get_rx_pdu(struct iscsi_conn *conn)
3948 {
3949         int ret;
3950         u8 *buffer, opcode;
3951         u32 checksum = 0, digest = 0;
3952         struct kvec iov;
3953
3954         buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL);
3955         if (!buffer)
3956                 return;
3957
3958         while (!kthread_should_stop()) {
3959                 /*
3960                  * Ensure that both TX and RX per connection kthreads
3961                  * are scheduled to run on the same CPU.
3962                  */
3963                 iscsit_thread_check_cpumask(conn, current, 0);
3964
3965                 memset(&iov, 0, sizeof(struct kvec));
3966
3967                 iov.iov_base    = buffer;
3968                 iov.iov_len     = ISCSI_HDR_LEN;
3969
3970                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3971                 if (ret != ISCSI_HDR_LEN) {
3972                         iscsit_rx_thread_wait_for_tcp(conn);
3973                         break;
3974                 }
3975
3976                 if (conn->conn_ops->HeaderDigest) {
3977                         iov.iov_base    = &digest;
3978                         iov.iov_len     = ISCSI_CRC_LEN;
3979
3980                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
3981                         if (ret != ISCSI_CRC_LEN) {
3982                                 iscsit_rx_thread_wait_for_tcp(conn);
3983                                 break;
3984                         }
3985
3986                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,
3987                                                   ISCSI_HDR_LEN, 0, NULL,
3988                                                   &checksum);
3989
3990                         if (digest != checksum) {
3991                                 pr_err("HeaderDigest CRC32C failed,"
3992                                         " received 0x%08x, computed 0x%08x\n",
3993                                         digest, checksum);
3994                                 /*
3995                                  * Set the PDU to 0xff so it will intentionally
3996                                  * hit default in the switch below.
3997                                  */
3998                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
3999                                 atomic_long_inc(&conn->sess->conn_digest_errors);
4000                         } else {
4001                                 pr_debug("Got HeaderDigest CRC32C"
4002                                                 " 0x%08x\n", checksum);
4003                         }
4004                 }
4005
4006                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
4007                         break;
4008
4009                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
4010
4011                 if (conn->sess->sess_ops->SessionType &&
4012                    ((!(opcode & ISCSI_OP_TEXT)) ||
4013                     (!(opcode & ISCSI_OP_LOGOUT)))) {
4014                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
4015                         " while in Discovery Session, rejecting.\n", opcode);
4016                         iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
4017                                           buffer);
4018                         break;
4019                 }
4020
4021                 ret = iscsi_target_rx_opcode(conn, buffer);
4022                 if (ret < 0)
4023                         break;
4024         }
4025
4026         kfree(buffer);
4027 }
4028
4029 int iscsi_target_rx_thread(void *arg)
4030 {
4031         int rc;
4032         struct iscsi_conn *conn = arg;
4033         bool conn_freed = false;
4034
4035         /*
4036          * Allow ourselves to be interrupted by SIGINT so that a
4037          * connection recovery / failure event can be triggered externally.
4038          */
4039         allow_signal(SIGINT);
4040         /*
4041          * Wait for iscsi_post_login_handler() to complete before allowing
4042          * incoming iscsi/tcp socket I/O, and/or failing the connection.
4043          */
4044         rc = wait_for_completion_interruptible(&conn->rx_login_comp);
4045         if (rc < 0 || iscsi_target_check_conn_state(conn))
4046                 goto out;
4047
4048         if (!conn->conn_transport->iscsit_get_rx_pdu)
4049                 return 0;
4050
4051         conn->conn_transport->iscsit_get_rx_pdu(conn);
4052
4053         if (!signal_pending(current))
4054                 atomic_set(&conn->transport_failed, 1);
4055         iscsit_take_action_for_connection_exit(conn, &conn_freed);
4056
4057 out:
4058         if (!conn_freed) {
4059                 while (!kthread_should_stop()) {
4060                         msleep(100);
4061                 }
4062         }
4063
4064         return 0;
4065 }
4066
4067 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
4068 {
4069         LIST_HEAD(tmp_list);
4070         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
4071         struct iscsi_session *sess = conn->sess;
4072         /*
4073          * We expect this function to only ever be called from either RX or TX
4074          * thread context via iscsit_close_connection() once the other context
4075          * has been reset -> returned sleeping pre-handler state.
4076          */
4077         spin_lock_bh(&conn->cmd_lock);
4078         list_splice_init(&conn->conn_cmd_list, &tmp_list);
4079
4080         list_for_each_entry(cmd, &tmp_list, i_conn_node) {
4081                 struct se_cmd *se_cmd = &cmd->se_cmd;
4082
4083                 if (se_cmd->se_tfo != NULL) {
4084                         spin_lock_irq(&se_cmd->t_state_lock);
4085                         se_cmd->transport_state |= CMD_T_FABRIC_STOP;
4086                         spin_unlock_irq(&se_cmd->t_state_lock);
4087                 }
4088         }
4089         spin_unlock_bh(&conn->cmd_lock);
4090
4091         list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
4092                 list_del_init(&cmd->i_conn_node);
4093
4094                 iscsit_increment_maxcmdsn(cmd, sess);
4095                 iscsit_free_cmd(cmd, true);
4096
4097         }
4098 }
4099
4100 static void iscsit_stop_timers_for_cmds(
4101         struct iscsi_conn *conn)
4102 {
4103         struct iscsi_cmd *cmd;
4104
4105         spin_lock_bh(&conn->cmd_lock);
4106         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
4107                 if (cmd->data_direction == DMA_TO_DEVICE)
4108                         iscsit_stop_dataout_timer(cmd);
4109         }
4110         spin_unlock_bh(&conn->cmd_lock);
4111 }
4112
4113 int iscsit_close_connection(
4114         struct iscsi_conn *conn)
4115 {
4116         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
4117         struct iscsi_session    *sess = conn->sess;
4118
4119         pr_debug("Closing iSCSI connection CID %hu on SID:"
4120                 " %u\n", conn->cid, sess->sid);
4121         /*
4122          * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD
4123          * case just in case the RX Thread in iscsi_target_rx_opcode() is
4124          * sleeping and the logout response never got sent because the
4125          * connection failed.
4126          *
4127          * However for iser-target, isert_wait4logout() is using conn_logout_comp
4128          * to signal logout response TX interrupt completion.  Go ahead and skip
4129          * this for iser since isert_rx_opcode() does not wait on logout failure,
4130          * and to avoid iscsi_conn pointer dereference in iser-target code.
4131          */
4132         if (!conn->conn_transport->rdma_shutdown)
4133                 complete(&conn->conn_logout_comp);
4134
4135         if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) {
4136                 if (conn->tx_thread &&
4137                     cmpxchg(&conn->tx_thread_active, true, false)) {
4138                         send_sig(SIGINT, conn->tx_thread, 1);
4139                         kthread_stop(conn->tx_thread);
4140                 }
4141         } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) {
4142                 if (conn->rx_thread &&
4143                     cmpxchg(&conn->rx_thread_active, true, false)) {
4144                         send_sig(SIGINT, conn->rx_thread, 1);
4145                         kthread_stop(conn->rx_thread);
4146                 }
4147         }
4148
4149         spin_lock(&iscsit_global->ts_bitmap_lock);
4150         bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
4151                               get_order(1));
4152         spin_unlock(&iscsit_global->ts_bitmap_lock);
4153
4154         iscsit_stop_timers_for_cmds(conn);
4155         iscsit_stop_nopin_response_timer(conn);
4156         iscsit_stop_nopin_timer(conn);
4157
4158         if (conn->conn_transport->iscsit_wait_conn)
4159                 conn->conn_transport->iscsit_wait_conn(conn);
4160
4161         /*
4162          * During Connection recovery drop unacknowledged out of order
4163          * commands for this connection, and prepare the other commands
4164          * for reallegiance.
4165          *
4166          * During normal operation clear the out of order commands (but
4167          * do not free the struct iscsi_ooo_cmdsn's) and release all
4168          * struct iscsi_cmds.
4169          */
4170         if (atomic_read(&conn->connection_recovery)) {
4171                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
4172                 iscsit_prepare_cmds_for_reallegiance(conn);
4173         } else {
4174                 iscsit_clear_ooo_cmdsns_for_conn(conn);
4175                 iscsit_release_commands_from_conn(conn);
4176         }
4177         iscsit_free_queue_reqs_for_conn(conn);
4178
4179         /*
4180          * Handle decrementing session or connection usage count if
4181          * a logout response was not able to be sent because the
4182          * connection failed.  Fall back to Session Recovery here.
4183          */
4184         if (atomic_read(&conn->conn_logout_remove)) {
4185                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4186                         iscsit_dec_conn_usage_count(conn);
4187                         iscsit_dec_session_usage_count(sess);
4188                 }
4189                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4190                         iscsit_dec_conn_usage_count(conn);
4191
4192                 atomic_set(&conn->conn_logout_remove, 0);
4193                 atomic_set(&sess->session_reinstatement, 0);
4194                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4195         }
4196
4197         spin_lock_bh(&sess->conn_lock);
4198         list_del(&conn->conn_list);
4199
4200         /*
4201          * Attempt to let the Initiator know this connection failed by
4202          * sending an Connection Dropped Async Message on another
4203          * active connection.
4204          */
4205         if (atomic_read(&conn->connection_recovery))
4206                 iscsit_build_conn_drop_async_message(conn);
4207
4208         spin_unlock_bh(&sess->conn_lock);
4209
4210         /*
4211          * If connection reinstatement is being performed on this connection,
4212          * up the connection reinstatement semaphore that is being blocked on
4213          * in iscsit_cause_connection_reinstatement().
4214          */
4215         spin_lock_bh(&conn->state_lock);
4216         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4217                 spin_unlock_bh(&conn->state_lock);
4218                 complete(&conn->conn_wait_comp);
4219                 wait_for_completion(&conn->conn_post_wait_comp);
4220                 spin_lock_bh(&conn->state_lock);
4221         }
4222
4223         /*
4224          * If connection reinstatement is being performed on this connection
4225          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4226          * connection wait rcfr semaphore that is being blocked on
4227          * an iscsit_connection_reinstatement_rcfr().
4228          */
4229         if (atomic_read(&conn->connection_wait_rcfr)) {
4230                 spin_unlock_bh(&conn->state_lock);
4231                 complete(&conn->conn_wait_rcfr_comp);
4232                 wait_for_completion(&conn->conn_post_wait_comp);
4233                 spin_lock_bh(&conn->state_lock);
4234         }
4235         atomic_set(&conn->connection_reinstatement, 1);
4236         spin_unlock_bh(&conn->state_lock);
4237
4238         /*
4239          * If any other processes are accessing this connection pointer we
4240          * must wait until they have completed.
4241          */
4242         iscsit_check_conn_usage_count(conn);
4243
4244         ahash_request_free(conn->conn_tx_hash);
4245         if (conn->conn_rx_hash) {
4246                 struct crypto_ahash *tfm;
4247
4248                 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
4249                 ahash_request_free(conn->conn_rx_hash);
4250                 crypto_free_ahash(tfm);
4251         }
4252
4253         if (conn->sock)
4254                 sock_release(conn->sock);
4255
4256         if (conn->conn_transport->iscsit_free_conn)
4257                 conn->conn_transport->iscsit_free_conn(conn);
4258
4259         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4260         conn->conn_state = TARG_CONN_STATE_FREE;
4261         iscsit_free_conn(conn);
4262
4263         spin_lock_bh(&sess->conn_lock);
4264         atomic_dec(&sess->nconn);
4265         pr_debug("Decremented iSCSI connection count to %hu from node:"
4266                 " %s\n", atomic_read(&sess->nconn),
4267                 sess->sess_ops->InitiatorName);
4268         /*
4269          * Make sure that if one connection fails in an non ERL=2 iSCSI
4270          * Session that they all fail.
4271          */
4272         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4273              !atomic_read(&sess->session_logout))
4274                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4275
4276         /*
4277          * If this was not the last connection in the session, and we are
4278          * performing session reinstatement or falling back to ERL=0, call
4279          * iscsit_stop_session() without sleeping to shutdown the other
4280          * active connections.
4281          */
4282         if (atomic_read(&sess->nconn)) {
4283                 if (!atomic_read(&sess->session_reinstatement) &&
4284                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4285                         spin_unlock_bh(&sess->conn_lock);
4286                         return 0;
4287                 }
4288                 if (!atomic_read(&sess->session_stop_active)) {
4289                         atomic_set(&sess->session_stop_active, 1);
4290                         spin_unlock_bh(&sess->conn_lock);
4291                         iscsit_stop_session(sess, 0, 0);
4292                         return 0;
4293                 }
4294                 spin_unlock_bh(&sess->conn_lock);
4295                 return 0;
4296         }
4297
4298         /*
4299          * If this was the last connection in the session and one of the
4300          * following is occurring:
4301          *
4302          * Session Reinstatement is not being performed, and are falling back
4303          * to ERL=0 call iscsit_close_session().
4304          *
4305          * Session Logout was requested.  iscsit_close_session() will be called
4306          * elsewhere.
4307          *
4308          * Session Continuation is not being performed, start the Time2Retain
4309          * handler and check if sleep_on_sess_wait_sem is active.
4310          */
4311         if (!atomic_read(&sess->session_reinstatement) &&
4312              atomic_read(&sess->session_fall_back_to_erl0)) {
4313                 spin_unlock_bh(&sess->conn_lock);
4314                 iscsit_close_session(sess);
4315
4316                 return 0;
4317         } else if (atomic_read(&sess->session_logout)) {
4318                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4319                 sess->session_state = TARG_SESS_STATE_FREE;
4320                 spin_unlock_bh(&sess->conn_lock);
4321
4322                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4323                         complete(&sess->session_wait_comp);
4324
4325                 return 0;
4326         } else {
4327                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4328                 sess->session_state = TARG_SESS_STATE_FAILED;
4329
4330                 if (!atomic_read(&sess->session_continuation)) {
4331                         spin_unlock_bh(&sess->conn_lock);
4332                         iscsit_start_time2retain_handler(sess);
4333                 } else
4334                         spin_unlock_bh(&sess->conn_lock);
4335
4336                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4337                         complete(&sess->session_wait_comp);
4338
4339                 return 0;
4340         }
4341 }
4342
4343 /*
4344  * If the iSCSI Session for the iSCSI Initiator Node exists,
4345  * forcefully shutdown the iSCSI NEXUS.
4346  */
4347 int iscsit_close_session(struct iscsi_session *sess)
4348 {
4349         struct iscsi_portal_group *tpg = sess->tpg;
4350         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4351
4352         if (atomic_read(&sess->nconn)) {
4353                 pr_err("%d connection(s) still exist for iSCSI session"
4354                         " to %s\n", atomic_read(&sess->nconn),
4355                         sess->sess_ops->InitiatorName);
4356                 BUG();
4357         }
4358
4359         spin_lock_bh(&se_tpg->session_lock);
4360         atomic_set(&sess->session_logout, 1);
4361         atomic_set(&sess->session_reinstatement, 1);
4362         iscsit_stop_time2retain_timer(sess);
4363         spin_unlock_bh(&se_tpg->session_lock);
4364
4365         /*
4366          * transport_deregister_session_configfs() will clear the
4367          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4368          * can be setting it again with __transport_register_session() in
4369          * iscsi_post_login_handler() again after the iscsit_stop_session()
4370          * completes in iscsi_np context.
4371          */
4372         transport_deregister_session_configfs(sess->se_sess);
4373
4374         /*
4375          * If any other processes are accessing this session pointer we must
4376          * wait until they have completed.  If we are in an interrupt (the
4377          * time2retain handler) and contain and active session usage count we
4378          * restart the timer and exit.
4379          */
4380         if (!in_interrupt()) {
4381                 if (iscsit_check_session_usage_count(sess) == 1)
4382                         iscsit_stop_session(sess, 1, 1);
4383         } else {
4384                 if (iscsit_check_session_usage_count(sess) == 2) {
4385                         atomic_set(&sess->session_logout, 0);
4386                         iscsit_start_time2retain_handler(sess);
4387                         return 0;
4388                 }
4389         }
4390
4391         transport_deregister_session(sess->se_sess);
4392
4393         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4394                 iscsit_free_connection_recovery_entries(sess);
4395
4396         iscsit_free_all_ooo_cmdsns(sess);
4397
4398         spin_lock_bh(&se_tpg->session_lock);
4399         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4400         sess->session_state = TARG_SESS_STATE_FREE;
4401         pr_debug("Released iSCSI session from node: %s\n",
4402                         sess->sess_ops->InitiatorName);
4403         tpg->nsessions--;
4404         if (tpg->tpg_tiqn)
4405                 tpg->tpg_tiqn->tiqn_nsessions--;
4406
4407         pr_debug("Decremented number of active iSCSI Sessions on"
4408                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4409
4410         ida_free(&sess_ida, sess->session_index);
4411         kfree(sess->sess_ops);
4412         sess->sess_ops = NULL;
4413         spin_unlock_bh(&se_tpg->session_lock);
4414
4415         kfree(sess);
4416         return 0;
4417 }
4418
4419 static void iscsit_logout_post_handler_closesession(
4420         struct iscsi_conn *conn)
4421 {
4422         struct iscsi_session *sess = conn->sess;
4423         int sleep = 1;
4424         /*
4425          * Traditional iscsi/tcp will invoke this logic from TX thread
4426          * context during session logout, so clear tx_thread_active and
4427          * sleep if iscsit_close_connection() has not already occured.
4428          *
4429          * Since iser-target invokes this logic from it's own workqueue,
4430          * always sleep waiting for RX/TX thread shutdown to complete
4431          * within iscsit_close_connection().
4432          */
4433         if (!conn->conn_transport->rdma_shutdown) {
4434                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4435                 if (!sleep)
4436                         return;
4437         }
4438
4439         atomic_set(&conn->conn_logout_remove, 0);
4440         complete(&conn->conn_logout_comp);
4441
4442         iscsit_dec_conn_usage_count(conn);
4443         iscsit_stop_session(sess, sleep, sleep);
4444         iscsit_dec_session_usage_count(sess);
4445         iscsit_close_session(sess);
4446 }
4447
4448 static void iscsit_logout_post_handler_samecid(
4449         struct iscsi_conn *conn)
4450 {
4451         int sleep = 1;
4452
4453         if (!conn->conn_transport->rdma_shutdown) {
4454                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4455                 if (!sleep)
4456                         return;
4457         }
4458
4459         atomic_set(&conn->conn_logout_remove, 0);
4460         complete(&conn->conn_logout_comp);
4461
4462         iscsit_cause_connection_reinstatement(conn, sleep);
4463         iscsit_dec_conn_usage_count(conn);
4464 }
4465
4466 static void iscsit_logout_post_handler_diffcid(
4467         struct iscsi_conn *conn,
4468         u16 cid)
4469 {
4470         struct iscsi_conn *l_conn;
4471         struct iscsi_session *sess = conn->sess;
4472         bool conn_found = false;
4473
4474         if (!sess)
4475                 return;
4476
4477         spin_lock_bh(&sess->conn_lock);
4478         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4479                 if (l_conn->cid == cid) {
4480                         iscsit_inc_conn_usage_count(l_conn);
4481                         conn_found = true;
4482                         break;
4483                 }
4484         }
4485         spin_unlock_bh(&sess->conn_lock);
4486
4487         if (!conn_found)
4488                 return;
4489
4490         if (l_conn->sock)
4491                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4492
4493         spin_lock_bh(&l_conn->state_lock);
4494         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4495         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4496         spin_unlock_bh(&l_conn->state_lock);
4497
4498         iscsit_cause_connection_reinstatement(l_conn, 1);
4499         iscsit_dec_conn_usage_count(l_conn);
4500 }
4501
4502 /*
4503  *      Return of 0 causes the TX thread to restart.
4504  */
4505 int iscsit_logout_post_handler(
4506         struct iscsi_cmd *cmd,
4507         struct iscsi_conn *conn)
4508 {
4509         int ret = 0;
4510
4511         switch (cmd->logout_reason) {
4512         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4513                 switch (cmd->logout_response) {
4514                 case ISCSI_LOGOUT_SUCCESS:
4515                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4516                 default:
4517                         iscsit_logout_post_handler_closesession(conn);
4518                         break;
4519                 }
4520                 ret = 0;
4521                 break;
4522         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4523                 if (conn->cid == cmd->logout_cid) {
4524                         switch (cmd->logout_response) {
4525                         case ISCSI_LOGOUT_SUCCESS:
4526                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4527                         default:
4528                                 iscsit_logout_post_handler_samecid(conn);
4529                                 break;
4530                         }
4531                         ret = 0;
4532                 } else {
4533                         switch (cmd->logout_response) {
4534                         case ISCSI_LOGOUT_SUCCESS:
4535                                 iscsit_logout_post_handler_diffcid(conn,
4536                                         cmd->logout_cid);
4537                                 break;
4538                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4539                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4540                         default:
4541                                 break;
4542                         }
4543                         ret = 1;
4544                 }
4545                 break;
4546         case ISCSI_LOGOUT_REASON_RECOVERY:
4547                 switch (cmd->logout_response) {
4548                 case ISCSI_LOGOUT_SUCCESS:
4549                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4550                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4551                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4552                 default:
4553                         break;
4554                 }
4555                 ret = 1;
4556                 break;
4557         default:
4558                 break;
4559
4560         }
4561         return ret;
4562 }
4563 EXPORT_SYMBOL(iscsit_logout_post_handler);
4564
4565 void iscsit_fail_session(struct iscsi_session *sess)
4566 {
4567         struct iscsi_conn *conn;
4568
4569         spin_lock_bh(&sess->conn_lock);
4570         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4571                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4572                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4573         }
4574         spin_unlock_bh(&sess->conn_lock);
4575
4576         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4577         sess->session_state = TARG_SESS_STATE_FAILED;
4578 }
4579
4580 int iscsit_free_session(struct iscsi_session *sess)
4581 {
4582         u16 conn_count = atomic_read(&sess->nconn);
4583         struct iscsi_conn *conn, *conn_tmp = NULL;
4584         int is_last;
4585
4586         spin_lock_bh(&sess->conn_lock);
4587         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4588
4589         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4590                         conn_list) {
4591                 if (conn_count == 0)
4592                         break;
4593
4594                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4595                         is_last = 1;
4596                 } else {
4597                         iscsit_inc_conn_usage_count(conn_tmp);
4598                         is_last = 0;
4599                 }
4600                 iscsit_inc_conn_usage_count(conn);
4601
4602                 spin_unlock_bh(&sess->conn_lock);
4603                 iscsit_cause_connection_reinstatement(conn, 1);
4604                 spin_lock_bh(&sess->conn_lock);
4605
4606                 iscsit_dec_conn_usage_count(conn);
4607                 if (is_last == 0)
4608                         iscsit_dec_conn_usage_count(conn_tmp);
4609
4610                 conn_count--;
4611         }
4612
4613         if (atomic_read(&sess->nconn)) {
4614                 spin_unlock_bh(&sess->conn_lock);
4615                 wait_for_completion(&sess->session_wait_comp);
4616         } else
4617                 spin_unlock_bh(&sess->conn_lock);
4618
4619         iscsit_close_session(sess);
4620         return 0;
4621 }
4622
4623 void iscsit_stop_session(
4624         struct iscsi_session *sess,
4625         int session_sleep,
4626         int connection_sleep)
4627 {
4628         u16 conn_count = atomic_read(&sess->nconn);
4629         struct iscsi_conn *conn, *conn_tmp = NULL;
4630         int is_last;
4631
4632         spin_lock_bh(&sess->conn_lock);
4633         if (session_sleep)
4634                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4635
4636         if (connection_sleep) {
4637                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4638                                 conn_list) {
4639                         if (conn_count == 0)
4640                                 break;
4641
4642                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4643                                 is_last = 1;
4644                         } else {
4645                                 iscsit_inc_conn_usage_count(conn_tmp);
4646                                 is_last = 0;
4647                         }
4648                         iscsit_inc_conn_usage_count(conn);
4649
4650                         spin_unlock_bh(&sess->conn_lock);
4651                         iscsit_cause_connection_reinstatement(conn, 1);
4652                         spin_lock_bh(&sess->conn_lock);
4653
4654                         iscsit_dec_conn_usage_count(conn);
4655                         if (is_last == 0)
4656                                 iscsit_dec_conn_usage_count(conn_tmp);
4657                         conn_count--;
4658                 }
4659         } else {
4660                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4661                         iscsit_cause_connection_reinstatement(conn, 0);
4662         }
4663
4664         if (session_sleep && atomic_read(&sess->nconn)) {
4665                 spin_unlock_bh(&sess->conn_lock);
4666                 wait_for_completion(&sess->session_wait_comp);
4667         } else
4668                 spin_unlock_bh(&sess->conn_lock);
4669 }
4670
4671 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4672 {
4673         struct iscsi_session *sess;
4674         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4675         struct se_session *se_sess, *se_sess_tmp;
4676         LIST_HEAD(free_list);
4677         int session_count = 0;
4678
4679         spin_lock_bh(&se_tpg->session_lock);
4680         if (tpg->nsessions && !force) {
4681                 spin_unlock_bh(&se_tpg->session_lock);
4682                 return -1;
4683         }
4684
4685         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4686                         sess_list) {
4687                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4688
4689                 spin_lock(&sess->conn_lock);
4690                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4691                     atomic_read(&sess->session_logout) ||
4692                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4693                         spin_unlock(&sess->conn_lock);
4694                         continue;
4695                 }
4696                 atomic_set(&sess->session_reinstatement, 1);
4697                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4698                 spin_unlock(&sess->conn_lock);
4699
4700                 list_move_tail(&se_sess->sess_list, &free_list);
4701         }
4702         spin_unlock_bh(&se_tpg->session_lock);
4703
4704         list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) {
4705                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4706
4707                 iscsit_free_session(sess);
4708                 session_count++;
4709         }
4710
4711         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4712                         " Group: %hu\n", session_count, tpg->tpgt);
4713         return 0;
4714 }
4715
4716 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4717 MODULE_VERSION("4.1.x");
4718 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4719 MODULE_LICENSE("GPL");
4720
4721 module_init(iscsi_target_init_module);
4722 module_exit(iscsi_target_cleanup_module);