drivers/target/iscsi/iscsi_target.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*******************************************************************************
   3  * This file contains main functions related to the iSCSI Target Core Driver.
   4  *
   5  * (c) Copyright 2007-2013 Datera, Inc.
   6  *
   7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  ******************************************************************************/
  10
  11 #include <crypto/hash.h>
  12 #include <linux/string.h>
  13 #include <linux/kthread.h>
  14 #include <linux/completion.h>
  15 #include <linux/module.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/idr.h>
  18 #include <linux/delay.h>
  19 #include <linux/sched/signal.h>
  20 #include <asm/unaligned.h>
  21 #include <linux/inet.h>
  22 #include <net/ipv6.h>
  23 #include <scsi/scsi_proto.h>
  24 #include <scsi/iscsi_proto.h>
  25 #include <scsi/scsi_tcq.h>
  26 #include <target/target_core_base.h>
  27 #include <target/target_core_fabric.h>
  28
  29 #include <target/iscsi/iscsi_target_core.h>
  30 #include "iscsi_target_parameters.h"
  31 #include "iscsi_target_seq_pdu_list.h"
  32 #include "iscsi_target_datain_values.h"
  33 #include "iscsi_target_erl0.h"
  34 #include "iscsi_target_erl1.h"
  35 #include "iscsi_target_erl2.h"
  36 #include "iscsi_target_login.h"
  37 #include "iscsi_target_tmr.h"
  38 #include "iscsi_target_tpg.h"
  39 #include "iscsi_target_util.h"
  40 #include "iscsi_target.h"
  41 #include "iscsi_target_device.h"
  42 #include <target/iscsi/iscsi_target_stat.h>
  43
  44 #include <target/iscsi/iscsi_transport.h>
  45
  46 static LIST_HEAD(g_tiqn_list);
  47 static LIST_HEAD(g_np_list);
  48 static DEFINE_SPINLOCK(tiqn_lock);
  49 static DEFINE_MUTEX(np_lock);
  50
  51 static struct idr tiqn_idr;
  52 DEFINE_IDA(sess_ida);
  53 struct mutex auth_id_lock;
  54
  55 struct iscsit_global *iscsit_global;
  56
  57 struct kmem_cache *lio_qr_cache;
  58 struct kmem_cache *lio_dr_cache;
  59 struct kmem_cache *lio_ooo_cache;
  60 struct kmem_cache *lio_r2t_cache;
  61
  62 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  63                         struct iscsi_scsi_req *, u32);
  64
  65 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  66 {
  67         struct iscsi_tiqn *tiqn = NULL;
  68
  69         spin_lock(&tiqn_lock);
  70         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  71                 if (!strcmp(tiqn->tiqn, buf)) {
  72
  73                         spin_lock(&tiqn->tiqn_state_lock);
  74                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  75                                 tiqn->tiqn_access_count++;
  76                                 spin_unlock(&tiqn->tiqn_state_lock);
  77                                 spin_unlock(&tiqn_lock);
  78                                 return tiqn;
  79                         }
  80                         spin_unlock(&tiqn->tiqn_state_lock);
  81                 }
  82         }
  83         spin_unlock(&tiqn_lock);
  84
  85         return NULL;
  86 }
  87
  88 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  89 {
  90         spin_lock(&tiqn->tiqn_state_lock);
  91         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  92                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
  93                 spin_unlock(&tiqn->tiqn_state_lock);
  94                 return 0;
  95         }
  96         spin_unlock(&tiqn->tiqn_state_lock);
  97
  98         return -1;
  99 }
 100
 101 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 102 {
 103         spin_lock(&tiqn->tiqn_state_lock);
 104         tiqn->tiqn_access_count--;
 105         spin_unlock(&tiqn->tiqn_state_lock);
 106 }
 107
 108 /*
 109  * Note that IQN formatting is expected to be done in userspace, and
 110  * no explict IQN format checks are done here.
 111  */
 112 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 113 {
 114         struct iscsi_tiqn *tiqn = NULL;
 115         int ret;
 116
 117         if (strlen(buf) >= ISCSI_IQN_LEN) {
 118                 pr_err("Target IQN exceeds %d bytes\n",
 119                                 ISCSI_IQN_LEN);
 120                 return ERR_PTR(-EINVAL);
 121         }
 122
 123         tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL);
 124         if (!tiqn)
 125                 return ERR_PTR(-ENOMEM);
 126
 127         sprintf(tiqn->tiqn, "%s", buf);
 128         INIT_LIST_HEAD(&tiqn->tiqn_list);
 129         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 130         spin_lock_init(&tiqn->tiqn_state_lock);
 131         spin_lock_init(&tiqn->tiqn_tpg_lock);
 132         spin_lock_init(&tiqn->sess_err_stats.lock);
 133         spin_lock_init(&tiqn->login_stats.lock);
 134         spin_lock_init(&tiqn->logout_stats.lock);
 135
 136         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 137
 138         idr_preload(GFP_KERNEL);
 139         spin_lock(&tiqn_lock);
 140
 141         ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
 142         if (ret < 0) {
 143                 pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
 144                 spin_unlock(&tiqn_lock);
 145                 idr_preload_end();
 146                 kfree(tiqn);
 147                 return ERR_PTR(ret);
 148         }
 149         tiqn->tiqn_index = ret;
 150         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 151
 152         spin_unlock(&tiqn_lock);
 153         idr_preload_end();
 154
 155         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 156
 157         return tiqn;
 158
 159 }
 160
 161 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 162 {
 163         /*
 164          * Wait for accesses to said struct iscsi_tiqn to end.
 165          */
 166         spin_lock(&tiqn->tiqn_state_lock);
 167         while (tiqn->tiqn_access_count != 0) {
 168                 spin_unlock(&tiqn->tiqn_state_lock);
 169                 msleep(10);
 170                 spin_lock(&tiqn->tiqn_state_lock);
 171         }
 172         spin_unlock(&tiqn->tiqn_state_lock);
 173 }
 174
 175 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 176 {
 177         /*
 178          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 179          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 180          * attempts to access this struct iscsi_tiqn will fail from both transport
 181          * fabric and control code paths.
 182          */
 183         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 184                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 185                 return;
 186         }
 187
 188         iscsit_wait_for_tiqn(tiqn);
 189
 190         spin_lock(&tiqn_lock);
 191         list_del(&tiqn->tiqn_list);
 192         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 193         spin_unlock(&tiqn_lock);
 194
 195         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 196                         tiqn->tiqn);
 197         kfree(tiqn);
 198 }
 199
 200 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 201 {
 202         int ret;
 203         /*
 204          * Determine if the network portal is accepting storage traffic.
 205          */
 206         spin_lock_bh(&np->np_thread_lock);
 207         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 208                 spin_unlock_bh(&np->np_thread_lock);
 209                 return -1;
 210         }
 211         spin_unlock_bh(&np->np_thread_lock);
 212         /*
 213          * Determine if the portal group is accepting storage traffic.
 214          */
 215         spin_lock_bh(&tpg->tpg_state_lock);
 216         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 217                 spin_unlock_bh(&tpg->tpg_state_lock);
 218                 return -1;
 219         }
 220         spin_unlock_bh(&tpg->tpg_state_lock);
 221
 222         /*
 223          * Here we serialize access across the TIQN+TPG Tuple.
 224          */
 225         ret = down_interruptible(&tpg->np_login_sem);
 226         if (ret != 0)
 227                 return -1;
 228
 229         spin_lock_bh(&tpg->tpg_state_lock);
 230         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 231                 spin_unlock_bh(&tpg->tpg_state_lock);
 232                 up(&tpg->np_login_sem);
 233                 return -1;
 234         }
 235         spin_unlock_bh(&tpg->tpg_state_lock);
 236
 237         return 0;
 238 }
 239
 240 void iscsit_login_kref_put(struct kref *kref)
 241 {
 242         struct iscsi_tpg_np *tpg_np = container_of(kref,
 243                                 struct iscsi_tpg_np, tpg_np_kref);
 244
 245         complete(&tpg_np->tpg_np_comp);
 246 }
 247
 248 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg,
 249                        struct iscsi_tpg_np *tpg_np)
 250 {
 251         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 252
 253         up(&tpg->np_login_sem);
 254
 255         if (tpg_np)
 256                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 257
 258         if (tiqn)
 259                 iscsit_put_tiqn_for_login(tiqn);
 260
 261         return 0;
 262 }
 263
 264 bool iscsit_check_np_match(
 265         struct sockaddr_storage *sockaddr,
 266         struct iscsi_np *np,
 267         int network_transport)
 268 {
 269         struct sockaddr_in *sock_in, *sock_in_e;
 270         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 271         bool ip_match = false;
 272         u16 port, port_e;
 273
 274         if (sockaddr->ss_family == AF_INET6) {
 275                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 276                 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 277
 278                 if (!memcmp(&sock_in6->sin6_addr.in6_u,
 279                             &sock_in6_e->sin6_addr.in6_u,
 280                             sizeof(struct in6_addr)))
 281                         ip_match = true;
 282
 283                 port = ntohs(sock_in6->sin6_port);
 284                 port_e = ntohs(sock_in6_e->sin6_port);
 285         } else {
 286                 sock_in = (struct sockaddr_in *)sockaddr;
 287                 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 288
 289                 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
 290                         ip_match = true;
 291
 292                 port = ntohs(sock_in->sin_port);
 293                 port_e = ntohs(sock_in_e->sin_port);
 294         }
 295
 296         if (ip_match && (port_e == port) &&
 297             (np->np_network_transport == network_transport))
 298                 return true;
 299
 300         return false;
 301 }
 302
 303 static struct iscsi_np *iscsit_get_np(
 304         struct sockaddr_storage *sockaddr,
 305         int network_transport)
 306 {
 307         struct iscsi_np *np;
 308         bool match;
 309
 310         lockdep_assert_held(&np_lock);
 311
 312         list_for_each_entry(np, &g_np_list, np_list) {
 313                 spin_lock_bh(&np->np_thread_lock);
 314                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 315                         spin_unlock_bh(&np->np_thread_lock);
 316                         continue;
 317                 }
 318
 319                 match = iscsit_check_np_match(sockaddr, np, network_transport);
 320                 if (match) {
 321                         /*
 322                          * Increment the np_exports reference count now to
 323                          * prevent iscsit_del_np() below from being called
 324                          * while iscsi_tpg_add_network_portal() is called.
 325                          */
 326                         np->np_exports++;
 327                         spin_unlock_bh(&np->np_thread_lock);
 328                         return np;
 329                 }
 330                 spin_unlock_bh(&np->np_thread_lock);
 331         }
 332
 333         return NULL;
 334 }
 335
 336 struct iscsi_np *iscsit_add_np(
 337         struct sockaddr_storage *sockaddr,
 338         int network_transport)
 339 {
 340         struct iscsi_np *np;
 341         int ret;
 342
 343         mutex_lock(&np_lock);
 344
 345         /*
 346          * Locate the existing struct iscsi_np if already active..
 347          */
 348         np = iscsit_get_np(sockaddr, network_transport);
 349         if (np) {
 350                 mutex_unlock(&np_lock);
 351                 return np;
 352         }
 353
 354         np = kzalloc(sizeof(*np), GFP_KERNEL);
 355         if (!np) {
 356                 mutex_unlock(&np_lock);
 357                 return ERR_PTR(-ENOMEM);
 358         }
 359
 360         np->np_flags |= NPF_IP_NETWORK;
 361         np->np_network_transport = network_transport;
 362         spin_lock_init(&np->np_thread_lock);
 363         init_completion(&np->np_restart_comp);
 364         INIT_LIST_HEAD(&np->np_list);
 365
 366         timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0);
 367
 368         ret = iscsi_target_setup_login_socket(np, sockaddr);
 369         if (ret != 0) {
 370                 kfree(np);
 371                 mutex_unlock(&np_lock);
 372                 return ERR_PTR(ret);
 373         }
 374
 375         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 376         if (IS_ERR(np->np_thread)) {
 377                 pr_err("Unable to create kthread: iscsi_np\n");
 378                 ret = PTR_ERR(np->np_thread);
 379                 kfree(np);
 380                 mutex_unlock(&np_lock);
 381                 return ERR_PTR(ret);
 382         }
 383         /*
 384          * Increment the np_exports reference count now to prevent
 385          * iscsit_del_np() below from being run while a new call to
 386          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 387          * active.  We don't need to hold np->np_thread_lock at this
 388          * point because iscsi_np has not been added to g_np_list yet.
 389          */
 390         np->np_exports = 1;
 391         np->np_thread_state = ISCSI_NP_THREAD_ACTIVE;
 392
 393         list_add_tail(&np->np_list, &g_np_list);
 394         mutex_unlock(&np_lock);
 395
 396         pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n",
 397                 &np->np_sockaddr, np->np_transport->name);
 398
 399         return np;
 400 }
 401
 402 int iscsit_reset_np_thread(
 403         struct iscsi_np *np,
 404         struct iscsi_tpg_np *tpg_np,
 405         struct iscsi_portal_group *tpg,
 406         bool shutdown)
 407 {
 408         spin_lock_bh(&np->np_thread_lock);
 409         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 410                 spin_unlock_bh(&np->np_thread_lock);
 411                 return 0;
 412         }
 413         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 414         atomic_inc(&np->np_reset_count);
 415
 416         if (np->np_thread) {
 417                 spin_unlock_bh(&np->np_thread_lock);
 418                 send_sig(SIGINT, np->np_thread, 1);
 419                 wait_for_completion(&np->np_restart_comp);
 420                 spin_lock_bh(&np->np_thread_lock);
 421         }
 422         spin_unlock_bh(&np->np_thread_lock);
 423
 424         if (tpg_np && shutdown) {
 425                 kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
 426
 427                 wait_for_completion(&tpg_np->tpg_np_comp);
 428         }
 429
 430         return 0;
 431 }
 432
 433 static void iscsit_free_np(struct iscsi_np *np)
 434 {
 435         if (np->np_socket)
 436                 sock_release(np->np_socket);
 437 }
 438
 439 int iscsit_del_np(struct iscsi_np *np)
 440 {
 441         spin_lock_bh(&np->np_thread_lock);
 442         np->np_exports--;
 443         if (np->np_exports) {
 444                 np->enabled = true;
 445                 spin_unlock_bh(&np->np_thread_lock);
 446                 return 0;
 447         }
 448         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 449         spin_unlock_bh(&np->np_thread_lock);
 450
 451         if (np->np_thread) {
 452                 /*
 453                  * We need to send the signal to wakeup Linux/Net
 454                  * which may be sleeping in sock_accept()..
 455                  */
 456                 send_sig(SIGINT, np->np_thread, 1);
 457                 kthread_stop(np->np_thread);
 458                 np->np_thread = NULL;
 459         }
 460
 461         np->np_transport->iscsit_free_np(np);
 462
 463         mutex_lock(&np_lock);
 464         list_del(&np->np_list);
 465         mutex_unlock(&np_lock);
 466
 467         pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n",
 468                 &np->np_sockaddr, np->np_transport->name);
 469
 470         iscsit_put_transport(np->np_transport);
 471         kfree(np);
 472         return 0;
 473 }
 474
 475 static void iscsit_get_rx_pdu(struct iscsi_conn *);
 476
 477 int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 478 {
 479         return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
 480 }
 481 EXPORT_SYMBOL(iscsit_queue_rsp);
 482
 483 void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
 484 {
 485         spin_lock_bh(&conn->cmd_lock);
 486         if (!list_empty(&cmd->i_conn_node))
 487                 list_del_init(&cmd->i_conn_node);
 488         spin_unlock_bh(&conn->cmd_lock);
 489
 490         __iscsit_free_cmd(cmd, true);
 491 }
 492 EXPORT_SYMBOL(iscsit_aborted_task);
 493
 494 static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
 495                                       u32, u32, const void *, void *);
 496 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *);
 497
 498 static int
 499 iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 500                           const void *data_buf, u32 data_buf_len)
 501 {
 502         struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu;
 503         struct kvec *iov;
 504         u32 niov = 0, tx_size = ISCSI_HDR_LEN;
 505         int ret;
 506
 507         iov = &cmd->iov_misc[0];
 508         iov[niov].iov_base      = cmd->pdu;
 509         iov[niov++].iov_len     = ISCSI_HDR_LEN;
 510
 511         if (conn->conn_ops->HeaderDigest) {
 512                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 513
 514                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
 515                                           ISCSI_HDR_LEN, 0, NULL,
 516                                           header_digest);
 517
 518                 iov[0].iov_len += ISCSI_CRC_LEN;
 519                 tx_size += ISCSI_CRC_LEN;
 520                 pr_debug("Attaching CRC32C HeaderDigest"
 521                          " to opcode 0x%x 0x%08x\n",
 522                          hdr->opcode, *header_digest);
 523         }
 524
 525         if (data_buf_len) {
 526                 u32 padding = ((-data_buf_len) & 3);
 527
 528                 iov[niov].iov_base      = (void *)data_buf;
 529                 iov[niov++].iov_len     = data_buf_len;
 530                 tx_size += data_buf_len;
 531
 532                 if (padding != 0) {
 533                         iov[niov].iov_base = &cmd->pad_bytes;
 534                         iov[niov++].iov_len = padding;
 535                         tx_size += padding;
 536                         pr_debug("Attaching %u additional"
 537                                  " padding bytes.\n", padding);
 538                 }
 539
 540                 if (conn->conn_ops->DataDigest) {
 541                         iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
 542                                                   data_buf, data_buf_len,
 543                                                   padding, &cmd->pad_bytes,
 544                                                   &cmd->data_crc);
 545
 546                         iov[niov].iov_base = &cmd->data_crc;
 547                         iov[niov++].iov_len = ISCSI_CRC_LEN;
 548                         tx_size += ISCSI_CRC_LEN;
 549                         pr_debug("Attached DataDigest for %u"
 550                                  " bytes opcode 0x%x, CRC 0x%08x\n",
 551                                  data_buf_len, hdr->opcode, cmd->data_crc);
 552                 }
 553         }
 554
 555         cmd->iov_misc_count = niov;
 556         cmd->tx_size = tx_size;
 557
 558         ret = iscsit_send_tx_data(cmd, conn, 1);
 559         if (ret < 0) {
 560                 iscsit_tx_thread_wait_for_tcp(conn);
 561                 return ret;
 562         }
 563
 564         return 0;
 565 }
 566
 567 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
 568                             u32 data_offset, u32 data_length);
 569 static void iscsit_unmap_iovec(struct iscsi_cmd *);
 570 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *,
 571                                     u32, u32, u32, u8 *);
 572 static int
 573 iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 574                        const struct iscsi_datain *datain)
 575 {
 576         struct kvec *iov;
 577         u32 iov_count = 0, tx_size = 0;
 578         int ret, iov_ret;
 579
 580         iov = &cmd->iov_data[0];
 581         iov[iov_count].iov_base = cmd->pdu;
 582         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
 583         tx_size += ISCSI_HDR_LEN;
 584
 585         if (conn->conn_ops->HeaderDigest) {
 586                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
 587
 588                 iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
 589                                           ISCSI_HDR_LEN, 0, NULL,
 590                                           header_digest);
 591
 592                 iov[0].iov_len += ISCSI_CRC_LEN;
 593                 tx_size += ISCSI_CRC_LEN;
 594
 595                 pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n",
 596                          *header_digest);
 597         }
 598
 599         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count],
 600                                    cmd->orig_iov_data_count - (iov_count + 2),
 601                                    datain->offset, datain->length);
 602         if (iov_ret < 0)
 603                 return -1;
 604
 605         iov_count += iov_ret;
 606         tx_size += datain->length;
 607
 608         cmd->padding = ((-datain->length) & 3);
 609         if (cmd->padding) {
 610                 iov[iov_count].iov_base         = cmd->pad_bytes;
 611                 iov[iov_count++].iov_len        = cmd->padding;
 612                 tx_size += cmd->padding;
 613
 614                 pr_debug("Attaching %u padding bytes\n", cmd->padding);
 615         }
 616
 617         if (conn->conn_ops->DataDigest) {
 618                 cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
 619                                                          cmd, datain->offset,
 620                                                          datain->length,
 621                                                          cmd->padding,
 622                                                          cmd->pad_bytes);
 623
 624                 iov[iov_count].iov_base = &cmd->data_crc;
 625                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
 626                 tx_size += ISCSI_CRC_LEN;
 627
 628                 pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n",
 629                          datain->length + cmd->padding, cmd->data_crc);
 630         }
 631
 632         cmd->iov_data_count = iov_count;
 633         cmd->tx_size = tx_size;
 634
 635         ret = iscsit_fe_sendpage_sg(cmd, conn);
 636
 637         iscsit_unmap_iovec(cmd);
 638
 639         if (ret < 0) {
 640                 iscsit_tx_thread_wait_for_tcp(conn);
 641                 return ret;
 642         }
 643
 644         return 0;
 645 }
 646
 647 static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 648                            struct iscsi_datain_req *dr, const void *buf,
 649                            u32 buf_len)
 650 {
 651         if (dr)
 652                 return iscsit_xmit_datain_pdu(conn, cmd, buf);
 653         else
 654                 return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
 655 }
 656
 657 static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn)
 658 {
 659         return TARGET_PROT_NORMAL;
 660 }
 661
 662 static struct iscsit_transport iscsi_target_transport = {
 663         .name                   = "iSCSI/TCP",
 664         .transport_type         = ISCSI_TCP,
 665         .rdma_shutdown          = false,
 666         .owner                  = NULL,
 667         .iscsit_setup_np        = iscsit_setup_np,
 668         .iscsit_accept_np       = iscsit_accept_np,
 669         .iscsit_free_np         = iscsit_free_np,
 670         .iscsit_get_login_rx    = iscsit_get_login_rx,
 671         .iscsit_put_login_tx    = iscsit_put_login_tx,
 672         .iscsit_get_dataout     = iscsit_build_r2ts_for_cmd,
 673         .iscsit_immediate_queue = iscsit_immediate_queue,
 674         .iscsit_response_queue  = iscsit_response_queue,
 675         .iscsit_queue_data_in   = iscsit_queue_rsp,
 676         .iscsit_queue_status    = iscsit_queue_rsp,
 677         .iscsit_aborted_task    = iscsit_aborted_task,
 678         .iscsit_xmit_pdu        = iscsit_xmit_pdu,
 679         .iscsit_get_rx_pdu      = iscsit_get_rx_pdu,
 680         .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops,
 681 };
 682
 683 static int __init iscsi_target_init_module(void)
 684 {
 685         int ret = 0, size;
 686
 687         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 688         iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL);
 689         if (!iscsit_global)
 690                 return -1;
 691
 692         spin_lock_init(&iscsit_global->ts_bitmap_lock);
 693         mutex_init(&auth_id_lock);
 694         idr_init(&tiqn_idr);
 695
 696         ret = target_register_template(&iscsi_ops);
 697         if (ret)
 698                 goto out;
 699
 700         size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
 701         iscsit_global->ts_bitmap = vzalloc(size);
 702         if (!iscsit_global->ts_bitmap)
 703                 goto configfs_out;
 704
 705         if (!zalloc_cpumask_var(&iscsit_global->allowed_cpumask, GFP_KERNEL)) {
 706                 pr_err("Unable to allocate iscsit_global->allowed_cpumask\n");
 707                 goto bitmap_out;
 708         }
 709         cpumask_setall(iscsit_global->allowed_cpumask);
 710
 711         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 712                         sizeof(struct iscsi_queue_req),
 713                         __alignof__(struct iscsi_queue_req), 0, NULL);
 714         if (!lio_qr_cache) {
 715                 pr_err("Unable to kmem_cache_create() for"
 716                                 " lio_qr_cache\n");
 717                 goto cpumask_out;
 718         }
 719
 720         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 721                         sizeof(struct iscsi_datain_req),
 722                         __alignof__(struct iscsi_datain_req), 0, NULL);
 723         if (!lio_dr_cache) {
 724                 pr_err("Unable to kmem_cache_create() for"
 725                                 " lio_dr_cache\n");
 726                 goto qr_out;
 727         }
 728
 729         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 730                         sizeof(struct iscsi_ooo_cmdsn),
 731                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 732         if (!lio_ooo_cache) {
 733                 pr_err("Unable to kmem_cache_create() for"
 734                                 " lio_ooo_cache\n");
 735                 goto dr_out;
 736         }
 737
 738         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 739                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 740                         0, NULL);
 741         if (!lio_r2t_cache) {
 742                 pr_err("Unable to kmem_cache_create() for"
 743                                 " lio_r2t_cache\n");
 744                 goto ooo_out;
 745         }
 746
 747         iscsit_register_transport(&iscsi_target_transport);
 748
 749         if (iscsit_load_discovery_tpg() < 0)
 750                 goto r2t_out;
 751
 752         return ret;
 753 r2t_out:
 754         iscsit_unregister_transport(&iscsi_target_transport);
 755         kmem_cache_destroy(lio_r2t_cache);
 756 ooo_out:
 757         kmem_cache_destroy(lio_ooo_cache);
 758 dr_out:
 759         kmem_cache_destroy(lio_dr_cache);
 760 qr_out:
 761         kmem_cache_destroy(lio_qr_cache);
 762 cpumask_out:
 763         free_cpumask_var(iscsit_global->allowed_cpumask);
 764 bitmap_out:
 765         vfree(iscsit_global->ts_bitmap);
 766 configfs_out:
 767         /* XXX: this probably wants it to be it's own unwind step.. */
 768         if (iscsit_global->discovery_tpg)
 769                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 770         target_unregister_template(&iscsi_ops);
 771 out:
 772         kfree(iscsit_global);
 773         return -ENOMEM;
 774 }
 775
 776 static void __exit iscsi_target_cleanup_module(void)
 777 {
 778         iscsit_release_discovery_tpg();
 779         iscsit_unregister_transport(&iscsi_target_transport);
 780         kmem_cache_destroy(lio_qr_cache);
 781         kmem_cache_destroy(lio_dr_cache);
 782         kmem_cache_destroy(lio_ooo_cache);
 783         kmem_cache_destroy(lio_r2t_cache);
 784
 785         /*
 786          * Shutdown discovery sessions and disable discovery TPG
 787          */
 788         if (iscsit_global->discovery_tpg)
 789                 iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
 790
 791         target_unregister_template(&iscsi_ops);
 792
 793         free_cpumask_var(iscsit_global->allowed_cpumask);
 794         vfree(iscsit_global->ts_bitmap);
 795         kfree(iscsit_global);
 796 }
 797
 798 int iscsit_add_reject(
 799         struct iscsi_conn *conn,
 800         u8 reason,
 801         unsigned char *buf)
 802 {
 803         struct iscsi_cmd *cmd;
 804
 805         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
 806         if (!cmd)
 807                 return -1;
 808
 809         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 810         cmd->reject_reason = reason;
 811
 812         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 813         if (!cmd->buf_ptr) {
 814                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 815                 iscsit_free_cmd(cmd, false);
 816                 return -1;
 817         }
 818
 819         spin_lock_bh(&conn->cmd_lock);
 820         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 821         spin_unlock_bh(&conn->cmd_lock);
 822
 823         cmd->i_state = ISTATE_SEND_REJECT;
 824         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 825
 826         return -1;
 827 }
 828 EXPORT_SYMBOL(iscsit_add_reject);
 829
 830 static int iscsit_add_reject_from_cmd(
 831         struct iscsi_cmd *cmd,
 832         u8 reason,
 833         bool add_to_conn,
 834         unsigned char *buf)
 835 {
 836         struct iscsi_conn *conn;
 837         const bool do_put = cmd->se_cmd.se_tfo != NULL;
 838
 839         if (!cmd->conn) {
 840                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 841                                 cmd->init_task_tag);
 842                 return -1;
 843         }
 844         conn = cmd->conn;
 845
 846         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 847         cmd->reject_reason = reason;
 848
 849         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 850         if (!cmd->buf_ptr) {
 851                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 852                 iscsit_free_cmd(cmd, false);
 853                 return -1;
 854         }
 855
 856         if (add_to_conn) {
 857                 spin_lock_bh(&conn->cmd_lock);
 858                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
 859                 spin_unlock_bh(&conn->cmd_lock);
 860         }
 861
 862         cmd->i_state = ISTATE_SEND_REJECT;
 863         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 864         /*
 865          * Perform the kref_put now if se_cmd has already been setup by
 866          * scsit_setup_scsi_cmd()
 867          */
 868         if (do_put) {
 869                 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
 870                 target_put_sess_cmd(&cmd->se_cmd);
 871         }
 872         return -1;
 873 }
 874
 875 static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason,
 876                                  unsigned char *buf)
 877 {
 878         return iscsit_add_reject_from_cmd(cmd, reason, true, buf);
 879 }
 880
 881 int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf)
 882 {
 883         return iscsit_add_reject_from_cmd(cmd, reason, false, buf);
 884 }
 885 EXPORT_SYMBOL(iscsit_reject_cmd);
 886
 887 /*
 888  * Map some portion of the allocated scatterlist to an iovec, suitable for
 889  * kernel sockets to copy data in/out.
 890  */
 891 static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
 892                             u32 data_offset, u32 data_length)
 893 {
 894         u32 i = 0, orig_data_length = data_length;
 895         struct scatterlist *sg;
 896         unsigned int page_off;
 897
 898         /*
 899          * We know each entry in t_data_sg contains a page.
 900          */
 901         u32 ent = data_offset / PAGE_SIZE;
 902
 903         if (!data_length)
 904                 return 0;
 905
 906         if (ent >= cmd->se_cmd.t_data_nents) {
 907                 pr_err("Initial page entry out-of-bounds\n");
 908                 goto overflow;
 909         }
 910
 911         sg = &cmd->se_cmd.t_data_sg[ent];
 912         page_off = (data_offset % PAGE_SIZE);
 913
 914         cmd->first_data_sg = sg;
 915         cmd->first_data_sg_off = page_off;
 916
 917         while (data_length) {
 918                 u32 cur_len;
 919
 920                 if (WARN_ON_ONCE(!sg || i >= nvec))
 921                         goto overflow;
 922
 923                 cur_len = min_t(u32, data_length, sg->length - page_off);
 924
 925                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 926                 iov[i].iov_len = cur_len;
 927
 928                 data_length -= cur_len;
 929                 page_off = 0;
 930                 sg = sg_next(sg);
 931                 i++;
 932         }
 933
 934         cmd->kmapped_nents = i;
 935
 936         return i;
 937
 938 overflow:
 939         pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n",
 940                data_offset, orig_data_length, i, nvec);
 941         for_each_sg(cmd->se_cmd.t_data_sg, sg,
 942                     cmd->se_cmd.t_data_nents, i) {
 943                 pr_err("[%d] off %d len %d\n",
 944                        i, sg->offset, sg->length);
 945         }
 946         return -1;
 947 }
 948
 949 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 950 {
 951         u32 i;
 952         struct scatterlist *sg;
 953
 954         sg = cmd->first_data_sg;
 955
 956         for (i = 0; i < cmd->kmapped_nents; i++)
 957                 kunmap(sg_page(&sg[i]));
 958 }
 959
 960 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 961 {
 962         LIST_HEAD(ack_list);
 963         struct iscsi_cmd *cmd, *cmd_p;
 964
 965         conn->exp_statsn = exp_statsn;
 966
 967         if (conn->sess->sess_ops->RDMAExtensions)
 968                 return;
 969
 970         spin_lock_bh(&conn->cmd_lock);
 971         list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) {
 972                 spin_lock(&cmd->istate_lock);
 973                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 974                     iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
 975                         cmd->i_state = ISTATE_REMOVE;
 976                         spin_unlock(&cmd->istate_lock);
 977                         list_move_tail(&cmd->i_conn_node, &ack_list);
 978                         continue;
 979                 }
 980                 spin_unlock(&cmd->istate_lock);
 981         }
 982         spin_unlock_bh(&conn->cmd_lock);
 983
 984         list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) {
 985                 list_del_init(&cmd->i_conn_node);
 986                 iscsit_free_cmd(cmd, false);
 987         }
 988 }
 989
 990 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 991 {
 992         u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
 993
 994         iov_count += ISCSI_IOV_DATA_BUFFER;
 995         cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL);
 996         if (!cmd->iov_data)
 997                 return -ENOMEM;
 998
 999         cmd->orig_iov_data_count = iov_count;
1000         return 0;
1001 }
1002
1003 int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1004                           unsigned char *buf)
1005 {
1006         int data_direction, payload_length;
1007         struct iscsi_scsi_req *hdr;
1008         int iscsi_task_attr;
1009         int sam_task_attr;
1010
1011         atomic_long_inc(&conn->sess->cmd_pdus);
1012
1013         hdr                     = (struct iscsi_scsi_req *) buf;
1014         payload_length          = ntoh24(hdr->dlength);
1015
1016         /* FIXME; Add checks for AdditionalHeaderSegment */
1017
1018         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
1019             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1020                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
1021                                 " not set. Bad iSCSI Initiator.\n");
1022                 return iscsit_add_reject_cmd(cmd,
1023                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1024         }
1025
1026         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
1027              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
1028                 /*
1029                  * From RFC-3720 Section 10.3.1:
1030                  *
1031                  * "Either or both of R and W MAY be 1 when either the
1032                  *  Expected Data Transfer Length and/or Bidirectional Read
1033                  *  Expected Data Transfer Length are 0"
1034                  *
1035                  * For this case, go ahead and clear the unnecssary bits
1036                  * to avoid any confusion with ->data_direction.
1037                  */
1038                 hdr->flags &= ~ISCSI_FLAG_CMD_READ;
1039                 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
1040
1041                 pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
1042                         " set when Expected Data Transfer Length is 0 for"
1043                         " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]);
1044         }
1045
1046         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
1047             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
1048                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
1049                         " MUST be set if Expected Data Transfer Length is not 0."
1050                         " Bad iSCSI Initiator\n");
1051                 return iscsit_add_reject_cmd(cmd,
1052                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1053         }
1054
1055         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
1056             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
1057                 pr_err("Bidirectional operations not supported!\n");
1058                 return iscsit_add_reject_cmd(cmd,
1059                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1060         }
1061
1062         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1063                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
1064                                 " Scsi Command PDU.\n");
1065                 return iscsit_add_reject_cmd(cmd,
1066                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1067         }
1068
1069         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
1070                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
1071                         " protocol error.\n", payload_length);
1072                 return iscsit_add_reject_cmd(cmd,
1073                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1074         }
1075
1076         if ((be32_to_cpu(hdr->data_length) == payload_length) &&
1077             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
1078                 pr_err("Expected Data Transfer Length and Length of"
1079                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
1080                         " bit is not set protocol error\n");
1081                 return iscsit_add_reject_cmd(cmd,
1082                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1083         }
1084
1085         if (payload_length > be32_to_cpu(hdr->data_length)) {
1086                 pr_err("DataSegmentLength: %u is greater than"
1087                         " EDTL: %u, protocol error.\n", payload_length,
1088                                 hdr->data_length);
1089                 return iscsit_add_reject_cmd(cmd,
1090                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1091         }
1092
1093         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1094                 pr_err("DataSegmentLength: %u is greater than"
1095                         " MaxXmitDataSegmentLength: %u, protocol error.\n",
1096                         payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
1097                 return iscsit_add_reject_cmd(cmd,
1098                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
1099         }
1100
1101         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
1102                 pr_err("DataSegmentLength: %u is greater than"
1103                         " FirstBurstLength: %u, protocol error.\n",
1104                         payload_length, conn->sess->sess_ops->FirstBurstLength);
1105                 return iscsit_add_reject_cmd(cmd,
1106                                              ISCSI_REASON_BOOKMARK_INVALID, buf);
1107         }
1108
1109         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
1110                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
1111                           DMA_NONE;
1112
1113         cmd->data_direction = data_direction;
1114         iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
1115         /*
1116          * Figure out the SAM Task Attribute for the incoming SCSI CDB
1117          */
1118         if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
1119             (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
1120                 sam_task_attr = TCM_SIMPLE_TAG;
1121         else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
1122                 sam_task_attr = TCM_ORDERED_TAG;
1123         else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
1124                 sam_task_attr = TCM_HEAD_TAG;
1125         else if (iscsi_task_attr == ISCSI_ATTR_ACA)
1126                 sam_task_attr = TCM_ACA_TAG;
1127         else {
1128                 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
1129                         " TCM_SIMPLE_TAG\n", iscsi_task_attr);
1130                 sam_task_attr = TCM_SIMPLE_TAG;
1131         }
1132
1133         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
1134         cmd->i_state            = ISTATE_NEW_CMD;
1135         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1136         cmd->immediate_data     = (payload_length) ? 1 : 0;
1137         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
1138                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
1139         if (cmd->unsolicited_data)
1140                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
1141
1142         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1143         if (hdr->flags & ISCSI_FLAG_CMD_READ)
1144                 cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
1145         else
1146                 cmd->targ_xfer_tag = 0xFFFFFFFF;
1147         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1148         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1149         cmd->first_burst_len    = payload_length;
1150
1151         if (!conn->sess->sess_ops->RDMAExtensions &&
1152              cmd->data_direction == DMA_FROM_DEVICE) {
1153                 struct iscsi_datain_req *dr;
1154
1155                 dr = iscsit_allocate_datain_req();
1156                 if (!dr)
1157                         return iscsit_add_reject_cmd(cmd,
1158                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1159
1160                 iscsit_attach_datain_req(cmd, dr);
1161         }
1162
1163         /*
1164          * Initialize struct se_cmd descriptor from target_core_mod infrastructure
1165          */
1166         __target_init_cmd(&cmd->se_cmd, &iscsi_ops,
1167                          conn->sess->se_sess, be32_to_cpu(hdr->data_length),
1168                          cmd->data_direction, sam_task_attr,
1169                          cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun));
1170
1171         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
1172                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
1173                 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
1174                 conn->cid);
1175
1176         target_get_sess_cmd(&cmd->se_cmd, true);
1177
1178         cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
1179         cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb,
1180                                                 GFP_KERNEL);
1181
1182         if (cmd->sense_reason) {
1183                 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
1184                         return iscsit_add_reject_cmd(cmd,
1185                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1186                 }
1187
1188                 goto attach_cmd;
1189         }
1190
1191         cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd);
1192         if (cmd->sense_reason)
1193                 goto attach_cmd;
1194
1195         cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd);
1196         if (cmd->sense_reason)
1197                 goto attach_cmd;
1198
1199         if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
1200                 return iscsit_add_reject_cmd(cmd,
1201                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1202         }
1203
1204 attach_cmd:
1205         spin_lock_bh(&conn->cmd_lock);
1206         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1207         spin_unlock_bh(&conn->cmd_lock);
1208         /*
1209          * Check if we need to delay processing because of ALUA
1210          * Active/NonOptimized primary access state..
1211          */
1212         core_alua_check_nonop_delay(&cmd->se_cmd);
1213
1214         return 0;
1215 }
1216 EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
1217
1218 void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd)
1219 {
1220         iscsit_set_dataout_sequence_values(cmd);
1221
1222         spin_lock_bh(&cmd->dataout_timeout_lock);
1223         iscsit_start_dataout_timer(cmd, cmd->conn);
1224         spin_unlock_bh(&cmd->dataout_timeout_lock);
1225 }
1226 EXPORT_SYMBOL(iscsit_set_unsolicited_dataout);
1227
1228 int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1229                             struct iscsi_scsi_req *hdr)
1230 {
1231         int cmdsn_ret = 0;
1232         /*
1233          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1234          * the Immediate Bit is not set, and no Immediate
1235          * Data is attached.
1236          *
1237          * A PDU/CmdSN carrying Immediate Data can only
1238          * be processed after the DataCRC has passed.
1239          * If the DataCRC fails, the CmdSN MUST NOT
1240          * be acknowledged. (See below)
1241          */
1242         if (!cmd->immediate_data) {
1243                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1244                                         (unsigned char *)hdr, hdr->cmdsn);
1245                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1246                         return -1;
1247                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1248                         target_put_sess_cmd(&cmd->se_cmd);
1249                         return 0;
1250                 }
1251         }
1252
1253         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1254
1255         /*
1256          * If no Immediate Data is attached, it's OK to return now.
1257          */
1258         if (!cmd->immediate_data) {
1259                 if (!cmd->sense_reason && cmd->unsolicited_data)
1260                         iscsit_set_unsolicited_dataout(cmd);
1261                 if (!cmd->sense_reason)
1262                         return 0;
1263
1264                 target_put_sess_cmd(&cmd->se_cmd);
1265                 return 0;
1266         }
1267
1268         /*
1269          * Early CHECK_CONDITIONs with ImmediateData never make it to command
1270          * execution.  These exceptions are processed in CmdSN order using
1271          * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
1272          */
1273         if (cmd->sense_reason)
1274                 return 1;
1275         /*
1276          * Call directly into transport_generic_new_cmd() to perform
1277          * the backend memory allocation.
1278          */
1279         cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
1280         if (cmd->sense_reason)
1281                 return 1;
1282
1283         return 0;
1284 }
1285 EXPORT_SYMBOL(iscsit_process_scsi_cmd);
1286
1287 static int
1288 iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
1289                           bool dump_payload)
1290 {
1291         int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1292         int rc;
1293
1294         /*
1295          * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
1296          */
1297         if (dump_payload) {
1298                 u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done,
1299                                  cmd->first_burst_len);
1300
1301                 pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n",
1302                          cmd->se_cmd.data_length, cmd->write_data_done,
1303                          cmd->first_burst_len, length);
1304                 rc = iscsit_dump_data_payload(cmd->conn, length, 1);
1305                 pr_debug("Finished dumping immediate data\n");
1306                 if (rc < 0)
1307                         immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER;
1308         } else {
1309                 immed_ret = iscsit_handle_immediate_data(cmd, hdr,
1310                                                          cmd->first_burst_len);
1311         }
1312
1313         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1314                 /*
1315                  * A PDU/CmdSN carrying Immediate Data passed
1316                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1317                  * Immediate Bit is not set.
1318                  */
1319                 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd,
1320                                         (unsigned char *)hdr, hdr->cmdsn);
1321                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1322                         return -1;
1323
1324                 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1325                         target_put_sess_cmd(&cmd->se_cmd);
1326
1327                         return 0;
1328                 } else if (cmd->unsolicited_data)
1329                         iscsit_set_unsolicited_dataout(cmd);
1330
1331         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1332                 /*
1333                  * Immediate Data failed DataCRC and ERL>=1,
1334                  * silently drop this PDU and let the initiator
1335                  * plug the CmdSN gap.
1336                  *
1337                  * FIXME: Send Unsolicited NOPIN with reserved
1338                  * TTT here to help the initiator figure out
1339                  * the missing CmdSN, although they should be
1340                  * intelligent enough to determine the missing
1341                  * CmdSN and issue a retry to plug the sequence.
1342                  */
1343                 cmd->i_state = ISTATE_REMOVE;
1344                 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
1345         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1346                 return -1;
1347
1348         return 0;
1349 }
1350
1351 static int
1352 iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1353                            unsigned char *buf)
1354 {
1355         struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1356         int rc, immed_data;
1357         bool dump_payload = false;
1358
1359         rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1360         if (rc < 0)
1361                 return 0;
1362         /*
1363          * Allocation iovecs needed for struct socket operations for
1364          * traditional iSCSI block I/O.
1365          */
1366         if (iscsit_allocate_iovecs(cmd) < 0) {
1367                 return iscsit_reject_cmd(cmd,
1368                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
1369         }
1370         immed_data = cmd->immediate_data;
1371
1372         rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1373         if (rc < 0)
1374                 return rc;
1375         else if (rc > 0)
1376                 dump_payload = true;
1377
1378         if (!immed_data)
1379                 return 0;
1380
1381         return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1382 }
1383
1384 static u32 iscsit_do_crypto_hash_sg(
1385         struct ahash_request *hash,
1386         struct iscsi_cmd *cmd,
1387         u32 data_offset,
1388         u32 data_length,
1389         u32 padding,
1390         u8 *pad_bytes)
1391 {
1392         u32 data_crc;
1393         struct scatterlist *sg;
1394         unsigned int page_off;
1395
1396         crypto_ahash_init(hash);
1397
1398         sg = cmd->first_data_sg;
1399         page_off = cmd->first_data_sg_off;
1400
1401         if (data_length && page_off) {
1402                 struct scatterlist first_sg;
1403                 u32 len = min_t(u32, data_length, sg->length - page_off);
1404
1405                 sg_init_table(&first_sg, 1);
1406                 sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off);
1407
1408                 ahash_request_set_crypt(hash, &first_sg, NULL, len);
1409                 crypto_ahash_update(hash);
1410
1411                 data_length -= len;
1412                 sg = sg_next(sg);
1413         }
1414
1415         while (data_length) {
1416                 u32 cur_len = min_t(u32, data_length, sg->length);
1417
1418                 ahash_request_set_crypt(hash, sg, NULL, cur_len);
1419                 crypto_ahash_update(hash);
1420
1421                 data_length -= cur_len;
1422                 /* iscsit_map_iovec has already checked for invalid sg pointers */
1423                 sg = sg_next(sg);
1424         }
1425
1426         if (padding) {
1427                 struct scatterlist pad_sg;
1428
1429                 sg_init_one(&pad_sg, pad_bytes, padding);
1430                 ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
1431                                         padding);
1432                 crypto_ahash_finup(hash);
1433         } else {
1434                 ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
1435                 crypto_ahash_final(hash);
1436         }
1437
1438         return data_crc;
1439 }
1440
1441 static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,
1442         const void *buf, u32 payload_length, u32 padding,
1443         const void *pad_bytes, void *data_crc)
1444 {
1445         struct scatterlist sg[2];
1446
1447         sg_init_table(sg, ARRAY_SIZE(sg));
1448         sg_set_buf(sg, buf, payload_length);
1449         if (padding)
1450                 sg_set_buf(sg + 1, pad_bytes, padding);
1451
1452         ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
1453
1454         crypto_ahash_digest(hash);
1455 }
1456
1457 int
1458 __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1459                            struct iscsi_cmd *cmd, u32 payload_length,
1460                            bool *success)
1461 {
1462         struct iscsi_data *hdr = buf;
1463         struct se_cmd *se_cmd;
1464         int rc;
1465
1466         /* iSCSI write */
1467         atomic_long_add(payload_length, &conn->sess->rx_data_octets);
1468
1469         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1470                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1471                 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
1472                 payload_length, conn->cid);
1473
1474         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1475                 pr_err("Command ITT: 0x%08x received DataOUT after"
1476                         " last DataOUT received, dumping payload\n",
1477                         cmd->init_task_tag);
1478                 return iscsit_dump_data_payload(conn, payload_length, 1);
1479         }
1480
1481         if (cmd->data_direction != DMA_TO_DEVICE) {
1482                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1483                         " NON-WRITE command.\n", cmd->init_task_tag);
1484                 return iscsit_dump_data_payload(conn, payload_length, 1);
1485         }
1486         se_cmd = &cmd->se_cmd;
1487         iscsit_mod_dataout_timer(cmd);
1488
1489         if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
1490                 pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n",
1491                        be32_to_cpu(hdr->offset), payload_length,
1492                        cmd->se_cmd.data_length);
1493                 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf);
1494         }
1495
1496         if (cmd->unsolicited_data) {
1497                 int dump_unsolicited_data = 0;
1498
1499                 if (conn->sess->sess_ops->InitialR2T) {
1500                         pr_err("Received unexpected unsolicited data"
1501                                 " while InitialR2T=Yes, protocol error.\n");
1502                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1503                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1504                         return -1;
1505                 }
1506                 /*
1507                  * Special case for dealing with Unsolicited DataOUT
1508                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1509                  * failures;
1510                  */
1511
1512                 /* Something's amiss if we're not in WRITE_PENDING state... */
1513                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1514                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
1515                         dump_unsolicited_data = 1;
1516
1517                 if (dump_unsolicited_data) {
1518                         /*
1519                          * Check if a delayed TASK_ABORTED status needs to
1520                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1521                          * received with the unsolicited data out.
1522                          */
1523                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1524                                 iscsit_stop_dataout_timer(cmd);
1525
1526                         return iscsit_dump_data_payload(conn, payload_length, 1);
1527                 }
1528         } else {
1529                 /*
1530                  * For the normal solicited data path:
1531                  *
1532                  * Check for a delayed TASK_ABORTED status and dump any
1533                  * incoming data out payload if one exists.  Also, when the
1534                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1535                  * data out sequence, we decrement outstanding_r2ts.  Once
1536                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1537                  * TASK_ABORTED status.
1538                  */
1539                 if (se_cmd->transport_state & CMD_T_ABORTED) {
1540                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL &&
1541                             --cmd->outstanding_r2ts < 1)
1542                                 iscsit_stop_dataout_timer(cmd);
1543
1544                         return iscsit_dump_data_payload(conn, payload_length, 1);
1545                 }
1546         }
1547         /*
1548          * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1549          * within-command recovery checks before receiving the payload.
1550          */
1551         rc = iscsit_check_pre_dataout(cmd, buf);
1552         if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
1553                 return 0;
1554         else if (rc == DATAOUT_CANNOT_RECOVER)
1555                 return -1;
1556         *success = true;
1557         return 0;
1558 }
1559 EXPORT_SYMBOL(__iscsit_check_dataout_hdr);
1560
1561 int
1562 iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
1563                          struct iscsi_cmd **out_cmd)
1564 {
1565         struct iscsi_data *hdr = buf;
1566         struct iscsi_cmd *cmd;
1567         u32 payload_length = ntoh24(hdr->dlength);
1568         int rc;
1569         bool success = false;
1570
1571         if (!payload_length) {
1572                 pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n");
1573                 return 0;
1574         }
1575
1576         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1577                 pr_err_ratelimited("DataSegmentLength: %u is greater than"
1578                         " MaxXmitDataSegmentLength: %u\n", payload_length,
1579                         conn->conn_ops->MaxXmitDataSegmentLength);
1580                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
1581         }
1582
1583         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length);
1584         if (!cmd)
1585                 return 0;
1586
1587         rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success);
1588
1589         if (success)
1590                 *out_cmd = cmd;
1591
1592         return rc;
1593 }
1594 EXPORT_SYMBOL(iscsit_check_dataout_hdr);
1595
1596 static int
1597 iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1598                    struct iscsi_data *hdr)
1599 {
1600         struct kvec *iov;
1601         u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
1602         u32 payload_length;
1603         int iov_ret, data_crc_failed = 0;
1604
1605         payload_length = min_t(u32, cmd->se_cmd.data_length,
1606                                ntoh24(hdr->dlength));
1607         rx_size += payload_length;
1608         iov = &cmd->iov_data[0];
1609
1610         iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2,
1611                                    be32_to_cpu(hdr->offset), payload_length);
1612         if (iov_ret < 0)
1613                 return -1;
1614
1615         iov_count += iov_ret;
1616
1617         padding = ((-payload_length) & 3);
1618         if (padding != 0) {
1619                 iov[iov_count].iov_base = cmd->pad_bytes;
1620                 iov[iov_count++].iov_len = padding;
1621                 rx_size += padding;
1622                 pr_debug("Receiving %u padding bytes.\n", padding);
1623         }
1624
1625         if (conn->conn_ops->DataDigest) {
1626                 iov[iov_count].iov_base = &checksum;
1627                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1628                 rx_size += ISCSI_CRC_LEN;
1629         }
1630
1631         WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
1632         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1633
1634         iscsit_unmap_iovec(cmd);
1635
1636         if (rx_got != rx_size)
1637                 return -1;
1638
1639         if (conn->conn_ops->DataDigest) {
1640                 u32 data_crc;
1641
1642                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
1643                                                     be32_to_cpu(hdr->offset),
1644                                                     payload_length, padding,
1645                                                     cmd->pad_bytes);
1646
1647                 if (checksum != data_crc) {
1648                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1649                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1650                                 " does not match computed 0x%08x\n",
1651                                 hdr->itt, hdr->offset, payload_length,
1652                                 hdr->datasn, checksum, data_crc);
1653                         data_crc_failed = 1;
1654                 } else {
1655                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1656                                 " %u bytes of Data Out\n", checksum,
1657                                 payload_length);
1658                 }
1659         }
1660
1661         return data_crc_failed;
1662 }
1663
1664 int
1665 iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
1666                              bool data_crc_failed)
1667 {
1668         struct iscsi_conn *conn = cmd->conn;
1669         int rc, ooo_cmdsn;
1670         /*
1671          * Increment post receive data and CRC values or perform
1672          * within-command recovery.
1673          */
1674         rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
1675         if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
1676                 return 0;
1677         else if (rc == DATAOUT_SEND_R2T) {
1678                 iscsit_set_dataout_sequence_values(cmd);
1679                 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
1680         } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
1681                 /*
1682                  * Handle extra special case for out of order
1683                  * Unsolicited Data Out.
1684                  */
1685                 spin_lock_bh(&cmd->istate_lock);
1686                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1687                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1688                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1689                 spin_unlock_bh(&cmd->istate_lock);
1690
1691                 iscsit_stop_dataout_timer(cmd);
1692                 if (ooo_cmdsn)
1693                         return 0;
1694                 target_execute_cmd(&cmd->se_cmd);
1695                 return 0;
1696         } else /* DATAOUT_CANNOT_RECOVER */
1697                 return -1;
1698
1699         return 0;
1700 }
1701 EXPORT_SYMBOL(iscsit_check_dataout_payload);
1702
1703 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1704 {
1705         struct iscsi_cmd *cmd = NULL;
1706         struct iscsi_data *hdr = (struct iscsi_data *)buf;
1707         int rc;
1708         bool data_crc_failed = false;
1709
1710         rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1711         if (rc < 0)
1712                 return 0;
1713         else if (!cmd)
1714                 return 0;
1715
1716         rc = iscsit_get_dataout(conn, cmd, hdr);
1717         if (rc < 0)
1718                 return rc;
1719         else if (rc > 0)
1720                 data_crc_failed = true;
1721
1722         return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
1723 }
1724
1725 int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1726                          struct iscsi_nopout *hdr)
1727 {
1728         u32 payload_length = ntoh24(hdr->dlength);
1729
1730         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
1731                 pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n");
1732                 if (!cmd)
1733                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1734                                                  (unsigned char *)hdr);
1735
1736                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1737                                          (unsigned char *)hdr);
1738         }
1739
1740         if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1741                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1742                         " not set, protocol error.\n");
1743                 if (!cmd)
1744                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1745                                                  (unsigned char *)hdr);
1746
1747                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1748                                          (unsigned char *)hdr);
1749         }
1750
1751         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
1752                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1753                         " greater than MaxXmitDataSegmentLength: %u, protocol"
1754                         " error.\n", payload_length,
1755                         conn->conn_ops->MaxXmitDataSegmentLength);
1756                 if (!cmd)
1757                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1758                                                  (unsigned char *)hdr);
1759
1760                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1761                                          (unsigned char *)hdr);
1762         }
1763
1764         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
1765                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1766                 hdr->itt == RESERVED_ITT ? "Response" : "Request",
1767                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1768                 payload_length);
1769         /*
1770          * This is not a response to a Unsolicited NopIN, which means
1771          * it can either be a NOPOUT ping request (with a valid ITT),
1772          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1773          * Either way, make sure we allocate an struct iscsi_cmd, as both
1774          * can contain ping data.
1775          */
1776         if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1777                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1778                 cmd->i_state            = ISTATE_SEND_NOPIN;
1779                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1780                                                 1 : 0);
1781                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1782                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1783                 cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
1784                 cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
1785                 cmd->data_direction     = DMA_NONE;
1786         }
1787
1788         return 0;
1789 }
1790 EXPORT_SYMBOL(iscsit_setup_nop_out);
1791
1792 int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1793                            struct iscsi_nopout *hdr)
1794 {
1795         struct iscsi_cmd *cmd_p = NULL;
1796         int cmdsn_ret = 0;
1797         /*
1798          * Initiator is expecting a NopIN ping reply..
1799          */
1800         if (hdr->itt != RESERVED_ITT) {
1801                 if (!cmd)
1802                         return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1803                                                 (unsigned char *)hdr);
1804
1805                 spin_lock_bh(&conn->cmd_lock);
1806                 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
1807                 spin_unlock_bh(&conn->cmd_lock);
1808
1809                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
1810
1811                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1812                         iscsit_add_cmd_to_response_queue(cmd, conn,
1813                                                          cmd->i_state);
1814                         return 0;
1815                 }
1816
1817                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
1818                                 (unsigned char *)hdr, hdr->cmdsn);
1819                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1820                         return 0;
1821                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1822                         return -1;
1823
1824                 return 0;
1825         }
1826         /*
1827          * This was a response to a unsolicited NOPIN ping.
1828          */
1829         if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1830                 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
1831                 if (!cmd_p)
1832                         return -EINVAL;
1833
1834                 iscsit_stop_nopin_response_timer(conn);
1835
1836                 cmd_p->i_state = ISTATE_REMOVE;
1837                 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
1838
1839                 iscsit_start_nopin_timer(conn);
1840                 return 0;
1841         }
1842         /*
1843          * Otherwise, initiator is not expecting a NOPIN is response.
1844          * Just ignore for now.
1845          */
1846
1847         if (cmd)
1848                 iscsit_free_cmd(cmd, false);
1849
1850         return 0;
1851 }
1852 EXPORT_SYMBOL(iscsit_process_nop_out);
1853
1854 static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1855                                  unsigned char *buf)
1856 {
1857         unsigned char *ping_data = NULL;
1858         struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1859         struct kvec *iov = NULL;
1860         u32 payload_length = ntoh24(hdr->dlength);
1861         int ret;
1862
1863         ret = iscsit_setup_nop_out(conn, cmd, hdr);
1864         if (ret < 0)
1865                 return 0;
1866         /*
1867          * Handle NOP-OUT payload for traditional iSCSI sockets
1868          */
1869         if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1870                 u32 checksum, data_crc, padding = 0;
1871                 int niov = 0, rx_got, rx_size = payload_length;
1872
1873                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1874                 if (!ping_data) {
1875                         ret = -1;
1876                         goto out;
1877                 }
1878
1879                 iov = &cmd->iov_misc[0];
1880                 iov[niov].iov_base      = ping_data;
1881                 iov[niov++].iov_len     = payload_length;
1882
1883                 padding = ((-payload_length) & 3);
1884                 if (padding != 0) {
1885                         pr_debug("Receiving %u additional bytes"
1886                                 " for padding.\n", padding);
1887                         iov[niov].iov_base      = &cmd->pad_bytes;
1888                         iov[niov++].iov_len     = padding;
1889                         rx_size += padding;
1890                 }
1891                 if (conn->conn_ops->DataDigest) {
1892                         iov[niov].iov_base      = &checksum;
1893                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1894                         rx_size += ISCSI_CRC_LEN;
1895                 }
1896
1897                 WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc));
1898                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1899                 if (rx_got != rx_size) {
1900                         ret = -1;
1901                         goto out;
1902                 }
1903
1904                 if (conn->conn_ops->DataDigest) {
1905                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,
1906                                                   payload_length, padding,
1907                                                   cmd->pad_bytes, &data_crc);
1908
1909                         if (checksum != data_crc) {
1910                                 pr_err("Ping data CRC32C DataDigest"
1911                                 " 0x%08x does not match computed 0x%08x\n",
1912                                         checksum, data_crc);
1913                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1914                                         pr_err("Unable to recover from"
1915                                         " NOPOUT Ping DataCRC failure while in"
1916                                                 " ERL=0.\n");
1917                                         ret = -1;
1918                                         goto out;
1919                                 } else {
1920                                         /*
1921                                          * Silently drop this PDU and let the
1922                                          * initiator plug the CmdSN gap.
1923                                          */
1924                                         pr_debug("Dropping NOPOUT"
1925                                         " Command CmdSN: 0x%08x due to"
1926                                         " DataCRC error.\n", hdr->cmdsn);
1927                                         ret = 0;
1928                                         goto out;
1929                                 }
1930                         } else {
1931                                 pr_debug("Got CRC32C DataDigest"
1932                                 " 0x%08x for %u bytes of ping data.\n",
1933                                         checksum, payload_length);
1934                         }
1935                 }
1936
1937                 ping_data[payload_length] = '\0';
1938                 /*
1939                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1940                  */
1941                 cmd->buf_ptr = ping_data;
1942                 cmd->buf_ptr_size = payload_length;
1943
1944                 pr_debug("Got %u bytes of NOPOUT ping"
1945                         " data.\n", payload_length);
1946                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1947         }
1948
1949         return iscsit_process_nop_out(conn, cmd, hdr);
1950 out:
1951         if (cmd)
1952                 iscsit_free_cmd(cmd, false);
1953
1954         kfree(ping_data);
1955         return ret;
1956 }
1957
1958 static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf)
1959 {
1960         switch (iscsi_tmf) {
1961         case ISCSI_TM_FUNC_ABORT_TASK:
1962                 return TMR_ABORT_TASK;
1963         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1964                 return TMR_ABORT_TASK_SET;
1965         case ISCSI_TM_FUNC_CLEAR_ACA:
1966                 return TMR_CLEAR_ACA;
1967         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1968                 return TMR_CLEAR_TASK_SET;
1969         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1970                 return TMR_LUN_RESET;
1971         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1972                 return TMR_TARGET_WARM_RESET;
1973         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1974                 return TMR_TARGET_COLD_RESET;
1975         default:
1976                 return TMR_UNKNOWN;
1977         }
1978 }
1979
1980 int
1981 iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1982                            unsigned char *buf)
1983 {
1984         struct se_tmr_req *se_tmr;
1985         struct iscsi_tmr_req *tmr_req;
1986         struct iscsi_tm *hdr;
1987         int out_of_order_cmdsn = 0, ret;
1988         u8 function, tcm_function = TMR_UNKNOWN;
1989
1990         hdr                     = (struct iscsi_tm *) buf;
1991         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1992         function = hdr->flags;
1993
1994         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1995                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1996                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1997                 hdr->rtt, hdr->refcmdsn, conn->cid);
1998
1999         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
2000             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
2001              hdr->rtt != RESERVED_ITT)) {
2002                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
2003                 hdr->rtt = RESERVED_ITT;
2004         }
2005
2006         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
2007                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2008                 pr_err("Task Management Request TASK_REASSIGN not"
2009                         " issued as immediate command, bad iSCSI Initiator"
2010                                 "implementation\n");
2011                 return iscsit_add_reject_cmd(cmd,
2012                                              ISCSI_REASON_PROTOCOL_ERROR, buf);
2013         }
2014         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
2015             be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
2016                 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
2017
2018         cmd->data_direction = DMA_NONE;
2019         cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL);
2020         if (!cmd->tmr_req) {
2021                 return iscsit_add_reject_cmd(cmd,
2022                                              ISCSI_REASON_BOOKMARK_NO_RESOURCES,
2023                                              buf);
2024         }
2025
2026         __target_init_cmd(&cmd->se_cmd, &iscsi_ops,
2027                           conn->sess->se_sess, 0, DMA_NONE,
2028                           TCM_SIMPLE_TAG, cmd->sense_buffer + 2,
2029                           scsilun_to_int(&hdr->lun));
2030
2031         target_get_sess_cmd(&cmd->se_cmd, true);
2032
2033         /*
2034          * TASK_REASSIGN for ERL=2 / connection stays inside of
2035          * LIO-Target $FABRIC_MOD
2036          */
2037         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2038                 tcm_function = iscsit_convert_tmf(function);
2039                 if (tcm_function == TMR_UNKNOWN) {
2040                         pr_err("Unknown iSCSI TMR Function:"
2041                                " 0x%02x\n", function);
2042                         return iscsit_add_reject_cmd(cmd,
2043                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2044                 }
2045         }
2046         ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function,
2047                                  GFP_KERNEL);
2048         if (ret < 0)
2049                 return iscsit_add_reject_cmd(cmd,
2050                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
2051
2052         cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
2053
2054         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
2055         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
2056         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2057         cmd->init_task_tag      = hdr->itt;
2058         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2059         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2060         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2061         se_tmr                  = cmd->se_cmd.se_tmr_req;
2062         tmr_req                 = cmd->tmr_req;
2063         /*
2064          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
2065          */
2066         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
2067                 ret = transport_lookup_tmr_lun(&cmd->se_cmd);
2068                 if (ret < 0) {
2069                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
2070                         goto attach;
2071                 }
2072         }
2073
2074         switch (function) {
2075         case ISCSI_TM_FUNC_ABORT_TASK:
2076                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
2077                 if (se_tmr->response)
2078                         goto attach;
2079                 break;
2080         case ISCSI_TM_FUNC_ABORT_TASK_SET:
2081         case ISCSI_TM_FUNC_CLEAR_ACA:
2082         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
2083         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
2084                 break;
2085         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
2086                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
2087                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2088                         goto attach;
2089                 }
2090                 break;
2091         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
2092                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
2093                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
2094                         goto attach;
2095                 }
2096                 break;
2097         case ISCSI_TM_FUNC_TASK_REASSIGN:
2098                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
2099                 /*
2100                  * Perform sanity checks on the ExpDataSN only if the
2101                  * TASK_REASSIGN was successful.
2102                  */
2103                 if (se_tmr->response)
2104                         break;
2105
2106                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
2107                         return iscsit_add_reject_cmd(cmd,
2108                                         ISCSI_REASON_BOOKMARK_INVALID, buf);
2109                 break;
2110         default:
2111                 pr_err("Unknown TMR function: 0x%02x, protocol"
2112                         " error.\n", function);
2113                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
2114                 goto attach;
2115         }
2116
2117         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
2118             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
2119                 se_tmr->call_transport = 1;
2120 attach:
2121         spin_lock_bh(&conn->cmd_lock);
2122         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2123         spin_unlock_bh(&conn->cmd_lock);
2124
2125         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2126                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2127                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) {
2128                         out_of_order_cmdsn = 1;
2129                 } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2130                         target_put_sess_cmd(&cmd->se_cmd);
2131                         return 0;
2132                 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2133                         return -1;
2134                 }
2135         }
2136         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2137
2138         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2139                 return 0;
2140         /*
2141          * Found the referenced task, send to transport for processing.
2142          */
2143         if (se_tmr->call_transport)
2144                 return transport_generic_handle_tmr(&cmd->se_cmd);
2145
2146         /*
2147          * Could not find the referenced LUN, task, or Task Management
2148          * command not authorized or supported.  Change state and
2149          * let the tx_thread send the response.
2150          *
2151          * For connection recovery, this is also the default action for
2152          * TMR TASK_REASSIGN.
2153          */
2154         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2155         target_put_sess_cmd(&cmd->se_cmd);
2156         return 0;
2157 }
2158 EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
2159
2160 /* #warning FIXME: Support Text Command parameters besides SendTargets */
2161 int
2162 iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2163                       struct iscsi_text *hdr)
2164 {
2165         u32 payload_length = ntoh24(hdr->dlength);
2166
2167         if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
2168                 pr_err("Unable to accept text parameter length: %u"
2169                         "greater than MaxXmitDataSegmentLength %u.\n",
2170                        payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
2171                 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2172                                          (unsigned char *)hdr);
2173         }
2174
2175         if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) ||
2176              (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) {
2177                 pr_err("Multi sequence text commands currently not supported\n");
2178                 return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED,
2179                                         (unsigned char *)hdr);
2180         }
2181
2182         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
2183                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
2184                 hdr->exp_statsn, payload_length);
2185
2186         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
2187         cmd->i_state            = ISTATE_SEND_TEXTRSP;
2188         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2189         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2190         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2191         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2192         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2193         cmd->data_direction     = DMA_NONE;
2194         kfree(cmd->text_in_ptr);
2195         cmd->text_in_ptr        = NULL;
2196
2197         return 0;
2198 }
2199 EXPORT_SYMBOL(iscsit_setup_text_cmd);
2200
2201 int
2202 iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2203                         struct iscsi_text *hdr)
2204 {
2205         unsigned char *text_in = cmd->text_in_ptr, *text_ptr;
2206         int cmdsn_ret;
2207
2208         if (!text_in) {
2209                 cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt);
2210                 if (cmd->targ_xfer_tag == 0xFFFFFFFF) {
2211                         pr_err("Unable to locate text_in buffer for sendtargets"
2212                                " discovery\n");
2213                         goto reject;
2214                 }
2215                 goto empty_sendtargets;
2216         }
2217         if (strncmp("SendTargets=", text_in, 12) != 0) {
2218                 pr_err("Received Text Data that is not"
2219                         " SendTargets, cannot continue.\n");
2220                 goto reject;
2221         }
2222         /* '=' confirmed in strncmp */
2223         text_ptr = strchr(text_in, '=');
2224         BUG_ON(!text_ptr);
2225         if (!strncmp("=All", text_ptr, 5)) {
2226                 cmd->cmd_flags |= ICF_SENDTARGETS_ALL;
2227         } else if (!strncmp("=iqn.", text_ptr, 5) ||
2228                    !strncmp("=eui.", text_ptr, 5)) {
2229                 cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE;
2230         } else {
2231                 pr_err("Unable to locate valid SendTargets%s value\n",
2232                        text_ptr);
2233                 goto reject;
2234         }
2235
2236         spin_lock_bh(&conn->cmd_lock);
2237         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2238         spin_unlock_bh(&conn->cmd_lock);
2239
2240 empty_sendtargets:
2241         iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2242
2243         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2244                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
2245                                 (unsigned char *)hdr, hdr->cmdsn);
2246                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2247                         return -1;
2248
2249                 return 0;
2250         }
2251
2252         return iscsit_execute_cmd(cmd, 0);
2253
2254 reject:
2255         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
2256                                  (unsigned char *)hdr);
2257 }
2258 EXPORT_SYMBOL(iscsit_process_text_cmd);
2259
2260 static int
2261 iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2262                        unsigned char *buf)
2263 {
2264         struct iscsi_text *hdr = (struct iscsi_text *)buf;
2265         char *text_in = NULL;
2266         u32 payload_length = ntoh24(hdr->dlength);
2267         int rx_size, rc;
2268
2269         rc = iscsit_setup_text_cmd(conn, cmd, hdr);
2270         if (rc < 0)
2271                 return 0;
2272
2273         rx_size = payload_length;
2274         if (payload_length) {
2275                 u32 checksum = 0, data_crc = 0;
2276                 u32 padding = 0;
2277                 int niov = 0, rx_got;
2278                 struct kvec iov[2];
2279
2280                 rx_size = ALIGN(payload_length, 4);
2281                 text_in = kzalloc(rx_size, GFP_KERNEL);
2282                 if (!text_in)
2283                         goto reject;
2284
2285                 cmd->text_in_ptr = text_in;
2286
2287                 memset(iov, 0, sizeof(iov));
2288                 iov[niov].iov_base      = text_in;
2289                 iov[niov++].iov_len     = rx_size;
2290
2291                 padding = rx_size - payload_length;
2292                 if (padding)
2293                         pr_debug("Receiving %u additional bytes"
2294                                         " for padding.\n", padding);
2295                 if (conn->conn_ops->DataDigest) {
2296                         iov[niov].iov_base      = &checksum;
2297                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
2298                         rx_size += ISCSI_CRC_LEN;
2299                 }
2300
2301                 WARN_ON_ONCE(niov > ARRAY_SIZE(iov));
2302                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
2303                 if (rx_got != rx_size)
2304                         goto reject;
2305
2306                 if (conn->conn_ops->DataDigest) {
2307                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
2308                                                   text_in, rx_size, 0, NULL,
2309                                                   &data_crc);
2310
2311                         if (checksum != data_crc) {
2312                                 pr_err("Text data CRC32C DataDigest"
2313                                         " 0x%08x does not match computed"
2314                                         " 0x%08x\n", checksum, data_crc);
2315                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2316                                         pr_err("Unable to recover from"
2317                                         " Text Data digest failure while in"
2318                                                 " ERL=0.\n");
2319                                         goto reject;
2320                                 } else {
2321                                         /*
2322                                          * Silently drop this PDU and let the
2323                                          * initiator plug the CmdSN gap.
2324                                          */
2325                                         pr_debug("Dropping Text"
2326                                         " Command CmdSN: 0x%08x due to"
2327                                         " DataCRC error.\n", hdr->cmdsn);
2328                                         kfree(text_in);
2329                                         return 0;
2330                                 }
2331                         } else {
2332                                 pr_debug("Got CRC32C DataDigest"
2333                                         " 0x%08x for %u bytes of text data.\n",
2334                                                 checksum, payload_length);
2335                         }
2336                 }
2337                 text_in[payload_length - 1] = '\0';
2338                 pr_debug("Successfully read %d bytes of text"
2339                                 " data.\n", payload_length);
2340         }
2341
2342         return iscsit_process_text_cmd(conn, cmd, hdr);
2343
2344 reject:
2345         kfree(cmd->text_in_ptr);
2346         cmd->text_in_ptr = NULL;
2347         return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
2348 }
2349
2350 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2351 {
2352         struct iscsi_conn *conn_p;
2353         struct iscsi_session *sess = conn->sess;
2354
2355         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2356                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2357
2358         atomic_set(&sess->session_logout, 1);
2359         atomic_set(&conn->conn_logout_remove, 1);
2360         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2361
2362         iscsit_inc_conn_usage_count(conn);
2363         iscsit_inc_session_usage_count(sess);
2364
2365         spin_lock_bh(&sess->conn_lock);
2366         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2367                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2368                         continue;
2369
2370                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2371                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2372         }
2373         spin_unlock_bh(&sess->conn_lock);
2374
2375         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2376
2377         return 0;
2378 }
2379
2380 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2381 {
2382         struct iscsi_conn *l_conn;
2383         struct iscsi_session *sess = conn->sess;
2384
2385         pr_debug("Received logout request CLOSECONNECTION for CID:"
2386                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2387
2388         /*
2389          * A Logout Request with a CLOSECONNECTION reason code for a CID
2390          * can arrive on a connection with a differing CID.
2391          */
2392         if (conn->cid == cmd->logout_cid) {
2393                 spin_lock_bh(&conn->state_lock);
2394                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2395                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2396
2397                 atomic_set(&conn->conn_logout_remove, 1);
2398                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2399                 iscsit_inc_conn_usage_count(conn);
2400
2401                 spin_unlock_bh(&conn->state_lock);
2402         } else {
2403                 /*
2404                  * Handle all different cid CLOSECONNECTION requests in
2405                  * iscsit_logout_post_handler_diffcid() as to give enough
2406                  * time for any non immediate command's CmdSN to be
2407                  * acknowledged on the connection in question.
2408                  *
2409                  * Here we simply make sure the CID is still around.
2410                  */
2411                 l_conn = iscsit_get_conn_from_cid(sess,
2412                                 cmd->logout_cid);
2413                 if (!l_conn) {
2414                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2415                         iscsit_add_cmd_to_response_queue(cmd, conn,
2416                                         cmd->i_state);
2417                         return 0;
2418                 }
2419
2420                 iscsit_dec_conn_usage_count(l_conn);
2421         }
2422
2423         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2424
2425         return 0;
2426 }
2427
2428 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2429 {
2430         struct iscsi_session *sess = conn->sess;
2431
2432         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2433                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2434
2435         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2436                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2437                         " while ERL!=2.\n");
2438                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2439                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2440                 return 0;
2441         }
2442
2443         if (conn->cid == cmd->logout_cid) {
2444                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2445                         " with CID: %hu on CID: %hu, implementation error.\n",
2446                                 cmd->logout_cid, conn->cid);
2447                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2448                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2449                 return 0;
2450         }
2451
2452         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2453
2454         return 0;
2455 }
2456
2457 int
2458 iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2459                         unsigned char *buf)
2460 {
2461         int cmdsn_ret, logout_remove = 0;
2462         u8 reason_code = 0;
2463         struct iscsi_logout *hdr;
2464         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2465
2466         hdr                     = (struct iscsi_logout *) buf;
2467         reason_code             = (hdr->flags & 0x7f);
2468
2469         if (tiqn) {
2470                 spin_lock(&tiqn->logout_stats.lock);
2471                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2472                         tiqn->logout_stats.normal_logouts++;
2473                 else
2474                         tiqn->logout_stats.abnormal_logouts++;
2475                 spin_unlock(&tiqn->logout_stats.lock);
2476         }
2477
2478         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2479                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2480                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2481                 hdr->cid, conn->cid);
2482
2483         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2484                 pr_err("Received logout request on connection that"
2485                         " is not in logged in state, ignoring request.\n");
2486                 iscsit_free_cmd(cmd, false);
2487                 return 0;
2488         }
2489
2490         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2491         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2492         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2493         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2494         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2495         cmd->cmd_sn             = be32_to_cpu(hdr->cmdsn);
2496         cmd->exp_stat_sn        = be32_to_cpu(hdr->exp_statsn);
2497         cmd->logout_cid         = be16_to_cpu(hdr->cid);
2498         cmd->logout_reason      = reason_code;
2499         cmd->data_direction     = DMA_NONE;
2500
2501         /*
2502          * We need to sleep in these cases (by returning 1) until the Logout
2503          * Response gets sent in the tx thread.
2504          */
2505         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2506            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2507             be16_to_cpu(hdr->cid) == conn->cid))
2508                 logout_remove = 1;
2509
2510         spin_lock_bh(&conn->cmd_lock);
2511         list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
2512         spin_unlock_bh(&conn->cmd_lock);
2513
2514         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2515                 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
2516
2517         /*
2518          * Immediate commands are executed, well, immediately.
2519          * Non-Immediate Logout Commands are executed in CmdSN order.
2520          */
2521         if (cmd->immediate_cmd) {
2522                 int ret = iscsit_execute_cmd(cmd, 0);
2523
2524                 if (ret < 0)
2525                         return ret;
2526         } else {
2527                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
2528                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
2529                         logout_remove = 0;
2530                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2531                         return -1;
2532         }
2533
2534         return logout_remove;
2535 }
2536 EXPORT_SYMBOL(iscsit_handle_logout_cmd);
2537
2538 int iscsit_handle_snack(
2539         struct iscsi_conn *conn,
2540         unsigned char *buf)
2541 {
2542         struct iscsi_snack *hdr;
2543
2544         hdr                     = (struct iscsi_snack *) buf;
2545         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2546
2547         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2548                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2549                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2550                         hdr->begrun, hdr->runlength, conn->cid);
2551
2552         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2553                 pr_err("Initiator sent SNACK request while in"
2554                         " ErrorRecoveryLevel=0.\n");
2555                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2556                                          buf);
2557         }
2558         /*
2559          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2560          * call from inside iscsi_send_recovery_datain_or_r2t().
2561          */
2562         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2563         case 0:
2564                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2565                         hdr->itt,
2566                         be32_to_cpu(hdr->ttt),
2567                         be32_to_cpu(hdr->begrun),
2568                         be32_to_cpu(hdr->runlength));
2569         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2570                 return iscsit_handle_status_snack(conn, hdr->itt,
2571                         be32_to_cpu(hdr->ttt),
2572                         be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
2573         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2574                 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
2575                         be32_to_cpu(hdr->begrun),
2576                         be32_to_cpu(hdr->runlength));
2577         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2578                 /* FIXME: Support R-Data SNACK */
2579                 pr_err("R-Data SNACK Not Supported.\n");
2580                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2581                                          buf);
2582         default:
2583                 pr_err("Unknown SNACK type 0x%02x, protocol"
2584                         " error.\n", hdr->flags & 0x0f);
2585                 return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
2586                                          buf);
2587         }
2588
2589         return 0;
2590 }
2591 EXPORT_SYMBOL(iscsit_handle_snack);
2592
2593 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2594 {
2595         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2596             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2597                 wait_for_completion_interruptible_timeout(
2598                                         &conn->rx_half_close_comp,
2599                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2600         }
2601 }
2602
2603 static int iscsit_handle_immediate_data(
2604         struct iscsi_cmd *cmd,
2605         struct iscsi_scsi_req *hdr,
2606         u32 length)
2607 {
2608         int iov_ret, rx_got = 0, rx_size = 0;
2609         u32 checksum, iov_count = 0, padding = 0;
2610         struct iscsi_conn *conn = cmd->conn;
2611         struct kvec *iov;
2612         void *overflow_buf = NULL;
2613
2614         BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length);
2615         rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length);
2616         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data,
2617                                    cmd->orig_iov_data_count - 2,
2618                                    cmd->write_data_done, rx_size);
2619         if (iov_ret < 0)
2620                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2621
2622         iov_count = iov_ret;
2623         iov = &cmd->iov_data[0];
2624         if (rx_size < length) {
2625                 /*
2626                  * Special case: length of immediate data exceeds the data
2627                  * buffer size derived from the CDB.
2628                  */
2629                 overflow_buf = kmalloc(length - rx_size, GFP_KERNEL);
2630                 if (!overflow_buf) {
2631                         iscsit_unmap_iovec(cmd);
2632                         return IMMEDIATE_DATA_CANNOT_RECOVER;
2633                 }
2634                 cmd->overflow_buf = overflow_buf;
2635                 iov[iov_count].iov_base = overflow_buf;
2636                 iov[iov_count].iov_len = length - rx_size;
2637                 iov_count++;
2638                 rx_size = length;
2639         }
2640
2641         padding = ((-length) & 3);
2642         if (padding != 0) {
2643                 iov[iov_count].iov_base = cmd->pad_bytes;
2644                 iov[iov_count++].iov_len = padding;
2645                 rx_size += padding;
2646         }
2647
2648         if (conn->conn_ops->DataDigest) {
2649                 iov[iov_count].iov_base         = &checksum;
2650                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2651                 rx_size += ISCSI_CRC_LEN;
2652         }
2653
2654         WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
2655         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2656
2657         iscsit_unmap_iovec(cmd);
2658
2659         if (rx_got != rx_size) {
2660                 iscsit_rx_thread_wait_for_tcp(conn);
2661                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2662         }
2663
2664         if (conn->conn_ops->DataDigest) {
2665                 u32 data_crc;
2666
2667                 data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
2668                                                     cmd->write_data_done, length, padding,
2669                                                     cmd->pad_bytes);
2670
2671                 if (checksum != data_crc) {
2672                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2673                                 " does not match computed 0x%08x\n", checksum,
2674                                 data_crc);
2675
2676                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2677                                 pr_err("Unable to recover from"
2678                                         " Immediate Data digest failure while"
2679                                         " in ERL=0.\n");
2680                                 iscsit_reject_cmd(cmd,
2681                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2682                                                 (unsigned char *)hdr);
2683                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2684                         } else {
2685                                 iscsit_reject_cmd(cmd,
2686                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2687                                                 (unsigned char *)hdr);
2688                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2689                         }
2690                 } else {
2691                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2692                                 " %u bytes of Immediate Data\n", checksum,
2693                                 length);
2694                 }
2695         }
2696
2697         cmd->write_data_done += length;
2698
2699         if (cmd->write_data_done == cmd->se_cmd.data_length) {
2700                 spin_lock_bh(&cmd->istate_lock);
2701                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2702                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2703                 spin_unlock_bh(&cmd->istate_lock);
2704         }
2705
2706         return IMMEDIATE_DATA_NORMAL_OPERATION;
2707 }
2708
2709 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2710         with active network interface */
2711 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2712 {
2713         struct iscsi_cmd *cmd;
2714         struct iscsi_conn *conn_p;
2715         bool found = false;
2716
2717         lockdep_assert_held(&conn->sess->conn_lock);
2718
2719         /*
2720          * Only send a Asynchronous Message on connections whos network
2721          * interface is still functional.
2722          */
2723         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2724                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2725                         iscsit_inc_conn_usage_count(conn_p);
2726                         found = true;
2727                         break;
2728                 }
2729         }
2730
2731         if (!found)
2732                 return;
2733
2734         cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING);
2735         if (!cmd) {
2736                 iscsit_dec_conn_usage_count(conn_p);
2737                 return;
2738         }
2739
2740         cmd->logout_cid = conn->cid;
2741         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2742         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2743
2744         spin_lock_bh(&conn_p->cmd_lock);
2745         list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
2746         spin_unlock_bh(&conn_p->cmd_lock);
2747
2748         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2749         iscsit_dec_conn_usage_count(conn_p);
2750 }
2751
2752 static int iscsit_send_conn_drop_async_message(
2753         struct iscsi_cmd *cmd,
2754         struct iscsi_conn *conn)
2755 {
2756         struct iscsi_async *hdr;
2757
2758         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2759
2760         hdr                     = (struct iscsi_async *) cmd->pdu;
2761         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2762         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2763         cmd->init_task_tag      = RESERVED_ITT;
2764         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2765         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2766         cmd->stat_sn            = conn->stat_sn++;
2767         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2768         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2769         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2770         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2771         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2772         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2773         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2774
2775         pr_debug("Sending Connection Dropped Async Message StatSN:"
2776                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2777                         cmd->logout_cid, conn->cid);
2778
2779         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
2780 }
2781
2782 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2783 {
2784         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2785             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2786                 wait_for_completion_interruptible_timeout(
2787                                         &conn->tx_half_close_comp,
2788                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2789         }
2790 }
2791
2792 void
2793 iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2794                         struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
2795                         bool set_statsn)
2796 {
2797         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2798         hdr->flags              = datain->flags;
2799         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2800                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2801                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2802                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2803                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2804                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2805                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2806                 }
2807         }
2808         hton24(hdr->dlength, datain->length);
2809         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2810                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2811                                 (struct scsi_lun *)&hdr->lun);
2812         else
2813                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2814
2815         hdr->itt                = cmd->init_task_tag;
2816
2817         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2818                 hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2819         else
2820                 hdr->ttt                = cpu_to_be32(0xFFFFFFFF);
2821         if (set_statsn)
2822                 hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2823         else
2824                 hdr->statsn             = cpu_to_be32(0xFFFFFFFF);
2825
2826         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2827         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2828         hdr->datasn             = cpu_to_be32(datain->data_sn);
2829         hdr->offset             = cpu_to_be32(datain->offset);
2830
2831         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2832                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2833                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2834                 ntohl(hdr->offset), datain->length, conn->cid);
2835 }
2836 EXPORT_SYMBOL(iscsit_build_datain_pdu);
2837
2838 static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2839 {
2840         struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
2841         struct iscsi_datain datain;
2842         struct iscsi_datain_req *dr;
2843         int eodr = 0, ret;
2844         bool set_statsn = false;
2845
2846         memset(&datain, 0, sizeof(struct iscsi_datain));
2847         dr = iscsit_get_datain_values(cmd, &datain);
2848         if (!dr) {
2849                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2850                                 cmd->init_task_tag);
2851                 return -1;
2852         }
2853         /*
2854          * Be paranoid and double check the logic for now.
2855          */
2856         if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
2857                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2858                         " datain.length: %u exceeds cmd->data_length: %u\n",
2859                         cmd->init_task_tag, datain.offset, datain.length,
2860                         cmd->se_cmd.data_length);
2861                 return -1;
2862         }
2863
2864         atomic_long_add(datain.length, &conn->sess->tx_data_octets);
2865         /*
2866          * Special case for successfully execution w/ both DATAIN
2867          * and Sense Data.
2868          */
2869         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2870             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2871                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2872         else {
2873                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2874                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2875                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2876                         cmd->stat_sn = conn->stat_sn++;
2877                         set_statsn = true;
2878                 } else if (dr->dr_complete ==
2879                            DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2880                         set_statsn = true;
2881         }
2882
2883         iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
2884
2885         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0);
2886         if (ret < 0)
2887                 return ret;
2888
2889         if (dr->dr_complete) {
2890                 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2891                                 2 : 1;
2892                 iscsit_free_datain_req(cmd, dr);
2893         }
2894
2895         return eodr;
2896 }
2897
2898 int
2899 iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2900                         struct iscsi_logout_rsp *hdr)
2901 {
2902         struct iscsi_conn *logout_conn = NULL;
2903         struct iscsi_conn_recovery *cr = NULL;
2904         struct iscsi_session *sess = conn->sess;
2905         /*
2906          * The actual shutting down of Sessions and/or Connections
2907          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2908          * is done in scsi_logout_post_handler().
2909          */
2910         switch (cmd->logout_reason) {
2911         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2912                 pr_debug("iSCSI session logout successful, setting"
2913                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2914                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2915                 break;
2916         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2917                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2918                         break;
2919                 /*
2920                  * For CLOSECONNECTION logout requests carrying
2921                  * a matching logout CID -> local CID, the reference
2922                  * for the local CID will have been incremented in
2923                  * iscsi_logout_closeconnection().
2924                  *
2925                  * For CLOSECONNECTION logout requests carrying
2926                  * a different CID than the connection it arrived
2927                  * on, the connection responding to cmd->logout_cid
2928                  * is stopped in iscsit_logout_post_handler_diffcid().
2929                  */
2930
2931                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2932                         " successful.\n", cmd->logout_cid, conn->cid);
2933                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2934                 break;
2935         case ISCSI_LOGOUT_REASON_RECOVERY:
2936                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2937                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2938                         break;
2939                 /*
2940                  * If the connection is still active from our point of view
2941                  * force connection recovery to occur.
2942                  */
2943                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2944                                 cmd->logout_cid);
2945                 if (logout_conn) {
2946                         iscsit_connection_reinstatement_rcfr(logout_conn);
2947                         iscsit_dec_conn_usage_count(logout_conn);
2948                 }
2949
2950                 cr = iscsit_get_inactive_connection_recovery_entry(
2951                                 conn->sess, cmd->logout_cid);
2952                 if (!cr) {
2953                         pr_err("Unable to locate CID: %hu for"
2954                         " REMOVECONNFORRECOVERY Logout Request.\n",
2955                                 cmd->logout_cid);
2956                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2957                         break;
2958                 }
2959
2960                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2961
2962                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2963                         " for recovery for CID: %hu on CID: %hu successful.\n",
2964                                 cmd->logout_cid, conn->cid);
2965                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2966                 break;
2967         default:
2968                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2969                                 cmd->logout_reason);
2970                 return -1;
2971         }
2972
2973         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2974         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2975         hdr->response           = cmd->logout_response;
2976         hdr->itt                = cmd->init_task_tag;
2977         cmd->stat_sn            = conn->stat_sn++;
2978         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2979
2980         iscsit_increment_maxcmdsn(cmd, conn->sess);
2981         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2982         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
2983
2984         pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
2985                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2986                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2987                 cmd->logout_cid, conn->cid);
2988
2989         return 0;
2990 }
2991 EXPORT_SYMBOL(iscsit_build_logout_rsp);
2992
2993 static int
2994 iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2995 {
2996         int rc;
2997
2998         rc = iscsit_build_logout_rsp(cmd, conn,
2999                         (struct iscsi_logout_rsp *)&cmd->pdu[0]);
3000         if (rc < 0)
3001                 return rc;
3002
3003         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3004 }
3005
3006 void
3007 iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3008                        struct iscsi_nopin *hdr, bool nopout_response)
3009 {
3010         hdr->opcode             = ISCSI_OP_NOOP_IN;
3011         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3012         hton24(hdr->dlength, cmd->buf_ptr_size);
3013         if (nopout_response)
3014                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
3015         hdr->itt                = cmd->init_task_tag;
3016         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3017         cmd->stat_sn            = (nopout_response) ? conn->stat_sn++ :
3018                                   conn->stat_sn;
3019         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3020
3021         if (nopout_response)
3022                 iscsit_increment_maxcmdsn(cmd, conn->sess);
3023
3024         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3025         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3026
3027         pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
3028                 " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
3029                 "Solicited" : "Unsolicited", cmd->init_task_tag,
3030                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
3031 }
3032 EXPORT_SYMBOL(iscsit_build_nopin_rsp);
3033
3034 /*
3035  *      Unsolicited NOPIN, either requesting a response or not.
3036  */
3037 static int iscsit_send_unsolicited_nopin(
3038         struct iscsi_cmd *cmd,
3039         struct iscsi_conn *conn,
3040         int want_response)
3041 {
3042         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3043         int ret;
3044
3045         iscsit_build_nopin_rsp(cmd, conn, hdr, false);
3046
3047         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
3048                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
3049
3050         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3051         if (ret < 0)
3052                 return ret;
3053
3054         spin_lock_bh(&cmd->istate_lock);
3055         cmd->i_state = want_response ?
3056                 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
3057         spin_unlock_bh(&cmd->istate_lock);
3058
3059         return 0;
3060 }
3061
3062 static int
3063 iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3064 {
3065         struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
3066
3067         iscsit_build_nopin_rsp(cmd, conn, hdr, true);
3068
3069         /*
3070          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
3071          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
3072          */
3073         pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size);
3074
3075         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3076                                                      cmd->buf_ptr,
3077                                                      cmd->buf_ptr_size);
3078 }
3079
3080 static int iscsit_send_r2t(
3081         struct iscsi_cmd *cmd,
3082         struct iscsi_conn *conn)
3083 {
3084         struct iscsi_r2t *r2t;
3085         struct iscsi_r2t_rsp *hdr;
3086         int ret;
3087
3088         r2t = iscsit_get_r2t_from_list(cmd);
3089         if (!r2t)
3090                 return -1;
3091
3092         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
3093         memset(hdr, 0, ISCSI_HDR_LEN);
3094         hdr->opcode             = ISCSI_OP_R2T;
3095         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3096         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
3097                         (struct scsi_lun *)&hdr->lun);
3098         hdr->itt                = cmd->init_task_tag;
3099         if (conn->conn_transport->iscsit_get_r2t_ttt)
3100                 conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t);
3101         else
3102                 r2t->targ_xfer_tag = session_get_next_ttt(conn->sess);
3103         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
3104         hdr->statsn             = cpu_to_be32(conn->stat_sn);
3105         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3106         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3107         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
3108         hdr->data_offset        = cpu_to_be32(r2t->offset);
3109         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
3110
3111         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
3112                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
3113                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
3114                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
3115                         r2t->offset, r2t->xfer_len, conn->cid);
3116
3117         spin_lock_bh(&cmd->r2t_lock);
3118         r2t->sent_r2t = 1;
3119         spin_unlock_bh(&cmd->r2t_lock);
3120
3121         ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3122         if (ret < 0) {
3123                 return ret;
3124         }
3125
3126         spin_lock_bh(&cmd->dataout_timeout_lock);
3127         iscsit_start_dataout_timer(cmd, conn);
3128         spin_unlock_bh(&cmd->dataout_timeout_lock);
3129
3130         return 0;
3131 }
3132
3133 /*
3134  *      @recovery: If called from iscsi_task_reassign_complete_write() for
3135  *              connection recovery.
3136  */
3137 int iscsit_build_r2ts_for_cmd(
3138         struct iscsi_conn *conn,
3139         struct iscsi_cmd *cmd,
3140         bool recovery)
3141 {
3142         int first_r2t = 1;
3143         u32 offset = 0, xfer_len = 0;
3144
3145         spin_lock_bh(&cmd->r2t_lock);
3146         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
3147                 spin_unlock_bh(&cmd->r2t_lock);
3148                 return 0;
3149         }
3150
3151         if (conn->sess->sess_ops->DataSequenceInOrder &&
3152             !recovery)
3153                 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
3154
3155         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
3156                 if (conn->sess->sess_ops->DataSequenceInOrder) {
3157                         offset = cmd->r2t_offset;
3158
3159                         if (first_r2t && recovery) {
3160                                 int new_data_end = offset +
3161                                         conn->sess->sess_ops->MaxBurstLength -
3162                                         cmd->next_burst_len;
3163
3164                                 if (new_data_end > cmd->se_cmd.data_length)
3165                                         xfer_len = cmd->se_cmd.data_length - offset;
3166                                 else
3167                                         xfer_len =
3168                                                 conn->sess->sess_ops->MaxBurstLength -
3169                                                 cmd->next_burst_len;
3170                         } else {
3171                                 int new_data_end = offset +
3172                                         conn->sess->sess_ops->MaxBurstLength;
3173
3174                                 if (new_data_end > cmd->se_cmd.data_length)
3175                                         xfer_len = cmd->se_cmd.data_length - offset;
3176                                 else
3177                                         xfer_len = conn->sess->sess_ops->MaxBurstLength;
3178                         }
3179
3180                         if ((s32)xfer_len < 0) {
3181                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3182                                 break;
3183                         }
3184
3185                         cmd->r2t_offset += xfer_len;
3186
3187                         if (cmd->r2t_offset == cmd->se_cmd.data_length)
3188                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3189                 } else {
3190                         struct iscsi_seq *seq;
3191
3192                         seq = iscsit_get_seq_holder_for_r2t(cmd);
3193                         if (!seq) {
3194                                 spin_unlock_bh(&cmd->r2t_lock);
3195                                 return -1;
3196                         }
3197
3198                         offset = seq->offset;
3199                         xfer_len = seq->xfer_len;
3200
3201                         if (cmd->seq_send_order == cmd->seq_count)
3202                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3203                 }
3204                 cmd->outstanding_r2ts++;
3205                 first_r2t = 0;
3206
3207                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3208                         spin_unlock_bh(&cmd->r2t_lock);
3209                         return -1;
3210                 }
3211
3212                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3213                         break;
3214         }
3215         spin_unlock_bh(&cmd->r2t_lock);
3216
3217         return 0;
3218 }
3219 EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd);
3220
3221 void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3222                         bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
3223 {
3224         if (inc_stat_sn)
3225                 cmd->stat_sn = conn->stat_sn++;
3226
3227         atomic_long_inc(&conn->sess->rsp_pdus);
3228
3229         memset(hdr, 0, ISCSI_HDR_LEN);
3230         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3231         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3232         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3233                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3234                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3235         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3236                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3237                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3238         }
3239         hdr->response           = cmd->iscsi_response;
3240         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3241         hdr->itt                = cmd->init_task_tag;
3242         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3243
3244         iscsit_increment_maxcmdsn(cmd, conn->sess);
3245         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3246         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3247
3248         pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3249                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3250                 cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
3251                 cmd->se_cmd.scsi_status, conn->cid);
3252 }
3253 EXPORT_SYMBOL(iscsit_build_rsp_pdu);
3254
3255 static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3256 {
3257         struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
3258         bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
3259         void *data_buf = NULL;
3260         u32 padding = 0, data_buf_len = 0;
3261
3262         iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
3263
3264         /*
3265          * Attach SENSE DATA payload to iSCSI Response PDU
3266          */
3267         if (cmd->se_cmd.sense_buffer &&
3268            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3269             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3270                 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
3271                 cmd->se_cmd.scsi_sense_length += sizeof (__be16);
3272
3273                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3274                 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
3275                 data_buf = cmd->sense_buffer;
3276                 data_buf_len = cmd->se_cmd.scsi_sense_length + padding;
3277
3278                 if (padding) {
3279                         memset(cmd->sense_buffer +
3280                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3281                         pr_debug("Adding %u bytes of padding to"
3282                                 " SENSE.\n", padding);
3283                 }
3284
3285                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3286                                 " Response PDU\n",
3287                                 cmd->se_cmd.scsi_sense_length);
3288         }
3289
3290         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf,
3291                                                      data_buf_len);
3292 }
3293
3294 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3295 {
3296         switch (se_tmr->response) {
3297         case TMR_FUNCTION_COMPLETE:
3298                 return ISCSI_TMF_RSP_COMPLETE;
3299         case TMR_TASK_DOES_NOT_EXIST:
3300                 return ISCSI_TMF_RSP_NO_TASK;
3301         case TMR_LUN_DOES_NOT_EXIST:
3302                 return ISCSI_TMF_RSP_NO_LUN;
3303         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3304                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3305         case TMR_FUNCTION_REJECTED:
3306         default:
3307                 return ISCSI_TMF_RSP_REJECTED;
3308         }
3309 }
3310
3311 void
3312 iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3313                           struct iscsi_tm_rsp *hdr)
3314 {
3315         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3316
3317         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3318         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3319         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3320         hdr->itt                = cmd->init_task_tag;
3321         cmd->stat_sn            = conn->stat_sn++;
3322         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3323
3324         iscsit_increment_maxcmdsn(cmd, conn->sess);
3325         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3326         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3327
3328         pr_debug("Built Task Management Response ITT: 0x%08x,"
3329                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3330                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3331 }
3332 EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
3333
3334 static int
3335 iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
3336 {
3337         struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
3338
3339         iscsit_build_task_mgt_rsp(cmd, conn, hdr);
3340
3341         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
3342 }
3343
3344 #define SENDTARGETS_BUF_LIMIT 32768U
3345
3346 static int
3347 iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
3348                                   enum iscsit_transport_type network_transport,
3349                                   int skip_bytes, bool *completed)
3350 {
3351         char *payload = NULL;
3352         struct iscsi_conn *conn = cmd->conn;
3353         struct iscsi_portal_group *tpg;
3354         struct iscsi_tiqn *tiqn;
3355         struct iscsi_tpg_np *tpg_np;
3356         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3357         int target_name_printed;
3358         unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
3359         unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL;
3360         bool active;
3361
3362         buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength,
3363                          SENDTARGETS_BUF_LIMIT);
3364
3365         payload = kzalloc(buffer_len, GFP_KERNEL);
3366         if (!payload)
3367                 return -ENOMEM;
3368
3369         /*
3370          * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE
3371          * explicit case..
3372          */
3373         if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) {
3374                 text_ptr = strchr(text_in, '=');
3375                 if (!text_ptr) {
3376                         pr_err("Unable to locate '=' string in text_in:"
3377                                " %s\n", text_in);
3378                         kfree(payload);
3379                         return -EINVAL;
3380                 }
3381                 /*
3382                  * Skip over '=' character..
3383                  */
3384                 text_ptr += 1;
3385         }
3386
3387         spin_lock(&tiqn_lock);
3388         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3389                 if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) &&
3390                      strcmp(tiqn->tiqn, text_ptr)) {
3391                         continue;
3392                 }
3393
3394                 target_name_printed = 0;
3395
3396                 spin_lock(&tiqn->tiqn_tpg_lock);
3397                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3398
3399                         /* If demo_mode_discovery=0 and generate_node_acls=0
3400                          * (demo mode dislabed) do not return
3401                          * TargetName+TargetAddress unless a NodeACL exists.
3402                          */
3403
3404                         if ((tpg->tpg_attrib.generate_node_acls == 0) &&
3405                             (tpg->tpg_attrib.demo_mode_discovery == 0) &&
3406                             (!target_tpg_has_node_acl(&tpg->tpg_se_tpg,
3407                                 cmd->conn->sess->sess_ops->InitiatorName))) {
3408                                 continue;
3409                         }
3410
3411                         spin_lock(&tpg->tpg_state_lock);
3412                         active = (tpg->tpg_state == TPG_STATE_ACTIVE);
3413                         spin_unlock(&tpg->tpg_state_lock);
3414
3415                         if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets)
3416                                 continue;
3417
3418                         spin_lock(&tpg->tpg_np_lock);
3419                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3420                                                 tpg_np_list) {
3421                                 struct iscsi_np *np = tpg_np->tpg_np;
3422                                 struct sockaddr_storage *sockaddr;
3423
3424                                 if (np->np_network_transport != network_transport)
3425                                         continue;
3426
3427                                 if (!target_name_printed) {
3428                                         len = sprintf(buf, "TargetName=%s",
3429                                                       tiqn->tiqn);
3430                                         len += 1;
3431
3432                                         if ((len + payload_len) > buffer_len) {
3433                                                 spin_unlock(&tpg->tpg_np_lock);
3434                                                 spin_unlock(&tiqn->tiqn_tpg_lock);
3435                                                 end_of_buf = 1;
3436                                                 goto eob;
3437                                         }
3438
3439                                         if (skip_bytes && len <= skip_bytes) {
3440                                                 skip_bytes -= len;
3441                                         } else {
3442                                                 memcpy(payload + payload_len, buf, len);
3443                                                 payload_len += len;
3444                                                 target_name_printed = 1;
3445                                                 if (len > skip_bytes)
3446                                                         skip_bytes = 0;
3447                                         }
3448                                 }
3449
3450                                 if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr))
3451                                         sockaddr = &conn->local_sockaddr;
3452                                 else
3453                                         sockaddr = &np->np_sockaddr;
3454
3455                                 len = sprintf(buf, "TargetAddress="
3456                                               "%pISpc,%hu",
3457                                               sockaddr,
3458                                               tpg->tpgt);
3459                                 len += 1;
3460
3461                                 if ((len + payload_len) > buffer_len) {
3462                                         spin_unlock(&tpg->tpg_np_lock);
3463                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3464                                         end_of_buf = 1;
3465                                         goto eob;
3466                                 }
3467
3468                                 if (skip_bytes && len <= skip_bytes) {
3469                                         skip_bytes -= len;
3470                                 } else {
3471                                         memcpy(payload + payload_len, buf, len);
3472                                         payload_len += len;
3473                                         if (len > skip_bytes)
3474                                                 skip_bytes = 0;
3475                                 }
3476                         }
3477                         spin_unlock(&tpg->tpg_np_lock);
3478                 }
3479                 spin_unlock(&tiqn->tiqn_tpg_lock);
3480 eob:
3481                 if (end_of_buf) {
3482                         *completed = false;
3483                         break;
3484                 }
3485
3486                 if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE)
3487                         break;
3488         }
3489         spin_unlock(&tiqn_lock);
3490
3491         cmd->buf_ptr = payload;
3492
3493         return payload_len;
3494 }
3495
3496 int
3497 iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3498                       struct iscsi_text_rsp *hdr,
3499                       enum iscsit_transport_type network_transport)
3500 {
3501         int text_length, padding;
3502         bool completed = true;
3503
3504         text_length = iscsit_build_sendtargets_response(cmd, network_transport,
3505                                                         cmd->read_data_done,
3506                                                         &completed);
3507         if (text_length < 0)
3508                 return text_length;
3509
3510         if (completed) {
3511                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
3512         } else {
3513                 hdr->flags = ISCSI_FLAG_TEXT_CONTINUE;
3514                 cmd->read_data_done += text_length;
3515                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
3516                         cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
3517         }
3518         hdr->opcode = ISCSI_OP_TEXT_RSP;
3519         padding = ((-text_length) & 3);
3520         hton24(hdr->dlength, text_length);
3521         hdr->itt = cmd->init_task_tag;
3522         hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
3523         cmd->stat_sn = conn->stat_sn++;
3524         hdr->statsn = cpu_to_be32(cmd->stat_sn);
3525
3526         iscsit_increment_maxcmdsn(cmd, conn->sess);
3527         /*
3528          * Reset maxcmdsn_inc in multi-part text payload exchanges to
3529          * correctly increment MaxCmdSN for each response answering a
3530          * non immediate text request with a valid CmdSN.
3531          */
3532         cmd->maxcmdsn_inc = 0;
3533         hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3534         hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3535
3536         pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x,"
3537                 " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag,
3538                 cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid,
3539                 !!(hdr->flags & ISCSI_FLAG_CMD_FINAL),
3540                 !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE));
3541
3542         return text_length + padding;
3543 }
3544 EXPORT_SYMBOL(iscsit_build_text_rsp);
3545
3546 static int iscsit_send_text_rsp(
3547         struct iscsi_cmd *cmd,
3548         struct iscsi_conn *conn)
3549 {
3550         struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu;
3551         int text_length;
3552
3553         text_length = iscsit_build_text_rsp(cmd, conn, hdr,
3554                                 conn->conn_transport->transport_type);
3555         if (text_length < 0)
3556                 return text_length;
3557
3558         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3559                                                      cmd->buf_ptr,
3560                                                      text_length);
3561 }
3562
3563 void
3564 iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
3565                     struct iscsi_reject *hdr)
3566 {
3567         hdr->opcode             = ISCSI_OP_REJECT;
3568         hdr->reason             = cmd->reject_reason;
3569         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3570         hton24(hdr->dlength, ISCSI_HDR_LEN);
3571         hdr->ffffffff           = cpu_to_be32(0xffffffff);
3572         cmd->stat_sn            = conn->stat_sn++;
3573         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3574         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3575         hdr->max_cmdsn          = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
3576
3577 }
3578 EXPORT_SYMBOL(iscsit_build_reject);
3579
3580 static int iscsit_send_reject(
3581         struct iscsi_cmd *cmd,
3582         struct iscsi_conn *conn)
3583 {
3584         struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0];
3585
3586         iscsit_build_reject(cmd, conn, hdr);
3587
3588         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3589                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3590
3591         return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
3592                                                      cmd->buf_ptr,
3593                                                      ISCSI_HDR_LEN);
3594 }
3595
3596 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3597 {
3598         int ord, cpu;
3599         cpumask_var_t conn_allowed_cpumask;
3600
3601         /*
3602          * bitmap_id is assigned from iscsit_global->ts_bitmap from
3603          * within iscsit_start_kthreads()
3604          *
3605          * Here we use bitmap_id to determine which CPU that this
3606          * iSCSI connection's RX/TX threads will be scheduled to
3607          * execute upon.
3608          */
3609         if (!zalloc_cpumask_var(&conn_allowed_cpumask, GFP_KERNEL)) {
3610                 ord = conn->bitmap_id % cpumask_weight(cpu_online_mask);
3611                 for_each_online_cpu(cpu) {
3612                         if (ord-- == 0) {
3613                                 cpumask_set_cpu(cpu, conn->conn_cpumask);
3614                                 return;
3615                         }
3616                 }
3617         } else {
3618                 cpumask_and(conn_allowed_cpumask, iscsit_global->allowed_cpumask,
3619                         cpu_online_mask);
3620
3621                 cpumask_clear(conn->conn_cpumask);
3622                 ord = conn->bitmap_id % cpumask_weight(conn_allowed_cpumask);
3623                 for_each_cpu(cpu, conn_allowed_cpumask) {
3624                         if (ord-- == 0) {
3625                                 cpumask_set_cpu(cpu, conn->conn_cpumask);
3626                                 free_cpumask_var(conn_allowed_cpumask);
3627                                 return;
3628                         }
3629                 }
3630                 free_cpumask_var(conn_allowed_cpumask);
3631         }
3632         /*
3633          * This should never be reached..
3634          */
3635         dump_stack();
3636         cpumask_setall(conn->conn_cpumask);
3637 }
3638
3639 static void iscsit_thread_reschedule(struct iscsi_conn *conn)
3640 {
3641         /*
3642          * If iscsit_global->allowed_cpumask modified, reschedule iSCSI
3643          * connection's RX/TX threads update conn->allowed_cpumask.
3644          */
3645         if (!cpumask_equal(iscsit_global->allowed_cpumask,
3646                            conn->allowed_cpumask)) {
3647                 iscsit_thread_get_cpumask(conn);
3648                 conn->conn_tx_reset_cpumask = 1;
3649                 conn->conn_rx_reset_cpumask = 1;
3650                 cpumask_copy(conn->allowed_cpumask,
3651                              iscsit_global->allowed_cpumask);
3652         }
3653 }
3654
3655 void iscsit_thread_check_cpumask(
3656         struct iscsi_conn *conn,
3657         struct task_struct *p,
3658         int mode)
3659 {
3660         /*
3661          * The TX and RX threads maybe call iscsit_thread_check_cpumask()
3662          * at the same time. The RX thread might be faster and return from
3663          * iscsit_thread_reschedule() with conn_rx_reset_cpumask set to 0.
3664          * Then the TX thread sets it back to 1.
3665          * The next time the RX thread loops, it sees conn_rx_reset_cpumask
3666          * set to 1 and calls set_cpus_allowed_ptr() again and set it to 0.
3667          */
3668         iscsit_thread_reschedule(conn);
3669
3670         /*
3671          * mode == 1 signals iscsi_target_tx_thread() usage.
3672          * mode == 0 signals iscsi_target_rx_thread() usage.
3673          */
3674         if (mode == 1) {
3675                 if (!conn->conn_tx_reset_cpumask)
3676                         return;
3677         } else {
3678                 if (!conn->conn_rx_reset_cpumask)
3679                         return;
3680         }
3681
3682         /*
3683          * Update the CPU mask for this single kthread so that
3684          * both TX and RX kthreads are scheduled to run on the
3685          * same CPU.
3686          */
3687         set_cpus_allowed_ptr(p, conn->conn_cpumask);
3688         if (mode == 1)
3689                 conn->conn_tx_reset_cpumask = 0;
3690         else
3691                 conn->conn_rx_reset_cpumask = 0;
3692 }
3693 EXPORT_SYMBOL(iscsit_thread_check_cpumask);
3694
3695 int
3696 iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3697 {
3698         int ret;
3699
3700         switch (state) {
3701         case ISTATE_SEND_R2T:
3702                 ret = iscsit_send_r2t(cmd, conn);
3703                 if (ret < 0)
3704                         goto err;
3705                 break;
3706         case ISTATE_REMOVE:
3707                 spin_lock_bh(&conn->cmd_lock);
3708                 list_del_init(&cmd->i_conn_node);
3709                 spin_unlock_bh(&conn->cmd_lock);
3710
3711                 iscsit_free_cmd(cmd, false);
3712                 break;
3713         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3714                 iscsit_mod_nopin_response_timer(conn);
3715                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
3716                 if (ret < 0)
3717                         goto err;
3718                 break;
3719         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3720                 ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
3721                 if (ret < 0)
3722                         goto err;
3723                 break;
3724         default:
3725                 pr_err("Unknown Opcode: 0x%02x ITT:"
3726                        " 0x%08x, i_state: %d on CID: %hu\n",
3727                        cmd->iscsi_opcode, cmd->init_task_tag, state,
3728                        conn->cid);
3729                 goto err;
3730         }
3731
3732         return 0;
3733
3734 err:
3735         return -1;
3736 }
3737 EXPORT_SYMBOL(iscsit_immediate_queue);
3738
3739 static int
3740 iscsit_handle_immediate_queue(struct iscsi_conn *conn)
3741 {
3742         struct iscsit_transport *t = conn->conn_transport;
3743         struct iscsi_queue_req *qr;
3744         struct iscsi_cmd *cmd;
3745         u8 state;
3746         int ret;
3747
3748         while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3749                 atomic_set(&conn->check_immediate_queue, 0);
3750                 cmd = qr->cmd;
3751                 state = qr->state;
3752                 kmem_cache_free(lio_qr_cache, qr);
3753
3754                 ret = t->iscsit_immediate_queue(conn, cmd, state);
3755                 if (ret < 0)
3756                         return ret;
3757         }
3758
3759         return 0;
3760 }
3761
3762 int
3763 iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
3764 {
3765         int ret;
3766
3767 check_rsp_state:
3768         switch (state) {
3769         case ISTATE_SEND_DATAIN:
3770                 ret = iscsit_send_datain(cmd, conn);
3771                 if (ret < 0)
3772                         goto err;
3773                 else if (!ret)
3774                         /* more drs */
3775                         goto check_rsp_state;
3776                 else if (ret == 1) {
3777                         /* all done */
3778                         spin_lock_bh(&cmd->istate_lock);
3779                         cmd->i_state = ISTATE_SENT_STATUS;
3780                         spin_unlock_bh(&cmd->istate_lock);
3781
3782                         if (atomic_read(&conn->check_immediate_queue))
3783                                 return 1;
3784
3785                         return 0;
3786                 } else if (ret == 2) {
3787                         /* Still must send status,
3788                            SCF_TRANSPORT_TASK_SENSE was set */
3789                         spin_lock_bh(&cmd->istate_lock);
3790                         cmd->i_state = ISTATE_SEND_STATUS;
3791                         spin_unlock_bh(&cmd->istate_lock);
3792                         state = ISTATE_SEND_STATUS;
3793                         goto check_rsp_state;
3794                 }
3795
3796                 break;
3797         case ISTATE_SEND_STATUS:
3798         case ISTATE_SEND_STATUS_RECOVERY:
3799                 ret = iscsit_send_response(cmd, conn);
3800                 break;
3801         case ISTATE_SEND_LOGOUTRSP:
3802                 ret = iscsit_send_logout(cmd, conn);
3803                 break;
3804         case ISTATE_SEND_ASYNCMSG:
3805                 ret = iscsit_send_conn_drop_async_message(
3806                         cmd, conn);
3807                 break;
3808         case ISTATE_SEND_NOPIN:
3809                 ret = iscsit_send_nopin(cmd, conn);
3810                 break;
3811         case ISTATE_SEND_REJECT:
3812                 ret = iscsit_send_reject(cmd, conn);
3813                 break;
3814         case ISTATE_SEND_TASKMGTRSP:
3815                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3816                 if (ret != 0)
3817                         break;
3818                 ret = iscsit_tmr_post_handler(cmd, conn);
3819                 if (ret != 0)
3820                         iscsit_fall_back_to_erl0(conn->sess);
3821                 break;
3822         case ISTATE_SEND_TEXTRSP:
3823                 ret = iscsit_send_text_rsp(cmd, conn);
3824                 break;
3825         default:
3826                 pr_err("Unknown Opcode: 0x%02x ITT:"
3827                        " 0x%08x, i_state: %d on CID: %hu\n",
3828                        cmd->iscsi_opcode, cmd->init_task_tag,
3829                        state, conn->cid);
3830                 goto err;
3831         }
3832         if (ret < 0)
3833                 goto err;
3834
3835         switch (state) {
3836         case ISTATE_SEND_LOGOUTRSP:
3837                 if (!iscsit_logout_post_handler(cmd, conn))
3838                         return -ECONNRESET;
3839                 fallthrough;
3840         case ISTATE_SEND_STATUS:
3841         case ISTATE_SEND_ASYNCMSG:
3842         case ISTATE_SEND_NOPIN:
3843         case ISTATE_SEND_STATUS_RECOVERY:
3844         case ISTATE_SEND_TEXTRSP:
3845         case ISTATE_SEND_TASKMGTRSP:
3846         case ISTATE_SEND_REJECT:
3847                 spin_lock_bh(&cmd->istate_lock);
3848                 cmd->i_state = ISTATE_SENT_STATUS;
3849                 spin_unlock_bh(&cmd->istate_lock);
3850                 break;
3851         default:
3852                 pr_err("Unknown Opcode: 0x%02x ITT:"
3853                        " 0x%08x, i_state: %d on CID: %hu\n",
3854                        cmd->iscsi_opcode, cmd->init_task_tag,
3855                        cmd->i_state, conn->cid);
3856                 goto err;
3857         }
3858
3859         if (atomic_read(&conn->check_immediate_queue))
3860                 return 1;
3861
3862         return 0;
3863
3864 err:
3865         return -1;
3866 }
3867 EXPORT_SYMBOL(iscsit_response_queue);
3868
3869 static int iscsit_handle_response_queue(struct iscsi_conn *conn)
3870 {
3871         struct iscsit_transport *t = conn->conn_transport;
3872         struct iscsi_queue_req *qr;
3873         struct iscsi_cmd *cmd;
3874         u8 state;
3875         int ret;
3876
3877         while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3878                 cmd = qr->cmd;
3879                 state = qr->state;
3880                 kmem_cache_free(lio_qr_cache, qr);
3881
3882                 ret = t->iscsit_response_queue(conn, cmd, state);
3883                 if (ret == 1 || ret < 0)
3884                         return ret;
3885         }
3886
3887         return 0;
3888 }
3889
3890 int iscsi_target_tx_thread(void *arg)
3891 {
3892         int ret = 0;
3893         struct iscsi_conn *conn = arg;
3894         bool conn_freed = false;
3895
3896         /*
3897          * Allow ourselves to be interrupted by SIGINT so that a
3898          * connection recovery / failure event can be triggered externally.
3899          */
3900         allow_signal(SIGINT);
3901
3902         while (!kthread_should_stop()) {
3903                 /*
3904                  * Ensure that both TX and RX per connection kthreads
3905                  * are scheduled to run on the same CPU.
3906                  */
3907                 iscsit_thread_check_cpumask(conn, current, 1);
3908
3909                 wait_event_interruptible(conn->queues_wq,
3910                                          !iscsit_conn_all_queues_empty(conn));
3911
3912                 if (signal_pending(current))
3913                         goto transport_err;
3914
3915 get_immediate:
3916                 ret = iscsit_handle_immediate_queue(conn);
3917                 if (ret < 0)
3918                         goto transport_err;
3919
3920                 ret = iscsit_handle_response_queue(conn);
3921                 if (ret == 1) {
3922                         goto get_immediate;
3923                 } else if (ret == -ECONNRESET) {
3924                         conn_freed = true;
3925                         goto out;
3926                 } else if (ret < 0) {
3927                         goto transport_err;
3928                 }
3929         }
3930
3931 transport_err:
3932         /*
3933          * Avoid the normal connection failure code-path if this connection
3934          * is still within LOGIN mode, and iscsi_np process context is
3935          * responsible for cleaning up the early connection failure.
3936          */
3937         if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN)
3938                 iscsit_take_action_for_connection_exit(conn, &conn_freed);
3939 out:
3940         if (!conn_freed) {
3941                 while (!kthread_should_stop()) {
3942                         msleep(100);
3943                 }
3944         }
3945         return 0;
3946 }
3947
3948 static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
3949 {
3950         struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
3951         struct iscsi_cmd *cmd;
3952         int ret = 0;
3953
3954         switch (hdr->opcode & ISCSI_OPCODE_MASK) {
3955         case ISCSI_OP_SCSI_CMD:
3956                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3957                 if (!cmd)
3958                         goto reject;
3959
3960                 ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
3961                 break;
3962         case ISCSI_OP_SCSI_DATA_OUT:
3963                 ret = iscsit_handle_data_out(conn, buf);
3964                 break;
3965         case ISCSI_OP_NOOP_OUT:
3966                 cmd = NULL;
3967                 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
3968                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3969                         if (!cmd)
3970                                 goto reject;
3971                 }
3972                 ret = iscsit_handle_nop_out(conn, cmd, buf);
3973                 break;
3974         case ISCSI_OP_SCSI_TMFUNC:
3975                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3976                 if (!cmd)
3977                         goto reject;
3978
3979                 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
3980                 break;
3981         case ISCSI_OP_TEXT:
3982                 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
3983                         cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
3984                         if (!cmd)
3985                                 goto reject;
3986                 } else {
3987                         cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3988                         if (!cmd)
3989                                 goto reject;
3990                 }
3991
3992                 ret = iscsit_handle_text_cmd(conn, cmd, buf);
3993                 break;
3994         case ISCSI_OP_LOGOUT:
3995                 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
3996                 if (!cmd)
3997                         goto reject;
3998
3999                 ret = iscsit_handle_logout_cmd(conn, cmd, buf);
4000                 if (ret > 0)
4001                         wait_for_completion_timeout(&conn->conn_logout_comp,
4002                                         SECONDS_FOR_LOGOUT_COMP * HZ);
4003                 break;
4004         case ISCSI_OP_SNACK:
4005                 ret = iscsit_handle_snack(conn, buf);
4006                 break;
4007         default:
4008                 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
4009                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
4010                         pr_err("Cannot recover from unknown"
4011                         " opcode while ERL=0, closing iSCSI connection.\n");
4012                         return -1;
4013                 }
4014                 pr_err("Unable to recover from unknown opcode while OFMarker=No,"
4015                        " closing iSCSI connection.\n");
4016                 ret = -1;
4017                 break;
4018         }
4019
4020         return ret;
4021 reject:
4022         return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
4023 }
4024
4025 static bool iscsi_target_check_conn_state(struct iscsi_conn *conn)
4026 {
4027         bool ret;
4028
4029         spin_lock_bh(&conn->state_lock);
4030         ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN);
4031         spin_unlock_bh(&conn->state_lock);
4032
4033         return ret;
4034 }
4035
4036 static void iscsit_get_rx_pdu(struct iscsi_conn *conn)
4037 {
4038         int ret;
4039         u8 *buffer, opcode;
4040         u32 checksum = 0, digest = 0;
4041         struct kvec iov;
4042
4043         buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL);
4044         if (!buffer)
4045                 return;
4046
4047         while (!kthread_should_stop()) {
4048                 /*
4049                  * Ensure that both TX and RX per connection kthreads
4050                  * are scheduled to run on the same CPU.
4051                  */
4052                 iscsit_thread_check_cpumask(conn, current, 0);
4053
4054                 memset(&iov, 0, sizeof(struct kvec));
4055
4056                 iov.iov_base    = buffer;
4057                 iov.iov_len     = ISCSI_HDR_LEN;
4058
4059                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
4060                 if (ret != ISCSI_HDR_LEN) {
4061                         iscsit_rx_thread_wait_for_tcp(conn);
4062                         break;
4063                 }
4064
4065                 if (conn->conn_ops->HeaderDigest) {
4066                         iov.iov_base    = &digest;
4067                         iov.iov_len     = ISCSI_CRC_LEN;
4068
4069                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
4070                         if (ret != ISCSI_CRC_LEN) {
4071                                 iscsit_rx_thread_wait_for_tcp(conn);
4072                                 break;
4073                         }
4074
4075                         iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,
4076                                                   ISCSI_HDR_LEN, 0, NULL,
4077                                                   &checksum);
4078
4079                         if (digest != checksum) {
4080                                 pr_err("HeaderDigest CRC32C failed,"
4081                                         " received 0x%08x, computed 0x%08x\n",
4082                                         digest, checksum);
4083                                 /*
4084                                  * Set the PDU to 0xff so it will intentionally
4085                                  * hit default in the switch below.
4086                                  */
4087                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
4088                                 atomic_long_inc(&conn->sess->conn_digest_errors);
4089                         } else {
4090                                 pr_debug("Got HeaderDigest CRC32C"
4091                                                 " 0x%08x\n", checksum);
4092                         }
4093                 }
4094
4095                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
4096                         break;
4097
4098                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
4099
4100                 if (conn->sess->sess_ops->SessionType &&
4101                    ((!(opcode & ISCSI_OP_TEXT)) ||
4102                     (!(opcode & ISCSI_OP_LOGOUT)))) {
4103                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
4104                         " while in Discovery Session, rejecting.\n", opcode);
4105                         iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
4106                                           buffer);
4107                         break;
4108                 }
4109
4110                 ret = iscsi_target_rx_opcode(conn, buffer);
4111                 if (ret < 0)
4112                         break;
4113         }
4114
4115         kfree(buffer);
4116 }
4117
4118 int iscsi_target_rx_thread(void *arg)
4119 {
4120         int rc;
4121         struct iscsi_conn *conn = arg;
4122         bool conn_freed = false;
4123
4124         /*
4125          * Allow ourselves to be interrupted by SIGINT so that a
4126          * connection recovery / failure event can be triggered externally.
4127          */
4128         allow_signal(SIGINT);
4129         /*
4130          * Wait for iscsi_post_login_handler() to complete before allowing
4131          * incoming iscsi/tcp socket I/O, and/or failing the connection.
4132          */
4133         rc = wait_for_completion_interruptible(&conn->rx_login_comp);
4134         if (rc < 0 || iscsi_target_check_conn_state(conn))
4135                 goto out;
4136
4137         if (!conn->conn_transport->iscsit_get_rx_pdu)
4138                 return 0;
4139
4140         conn->conn_transport->iscsit_get_rx_pdu(conn);
4141
4142         if (!signal_pending(current))
4143                 atomic_set(&conn->transport_failed, 1);
4144         iscsit_take_action_for_connection_exit(conn, &conn_freed);
4145
4146 out:
4147         if (!conn_freed) {
4148                 while (!kthread_should_stop()) {
4149                         msleep(100);
4150                 }
4151         }
4152
4153         return 0;
4154 }
4155
4156 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
4157 {
4158         LIST_HEAD(tmp_list);
4159         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
4160         struct iscsi_session *sess = conn->sess;
4161         /*
4162          * We expect this function to only ever be called from either RX or TX
4163          * thread context via iscsit_close_connection() once the other context
4164          * has been reset -> returned sleeping pre-handler state.
4165          */
4166         spin_lock_bh(&conn->cmd_lock);
4167         list_splice_init(&conn->conn_cmd_list, &tmp_list);
4168
4169         list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
4170                 struct se_cmd *se_cmd = &cmd->se_cmd;
4171
4172                 if (se_cmd->se_tfo != NULL) {
4173                         spin_lock_irq(&se_cmd->t_state_lock);
4174                         if (se_cmd->transport_state & CMD_T_ABORTED) {
4175                                 /*
4176                                  * LIO's abort path owns the cleanup for this,
4177                                  * so put it back on the list and let
4178                                  * aborted_task handle it.
4179                                  */
4180                                 list_move_tail(&cmd->i_conn_node,
4181                                                &conn->conn_cmd_list);
4182                         } else {
4183                                 se_cmd->transport_state |= CMD_T_FABRIC_STOP;
4184                         }
4185                         spin_unlock_irq(&se_cmd->t_state_lock);
4186                 }
4187         }
4188         spin_unlock_bh(&conn->cmd_lock);
4189
4190         list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
4191                 list_del_init(&cmd->i_conn_node);
4192
4193                 iscsit_increment_maxcmdsn(cmd, sess);
4194                 iscsit_free_cmd(cmd, true);
4195
4196         }
4197 }
4198
4199 static void iscsit_stop_timers_for_cmds(
4200         struct iscsi_conn *conn)
4201 {
4202         struct iscsi_cmd *cmd;
4203
4204         spin_lock_bh(&conn->cmd_lock);
4205         list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
4206                 if (cmd->data_direction == DMA_TO_DEVICE)
4207                         iscsit_stop_dataout_timer(cmd);
4208         }
4209         spin_unlock_bh(&conn->cmd_lock);
4210 }
4211
4212 int iscsit_close_connection(
4213         struct iscsi_conn *conn)
4214 {
4215         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
4216         struct iscsi_session    *sess = conn->sess;
4217
4218         pr_debug("Closing iSCSI connection CID %hu on SID:"
4219                 " %u\n", conn->cid, sess->sid);
4220         /*
4221          * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD
4222          * case just in case the RX Thread in iscsi_target_rx_opcode() is
4223          * sleeping and the logout response never got sent because the
4224          * connection failed.
4225          *
4226          * However for iser-target, isert_wait4logout() is using conn_logout_comp
4227          * to signal logout response TX interrupt completion.  Go ahead and skip
4228          * this for iser since isert_rx_opcode() does not wait on logout failure,
4229          * and to avoid iscsi_conn pointer dereference in iser-target code.
4230          */
4231         if (!conn->conn_transport->rdma_shutdown)
4232                 complete(&conn->conn_logout_comp);
4233
4234         if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) {
4235                 if (conn->tx_thread &&
4236                     cmpxchg(&conn->tx_thread_active, true, false)) {
4237                         send_sig(SIGINT, conn->tx_thread, 1);
4238                         kthread_stop(conn->tx_thread);
4239                 }
4240         } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) {
4241                 if (conn->rx_thread &&
4242                     cmpxchg(&conn->rx_thread_active, true, false)) {
4243                         send_sig(SIGINT, conn->rx_thread, 1);
4244                         kthread_stop(conn->rx_thread);
4245                 }
4246         }
4247
4248         spin_lock(&iscsit_global->ts_bitmap_lock);
4249         bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
4250                               get_order(1));
4251         spin_unlock(&iscsit_global->ts_bitmap_lock);
4252
4253         iscsit_stop_timers_for_cmds(conn);
4254         iscsit_stop_nopin_response_timer(conn);
4255         iscsit_stop_nopin_timer(conn);
4256
4257         if (conn->conn_transport->iscsit_wait_conn)
4258                 conn->conn_transport->iscsit_wait_conn(conn);
4259
4260         /*
4261          * During Connection recovery drop unacknowledged out of order
4262          * commands for this connection, and prepare the other commands
4263          * for reallegiance.
4264          *
4265          * During normal operation clear the out of order commands (but
4266          * do not free the struct iscsi_ooo_cmdsn's) and release all
4267          * struct iscsi_cmds.
4268          */
4269         if (atomic_read(&conn->connection_recovery)) {
4270                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
4271                 iscsit_prepare_cmds_for_reallegiance(conn);
4272         } else {
4273                 iscsit_clear_ooo_cmdsns_for_conn(conn);
4274                 iscsit_release_commands_from_conn(conn);
4275         }
4276         iscsit_free_queue_reqs_for_conn(conn);
4277
4278         /*
4279          * Handle decrementing session or connection usage count if
4280          * a logout response was not able to be sent because the
4281          * connection failed.  Fall back to Session Recovery here.
4282          */
4283         if (atomic_read(&conn->conn_logout_remove)) {
4284                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4285                         iscsit_dec_conn_usage_count(conn);
4286                         iscsit_dec_session_usage_count(sess);
4287                 }
4288                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4289                         iscsit_dec_conn_usage_count(conn);
4290
4291                 atomic_set(&conn->conn_logout_remove, 0);
4292                 atomic_set(&sess->session_reinstatement, 0);
4293                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4294         }
4295
4296         spin_lock_bh(&sess->conn_lock);
4297         list_del(&conn->conn_list);
4298
4299         /*
4300          * Attempt to let the Initiator know this connection failed by
4301          * sending an Connection Dropped Async Message on another
4302          * active connection.
4303          */
4304         if (atomic_read(&conn->connection_recovery))
4305                 iscsit_build_conn_drop_async_message(conn);
4306
4307         spin_unlock_bh(&sess->conn_lock);
4308
4309         /*
4310          * If connection reinstatement is being performed on this connection,
4311          * up the connection reinstatement semaphore that is being blocked on
4312          * in iscsit_cause_connection_reinstatement().
4313          */
4314         spin_lock_bh(&conn->state_lock);
4315         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4316                 spin_unlock_bh(&conn->state_lock);
4317                 complete(&conn->conn_wait_comp);
4318                 wait_for_completion(&conn->conn_post_wait_comp);
4319                 spin_lock_bh(&conn->state_lock);
4320         }
4321
4322         /*
4323          * If connection reinstatement is being performed on this connection
4324          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4325          * connection wait rcfr semaphore that is being blocked on
4326          * an iscsit_connection_reinstatement_rcfr().
4327          */
4328         if (atomic_read(&conn->connection_wait_rcfr)) {
4329                 spin_unlock_bh(&conn->state_lock);
4330                 complete(&conn->conn_wait_rcfr_comp);
4331                 wait_for_completion(&conn->conn_post_wait_comp);
4332                 spin_lock_bh(&conn->state_lock);
4333         }
4334         atomic_set(&conn->connection_reinstatement, 1);
4335         spin_unlock_bh(&conn->state_lock);
4336
4337         /*
4338          * If any other processes are accessing this connection pointer we
4339          * must wait until they have completed.
4340          */
4341         iscsit_check_conn_usage_count(conn);
4342
4343         ahash_request_free(conn->conn_tx_hash);
4344         if (conn->conn_rx_hash) {
4345                 struct crypto_ahash *tfm;
4346
4347                 tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
4348                 ahash_request_free(conn->conn_rx_hash);
4349                 crypto_free_ahash(tfm);
4350         }
4351
4352         if (conn->sock)
4353                 sock_release(conn->sock);
4354
4355         if (conn->conn_transport->iscsit_free_conn)
4356                 conn->conn_transport->iscsit_free_conn(conn);
4357
4358         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4359         conn->conn_state = TARG_CONN_STATE_FREE;
4360         iscsit_free_conn(conn);
4361
4362         spin_lock_bh(&sess->conn_lock);
4363         atomic_dec(&sess->nconn);
4364         pr_debug("Decremented iSCSI connection count to %hu from node:"
4365                 " %s\n", atomic_read(&sess->nconn),
4366                 sess->sess_ops->InitiatorName);
4367         /*
4368          * Make sure that if one connection fails in an non ERL=2 iSCSI
4369          * Session that they all fail.
4370          */
4371         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4372              !atomic_read(&sess->session_logout))
4373                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4374
4375         /*
4376          * If this was not the last connection in the session, and we are
4377          * performing session reinstatement or falling back to ERL=0, call
4378          * iscsit_stop_session() without sleeping to shutdown the other
4379          * active connections.
4380          */
4381         if (atomic_read(&sess->nconn)) {
4382                 if (!atomic_read(&sess->session_reinstatement) &&
4383                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4384                         spin_unlock_bh(&sess->conn_lock);
4385                         return 0;
4386                 }
4387                 if (!atomic_read(&sess->session_stop_active)) {
4388                         atomic_set(&sess->session_stop_active, 1);
4389                         spin_unlock_bh(&sess->conn_lock);
4390                         iscsit_stop_session(sess, 0, 0);
4391                         return 0;
4392                 }
4393                 spin_unlock_bh(&sess->conn_lock);
4394                 return 0;
4395         }
4396
4397         /*
4398          * If this was the last connection in the session and one of the
4399          * following is occurring:
4400          *
4401          * Session Reinstatement is not being performed, and are falling back
4402          * to ERL=0 call iscsit_close_session().
4403          *
4404          * Session Logout was requested.  iscsit_close_session() will be called
4405          * elsewhere.
4406          *
4407          * Session Continuation is not being performed, start the Time2Retain
4408          * handler and check if sleep_on_sess_wait_sem is active.
4409          */
4410         if (!atomic_read(&sess->session_reinstatement) &&
4411              atomic_read(&sess->session_fall_back_to_erl0)) {
4412                 spin_unlock_bh(&sess->conn_lock);
4413                 complete_all(&sess->session_wait_comp);
4414                 iscsit_close_session(sess, true);
4415
4416                 return 0;
4417         } else if (atomic_read(&sess->session_logout)) {
4418                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4419                 sess->session_state = TARG_SESS_STATE_FREE;
4420
4421                 if (atomic_read(&sess->session_close)) {
4422                         spin_unlock_bh(&sess->conn_lock);
4423                         complete_all(&sess->session_wait_comp);
4424                         iscsit_close_session(sess, true);
4425                 } else {
4426                         spin_unlock_bh(&sess->conn_lock);
4427                 }
4428
4429                 return 0;
4430         } else {
4431                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4432                 sess->session_state = TARG_SESS_STATE_FAILED;
4433
4434                 if (!atomic_read(&sess->session_continuation))
4435                         iscsit_start_time2retain_handler(sess);
4436
4437                 if (atomic_read(&sess->session_close)) {
4438                         spin_unlock_bh(&sess->conn_lock);
4439                         complete_all(&sess->session_wait_comp);
4440                         iscsit_close_session(sess, true);
4441                 } else {
4442                         spin_unlock_bh(&sess->conn_lock);
4443                 }
4444
4445                 return 0;
4446         }
4447 }
4448
4449 /*
4450  * If the iSCSI Session for the iSCSI Initiator Node exists,
4451  * forcefully shutdown the iSCSI NEXUS.
4452  */
4453 int iscsit_close_session(struct iscsi_session *sess, bool can_sleep)
4454 {
4455         struct iscsi_portal_group *tpg = sess->tpg;
4456         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4457
4458         if (atomic_read(&sess->nconn)) {
4459                 pr_err("%d connection(s) still exist for iSCSI session"
4460                         " to %s\n", atomic_read(&sess->nconn),
4461                         sess->sess_ops->InitiatorName);
4462                 BUG();
4463         }
4464
4465         spin_lock_bh(&se_tpg->session_lock);
4466         atomic_set(&sess->session_logout, 1);
4467         atomic_set(&sess->session_reinstatement, 1);
4468         iscsit_stop_time2retain_timer(sess);
4469         spin_unlock_bh(&se_tpg->session_lock);
4470
4471         /*
4472          * transport_deregister_session_configfs() will clear the
4473          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4474          * can be setting it again with __transport_register_session() in
4475          * iscsi_post_login_handler() again after the iscsit_stop_session()
4476          * completes in iscsi_np context.
4477          */
4478         transport_deregister_session_configfs(sess->se_sess);
4479
4480         /*
4481          * If any other processes are accessing this session pointer we must
4482          * wait until they have completed.  If we are in an interrupt (the
4483          * time2retain handler) and contain and active session usage count we
4484          * restart the timer and exit.
4485          */
4486         if (iscsit_check_session_usage_count(sess, can_sleep)) {
4487                 atomic_set(&sess->session_logout, 0);
4488                 iscsit_start_time2retain_handler(sess);
4489                 return 0;
4490         }
4491
4492         transport_deregister_session(sess->se_sess);
4493
4494         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4495                 iscsit_free_connection_recovery_entries(sess);
4496
4497         iscsit_free_all_ooo_cmdsns(sess);
4498
4499         spin_lock_bh(&se_tpg->session_lock);
4500         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4501         sess->session_state = TARG_SESS_STATE_FREE;
4502         pr_debug("Released iSCSI session from node: %s\n",
4503                         sess->sess_ops->InitiatorName);
4504         tpg->nsessions--;
4505         if (tpg->tpg_tiqn)
4506                 tpg->tpg_tiqn->tiqn_nsessions--;
4507
4508         pr_debug("Decremented number of active iSCSI Sessions on"
4509                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4510
4511         ida_free(&sess_ida, sess->session_index);
4512         kfree(sess->sess_ops);
4513         sess->sess_ops = NULL;
4514         spin_unlock_bh(&se_tpg->session_lock);
4515
4516         kfree(sess);
4517         return 0;
4518 }
4519
4520 static void iscsit_logout_post_handler_closesession(
4521         struct iscsi_conn *conn)
4522 {
4523         struct iscsi_session *sess = conn->sess;
4524         int sleep = 1;
4525         /*
4526          * Traditional iscsi/tcp will invoke this logic from TX thread
4527          * context during session logout, so clear tx_thread_active and
4528          * sleep if iscsit_close_connection() has not already occured.
4529          *
4530          * Since iser-target invokes this logic from it's own workqueue,
4531          * always sleep waiting for RX/TX thread shutdown to complete
4532          * within iscsit_close_connection().
4533          */
4534         if (!conn->conn_transport->rdma_shutdown) {
4535                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4536                 if (!sleep)
4537                         return;
4538         }
4539
4540         atomic_set(&conn->conn_logout_remove, 0);
4541         complete(&conn->conn_logout_comp);
4542
4543         iscsit_dec_conn_usage_count(conn);
4544         atomic_set(&sess->session_close, 1);
4545         iscsit_stop_session(sess, sleep, sleep);
4546         iscsit_dec_session_usage_count(sess);
4547 }
4548
4549 static void iscsit_logout_post_handler_samecid(
4550         struct iscsi_conn *conn)
4551 {
4552         int sleep = 1;
4553
4554         if (!conn->conn_transport->rdma_shutdown) {
4555                 sleep = cmpxchg(&conn->tx_thread_active, true, false);
4556                 if (!sleep)
4557                         return;
4558         }
4559
4560         atomic_set(&conn->conn_logout_remove, 0);
4561         complete(&conn->conn_logout_comp);
4562
4563         iscsit_cause_connection_reinstatement(conn, sleep);
4564         iscsit_dec_conn_usage_count(conn);
4565 }
4566
4567 static void iscsit_logout_post_handler_diffcid(
4568         struct iscsi_conn *conn,
4569         u16 cid)
4570 {
4571         struct iscsi_conn *l_conn;
4572         struct iscsi_session *sess = conn->sess;
4573         bool conn_found = false;
4574
4575         if (!sess)
4576                 return;
4577
4578         spin_lock_bh(&sess->conn_lock);
4579         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4580                 if (l_conn->cid == cid) {
4581                         iscsit_inc_conn_usage_count(l_conn);
4582                         conn_found = true;
4583                         break;
4584                 }
4585         }
4586         spin_unlock_bh(&sess->conn_lock);
4587
4588         if (!conn_found)
4589                 return;
4590
4591         if (l_conn->sock)
4592                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4593
4594         spin_lock_bh(&l_conn->state_lock);
4595         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4596         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4597         spin_unlock_bh(&l_conn->state_lock);
4598
4599         iscsit_cause_connection_reinstatement(l_conn, 1);
4600         iscsit_dec_conn_usage_count(l_conn);
4601 }
4602
4603 /*
4604  *      Return of 0 causes the TX thread to restart.
4605  */
4606 int iscsit_logout_post_handler(
4607         struct iscsi_cmd *cmd,
4608         struct iscsi_conn *conn)
4609 {
4610         int ret = 0;
4611
4612         switch (cmd->logout_reason) {
4613         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4614                 switch (cmd->logout_response) {
4615                 case ISCSI_LOGOUT_SUCCESS:
4616                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4617                 default:
4618                         iscsit_logout_post_handler_closesession(conn);
4619                         break;
4620                 }
4621                 break;
4622         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4623                 if (conn->cid == cmd->logout_cid) {
4624                         switch (cmd->logout_response) {
4625                         case ISCSI_LOGOUT_SUCCESS:
4626                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4627                         default:
4628                                 iscsit_logout_post_handler_samecid(conn);
4629                                 break;
4630                         }
4631                 } else {
4632                         switch (cmd->logout_response) {
4633                         case ISCSI_LOGOUT_SUCCESS:
4634                                 iscsit_logout_post_handler_diffcid(conn,
4635                                         cmd->logout_cid);
4636                                 break;
4637                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4638                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4639                         default:
4640                                 break;
4641                         }
4642                         ret = 1;
4643                 }
4644                 break;
4645         case ISCSI_LOGOUT_REASON_RECOVERY:
4646                 switch (cmd->logout_response) {
4647                 case ISCSI_LOGOUT_SUCCESS:
4648                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4649                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4650                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4651                 default:
4652                         break;
4653                 }
4654                 ret = 1;
4655                 break;
4656         default:
4657                 break;
4658
4659         }
4660         return ret;
4661 }
4662 EXPORT_SYMBOL(iscsit_logout_post_handler);
4663
4664 void iscsit_fail_session(struct iscsi_session *sess)
4665 {
4666         struct iscsi_conn *conn;
4667
4668         spin_lock_bh(&sess->conn_lock);
4669         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4670                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4671                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4672         }
4673         spin_unlock_bh(&sess->conn_lock);
4674
4675         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4676         sess->session_state = TARG_SESS_STATE_FAILED;
4677 }
4678
4679 void iscsit_stop_session(
4680         struct iscsi_session *sess,
4681         int session_sleep,
4682         int connection_sleep)
4683 {
4684         u16 conn_count = atomic_read(&sess->nconn);
4685         struct iscsi_conn *conn, *conn_tmp = NULL;
4686         int is_last;
4687
4688         spin_lock_bh(&sess->conn_lock);
4689
4690         if (connection_sleep) {
4691                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4692                                 conn_list) {
4693                         if (conn_count == 0)
4694                                 break;
4695
4696                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4697                                 is_last = 1;
4698                         } else {
4699                                 iscsit_inc_conn_usage_count(conn_tmp);
4700                                 is_last = 0;
4701                         }
4702                         iscsit_inc_conn_usage_count(conn);
4703
4704                         spin_unlock_bh(&sess->conn_lock);
4705                         iscsit_cause_connection_reinstatement(conn, 1);
4706                         spin_lock_bh(&sess->conn_lock);
4707
4708                         iscsit_dec_conn_usage_count(conn);
4709                         if (is_last == 0)
4710                                 iscsit_dec_conn_usage_count(conn_tmp);
4711                         conn_count--;
4712                 }
4713         } else {
4714                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4715                         iscsit_cause_connection_reinstatement(conn, 0);
4716         }
4717
4718         if (session_sleep && atomic_read(&sess->nconn)) {
4719                 spin_unlock_bh(&sess->conn_lock);
4720                 wait_for_completion(&sess->session_wait_comp);
4721         } else
4722                 spin_unlock_bh(&sess->conn_lock);
4723 }
4724
4725 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4726 {
4727         struct iscsi_session *sess;
4728         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4729         struct se_session *se_sess, *se_sess_tmp;
4730         LIST_HEAD(free_list);
4731         int session_count = 0;
4732
4733         spin_lock_bh(&se_tpg->session_lock);
4734         if (tpg->nsessions && !force) {
4735                 spin_unlock_bh(&se_tpg->session_lock);
4736                 return -1;
4737         }
4738
4739         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4740                         sess_list) {
4741                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4742
4743                 spin_lock(&sess->conn_lock);
4744                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4745                     atomic_read(&sess->session_logout) ||
4746                     atomic_read(&sess->session_close) ||
4747                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4748                         spin_unlock(&sess->conn_lock);
4749                         continue;
4750                 }
4751                 iscsit_inc_session_usage_count(sess);
4752                 atomic_set(&sess->session_reinstatement, 1);
4753                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4754                 atomic_set(&sess->session_close, 1);
4755                 spin_unlock(&sess->conn_lock);
4756
4757                 list_move_tail(&se_sess->sess_list, &free_list);
4758         }
4759         spin_unlock_bh(&se_tpg->session_lock);
4760
4761         list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) {
4762                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4763
4764                 list_del_init(&se_sess->sess_list);
4765                 iscsit_stop_session(sess, 1, 1);
4766                 iscsit_dec_session_usage_count(sess);
4767                 session_count++;
4768         }
4769
4770         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4771                         " Group: %hu\n", session_count, tpg->tpgt);
4772         return 0;
4773 }
4774
4775 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4776 MODULE_VERSION("4.1.x");
4777 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4778 MODULE_LICENSE("GPL");
4779
4780 module_init(iscsi_target_init_module);
4781 module_exit(iscsi_target_cleanup_module);