projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] target: Fix bug with task_sg chained transport_free_dev_tasks release
[linux-block.git] / drivers / target / target_core_transport.c
CommitLineData
c66ac9db
NB		 1/*******************************************************************************
2 * Filename: target_core_transport.c
3 *
4 * This file contains the Generic Target Engine Core.
5 *
6 * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc.
7 * Copyright (c) 2005, 2006, 2007 SBE, Inc.
8 * Copyright (c) 2007-2010 Rising Tide Systems
9 * Copyright (c) 2008-2010 Linux-iSCSI.org
10 *
11 * Nicholas A. Bellinger <nab@kernel.org>
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 *
27 ******************************************************************************/
28
29#include <linux/version.h>
30#include <linux/net.h>
31#include <linux/delay.h>
32#include <linux/string.h>
33#include <linux/timer.h>
34#include <linux/slab.h>
35#include <linux/blkdev.h>
36#include <linux/spinlock.h>
c66ac9db
NB		 37#include <linux/kthread.h>
38#include <linux/in.h>
39#include <linux/cdrom.h>
40#include <asm/unaligned.h>
41#include <net/sock.h>
42#include <net/tcp.h>
43#include <scsi/scsi.h>
44#include <scsi/scsi_cmnd.h>
45#include <scsi/libsas.h> /* For TASK_ATTR_* */
46
47#include <target/target_core_base.h>
48#include <target/target_core_device.h>
49#include <target/target_core_tmr.h>
50#include <target/target_core_tpg.h>
51#include <target/target_core_transport.h>
52#include <target/target_core_fabric_ops.h>
53#include <target/target_core_configfs.h>
54
55#include "target_core_alua.h"
56#include "target_core_hba.h"
57#include "target_core_pr.h"
58#include "target_core_scdb.h"
59#include "target_core_ua.h"
60
61/* #define DEBUG_CDB_HANDLER */
62#ifdef DEBUG_CDB_HANDLER
63#define DEBUG_CDB_H(x...) printk(KERN_INFO x)
64#else
65#define DEBUG_CDB_H(x...)
66#endif
67
68/* #define DEBUG_CMD_MAP */
69#ifdef DEBUG_CMD_MAP
70#define DEBUG_CMD_M(x...) printk(KERN_INFO x)
71#else
72#define DEBUG_CMD_M(x...)
73#endif
74
75/* #define DEBUG_MEM_ALLOC */
76#ifdef DEBUG_MEM_ALLOC
77#define DEBUG_MEM(x...) printk(KERN_INFO x)
78#else
79#define DEBUG_MEM(x...)
80#endif
81
82/* #define DEBUG_MEM2_ALLOC */
83#ifdef DEBUG_MEM2_ALLOC
84#define DEBUG_MEM2(x...) printk(KERN_INFO x)
85#else
86#define DEBUG_MEM2(x...)
87#endif
88
89/* #define DEBUG_SG_CALC */
90#ifdef DEBUG_SG_CALC
91#define DEBUG_SC(x...) printk(KERN_INFO x)
92#else
93#define DEBUG_SC(x...)
94#endif
95
96/* #define DEBUG_SE_OBJ */
97#ifdef DEBUG_SE_OBJ
98#define DEBUG_SO(x...) printk(KERN_INFO x)
99#else
100#define DEBUG_SO(x...)
101#endif
102
103/* #define DEBUG_CMD_VOL */
104#ifdef DEBUG_CMD_VOL
105#define DEBUG_VOL(x...) printk(KERN_INFO x)
106#else
107#define DEBUG_VOL(x...)
108#endif
109
110/* #define DEBUG_CMD_STOP */
111#ifdef DEBUG_CMD_STOP
112#define DEBUG_CS(x...) printk(KERN_INFO x)
113#else
114#define DEBUG_CS(x...)
115#endif
116
117/* #define DEBUG_PASSTHROUGH */
118#ifdef DEBUG_PASSTHROUGH
119#define DEBUG_PT(x...) printk(KERN_INFO x)
120#else
121#define DEBUG_PT(x...)
122#endif
123
124/* #define DEBUG_TASK_STOP */
125#ifdef DEBUG_TASK_STOP
126#define DEBUG_TS(x...) printk(KERN_INFO x)
127#else
128#define DEBUG_TS(x...)
129#endif
130
131/* #define DEBUG_TRANSPORT_STOP */
132#ifdef DEBUG_TRANSPORT_STOP
133#define DEBUG_TRANSPORT_S(x...) printk(KERN_INFO x)
134#else
135#define DEBUG_TRANSPORT_S(x...)
136#endif
137
138/* #define DEBUG_TASK_FAILURE */
139#ifdef DEBUG_TASK_FAILURE
140#define DEBUG_TF(x...) printk(KERN_INFO x)
141#else
142#define DEBUG_TF(x...)
143#endif
144
145/* #define DEBUG_DEV_OFFLINE */
146#ifdef DEBUG_DEV_OFFLINE
147#define DEBUG_DO(x...) printk(KERN_INFO x)
148#else
149#define DEBUG_DO(x...)
150#endif
151
152/* #define DEBUG_TASK_STATE */
153#ifdef DEBUG_TASK_STATE
154#define DEBUG_TSTATE(x...) printk(KERN_INFO x)
155#else
156#define DEBUG_TSTATE(x...)
157#endif
158
159/* #define DEBUG_STATUS_THR */
160#ifdef DEBUG_STATUS_THR
161#define DEBUG_ST(x...) printk(KERN_INFO x)
162#else
163#define DEBUG_ST(x...)
164#endif
165
166/* #define DEBUG_TASK_TIMEOUT */
167#ifdef DEBUG_TASK_TIMEOUT
168#define DEBUG_TT(x...) printk(KERN_INFO x)
169#else
170#define DEBUG_TT(x...)
171#endif
172
173/* #define DEBUG_GENERIC_REQUEST_FAILURE */
174#ifdef DEBUG_GENERIC_REQUEST_FAILURE
175#define DEBUG_GRF(x...) printk(KERN_INFO x)
176#else
177#define DEBUG_GRF(x...)
178#endif
179
180/* #define DEBUG_SAM_TASK_ATTRS */
181#ifdef DEBUG_SAM_TASK_ATTRS
182#define DEBUG_STA(x...) printk(KERN_INFO x)
183#else
184#define DEBUG_STA(x...)
185#endif
186
187struct se_global *se_global;
188
189static struct kmem_cache *se_cmd_cache;
190static struct kmem_cache *se_sess_cache;
191struct kmem_cache *se_tmr_req_cache;
192struct kmem_cache *se_ua_cache;
193struct kmem_cache *se_mem_cache;
194struct kmem_cache *t10_pr_reg_cache;
195struct kmem_cache *t10_alua_lu_gp_cache;
196struct kmem_cache *t10_alua_lu_gp_mem_cache;
197struct kmem_cache *t10_alua_tg_pt_gp_cache;
198struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;
199
200/* Used for transport_dev_get_map_*() */
201typedef int (*map_func_t)(struct se_task *, u32);
202
203static int transport_generic_write_pending(struct se_cmd *);
204static int transport_processing_thread(void *);
205static int __transport_execute_tasks(struct se_device *dev);
206static void transport_complete_task_attr(struct se_cmd *cmd);
207static void transport_direct_request_timeout(struct se_cmd *cmd);
208static void transport_free_dev_tasks(struct se_cmd *cmd);
209static u32 transport_generic_get_cdb_count(struct se_cmd *cmd,
210 unsigned long long starting_lba, u32 sectors,
211 enum dma_data_direction data_direction,
212 struct list_head *mem_list, int set_counts);
213static int transport_generic_get_mem(struct se_cmd *cmd, u32 length,
214 u32 dma_size);
215static int transport_generic_remove(struct se_cmd *cmd,
216 int release_to_pool, int session_reinstatement);
217static int transport_get_sectors(struct se_cmd *cmd);
218static struct list_head *transport_init_se_mem_list(void);
219static int transport_map_sg_to_mem(struct se_cmd *cmd,
220 struct list_head *se_mem_list, void *in_mem,
221 u32 *se_mem_cnt);
222static void transport_memcpy_se_mem_read_contig(struct se_cmd *cmd,
223 unsigned char *dst, struct list_head *se_mem_list);
224static void transport_release_fe_cmd(struct se_cmd *cmd);
225static void transport_remove_cmd_from_queue(struct se_cmd *cmd,
226 struct se_queue_obj *qobj);
227static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq);
228static void transport_stop_all_task_timers(struct se_cmd *cmd);
229
c66ac9db
NB		 230int init_se_global(void)
231{
232 struct se_global *global;
233
234 global = kzalloc(sizeof(struct se_global), GFP_KERNEL);
235 if (!(global)) {
236 printk(KERN_ERR "Unable to allocate memory for struct se_global\n");
237 return -1;
238 }
239
240 INIT_LIST_HEAD(&global->g_lu_gps_list);
241 INIT_LIST_HEAD(&global->g_se_tpg_list);
242 INIT_LIST_HEAD(&global->g_hba_list);
243 INIT_LIST_HEAD(&global->g_se_dev_list);
244 spin_lock_init(&global->g_device_lock);
245 spin_lock_init(&global->hba_lock);
246 spin_lock_init(&global->se_tpg_lock);
247 spin_lock_init(&global->lu_gps_lock);
248 spin_lock_init(&global->plugin_class_lock);
249
250 se_cmd_cache = kmem_cache_create("se_cmd_cache",
251 sizeof(struct se_cmd), __alignof__(struct se_cmd), 0, NULL);
252 if (!(se_cmd_cache)) {
253 printk(KERN_ERR "kmem_cache_create for struct se_cmd failed\n");
254 goto out;
255 }
256 se_tmr_req_cache = kmem_cache_create("se_tmr_cache",
257 sizeof(struct se_tmr_req), __alignof__(struct se_tmr_req),
258 0, NULL);
259 if (!(se_tmr_req_cache)) {
260 printk(KERN_ERR "kmem_cache_create() for struct se_tmr_req"
261 " failed\n");
262 goto out;
263 }
264 se_sess_cache = kmem_cache_create("se_sess_cache",
265 sizeof(struct se_session), __alignof__(struct se_session),
266 0, NULL);
267 if (!(se_sess_cache)) {
268 printk(KERN_ERR "kmem_cache_create() for struct se_session"
269 " failed\n");
270 goto out;
271 }
272 se_ua_cache = kmem_cache_create("se_ua_cache",
273 sizeof(struct se_ua), __alignof__(struct se_ua),
274 0, NULL);
275 if (!(se_ua_cache)) {
276 printk(KERN_ERR "kmem_cache_create() for struct se_ua failed\n");
277 goto out;
278 }
279 se_mem_cache = kmem_cache_create("se_mem_cache",
280 sizeof(struct se_mem), __alignof__(struct se_mem), 0, NULL);
281 if (!(se_mem_cache)) {
282 printk(KERN_ERR "kmem_cache_create() for struct se_mem failed\n");
283 goto out;
284 }
285 t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
286 sizeof(struct t10_pr_registration),
287 __alignof__(struct t10_pr_registration), 0, NULL);
288 if (!(t10_pr_reg_cache)) {
289 printk(KERN_ERR "kmem_cache_create() for struct t10_pr_registration"
290 " failed\n");
291 goto out;
292 }
293 t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
294 sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
295 0, NULL);
296 if (!(t10_alua_lu_gp_cache)) {
297 printk(KERN_ERR "kmem_cache_create() for t10_alua_lu_gp_cache"
298 " failed\n");
299 goto out;
300 }
301 t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
302 sizeof(struct t10_alua_lu_gp_member),
303 __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
304 if (!(t10_alua_lu_gp_mem_cache)) {
305 printk(KERN_ERR "kmem_cache_create() for t10_alua_lu_gp_mem_"
306 "cache failed\n");
307 goto out;
308 }
309 t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
310 sizeof(struct t10_alua_tg_pt_gp),
311 __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
312 if (!(t10_alua_tg_pt_gp_cache)) {
313 printk(KERN_ERR "kmem_cache_create() for t10_alua_tg_pt_gp_"
314 "cache failed\n");
315 goto out;
316 }
317 t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
318 "t10_alua_tg_pt_gp_mem_cache",
319 sizeof(struct t10_alua_tg_pt_gp_member),
320 __alignof__(struct t10_alua_tg_pt_gp_member),
321 0, NULL);
322 if (!(t10_alua_tg_pt_gp_mem_cache)) {
323 printk(KERN_ERR "kmem_cache_create() for t10_alua_tg_pt_gp_"
324 "mem_t failed\n");
325 goto out;
326 }
327
328 se_global = global;
329
330 return 0;
331out:
332 if (se_cmd_cache)
333 kmem_cache_destroy(se_cmd_cache);
334 if (se_tmr_req_cache)
335 kmem_cache_destroy(se_tmr_req_cache);
336 if (se_sess_cache)
337 kmem_cache_destroy(se_sess_cache);
338 if (se_ua_cache)
339 kmem_cache_destroy(se_ua_cache);
340 if (se_mem_cache)
341 kmem_cache_destroy(se_mem_cache);
342 if (t10_pr_reg_cache)
343 kmem_cache_destroy(t10_pr_reg_cache);
344 if (t10_alua_lu_gp_cache)
345 kmem_cache_destroy(t10_alua_lu_gp_cache);
346 if (t10_alua_lu_gp_mem_cache)
347 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
348 if (t10_alua_tg_pt_gp_cache)
349 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
350 if (t10_alua_tg_pt_gp_mem_cache)
351 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
352 kfree(global);
353 return -1;
354}
355
356void release_se_global(void)
357{
358 struct se_global *global;
359
360 global = se_global;
361 if (!(global))
362 return;
363
364 kmem_cache_destroy(se_cmd_cache);
365 kmem_cache_destroy(se_tmr_req_cache);
366 kmem_cache_destroy(se_sess_cache);
367 kmem_cache_destroy(se_ua_cache);
368 kmem_cache_destroy(se_mem_cache);
369 kmem_cache_destroy(t10_pr_reg_cache);
370 kmem_cache_destroy(t10_alua_lu_gp_cache);
371 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
372 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
373 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
374 kfree(global);
375
376 se_global = NULL;
377}
378
e89d15ee
NB		 379/* SCSI statistics table index */
380static struct scsi_index_table scsi_index_table;
381
382/*
383 * Initialize the index table for allocating unique row indexes to various mib
384 * tables.
385 */
386void init_scsi_index_table(void)
387{
388 memset(&scsi_index_table, 0, sizeof(struct scsi_index_table));
389 spin_lock_init(&scsi_index_table.lock);
390}
391
392/*
393 * Allocate a new row index for the entry type specified
394 */
395u32 scsi_get_new_index(scsi_index_t type)
396{
397 u32 new_index;
398
399 if ((type < 0) || (type >= SCSI_INDEX_TYPE_MAX)) {
400 printk(KERN_ERR "Invalid index type %d\n", type);
401 return -EINVAL;
402 }
403
404 spin_lock(&scsi_index_table.lock);
405 new_index = ++scsi_index_table.scsi_mib_index[type];
406 if (new_index == 0)
407 new_index = ++scsi_index_table.scsi_mib_index[type];
408 spin_unlock(&scsi_index_table.lock);
409
410 return new_index;
411}
412
c66ac9db
NB		 413void transport_init_queue_obj(struct se_queue_obj *qobj)
414{
415 atomic_set(&qobj->queue_cnt, 0);
416 INIT_LIST_HEAD(&qobj->qobj_list);
417 init_waitqueue_head(&qobj->thread_wq);
418 spin_lock_init(&qobj->cmd_queue_lock);
419}
420EXPORT_SYMBOL(transport_init_queue_obj);
421
422static int transport_subsystem_reqmods(void)
423{
424 int ret;
425
426 ret = request_module("target_core_iblock");
427 if (ret != 0)
428 printk(KERN_ERR "Unable to load target_core_iblock\n");
429
430 ret = request_module("target_core_file");
431 if (ret != 0)
432 printk(KERN_ERR "Unable to load target_core_file\n");
433
434 ret = request_module("target_core_pscsi");
435 if (ret != 0)
436 printk(KERN_ERR "Unable to load target_core_pscsi\n");
437
438 ret = request_module("target_core_stgt");
439 if (ret != 0)
440 printk(KERN_ERR "Unable to load target_core_stgt\n");
441
442 return 0;
443}
444
445int transport_subsystem_check_init(void)
446{
447 if (se_global->g_sub_api_initialized)
448 return 0;
449 /*
450 * Request the loading of known TCM subsystem plugins..
451 */
452 if (transport_subsystem_reqmods() < 0)
453 return -1;
454
455 se_global->g_sub_api_initialized = 1;
456 return 0;
457}
458
459struct se_session *transport_init_session(void)
460{
461 struct se_session *se_sess;
462
463 se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
464 if (!(se_sess)) {
465 printk(KERN_ERR "Unable to allocate struct se_session from"
466 " se_sess_cache\n");
467 return ERR_PTR(-ENOMEM);
468 }
469 INIT_LIST_HEAD(&se_sess->sess_list);
470 INIT_LIST_HEAD(&se_sess->sess_acl_list);
c66ac9db
NB		 471
472 return se_sess;
473}
474EXPORT_SYMBOL(transport_init_session);
475
476/*
477 * Called with spin_lock_bh(&struct se_portal_group->session_lock called.
478 */
479void __transport_register_session(
480 struct se_portal_group *se_tpg,
481 struct se_node_acl *se_nacl,
482 struct se_session *se_sess,
483 void *fabric_sess_ptr)
484{
485 unsigned char buf[PR_REG_ISID_LEN];
486
487 se_sess->se_tpg = se_tpg;
488 se_sess->fabric_sess_ptr = fabric_sess_ptr;
489 /*
490 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
491 *
492 * Only set for struct se_session's that will actually be moving I/O.
493 * eg: *NOT* discovery sessions.
494 */
495 if (se_nacl) {
496 /*
497 * If the fabric module supports an ISID based TransportID,
498 * save this value in binary from the fabric I_T Nexus now.
499 */
500 if (TPG_TFO(se_tpg)->sess_get_initiator_sid != NULL) {
501 memset(&buf[0], 0, PR_REG_ISID_LEN);
502 TPG_TFO(se_tpg)->sess_get_initiator_sid(se_sess,
503 &buf[0], PR_REG_ISID_LEN);
504 se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
505 }
506 spin_lock_irq(&se_nacl->nacl_sess_lock);
507 /*
508 * The se_nacl->nacl_sess pointer will be set to the
509 * last active I_T Nexus for each struct se_node_acl.
510 */
511 se_nacl->nacl_sess = se_sess;
512
513 list_add_tail(&se_sess->sess_acl_list,
514 &se_nacl->acl_sess_list);
515 spin_unlock_irq(&se_nacl->nacl_sess_lock);
516 }
517 list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
518
519 printk(KERN_INFO "TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
520 TPG_TFO(se_tpg)->get_fabric_name(), se_sess->fabric_sess_ptr);
521}
522EXPORT_SYMBOL(__transport_register_session);
523
524void transport_register_session(
525 struct se_portal_group *se_tpg,
526 struct se_node_acl *se_nacl,
527 struct se_session *se_sess,
528 void *fabric_sess_ptr)
529{
530 spin_lock_bh(&se_tpg->session_lock);
531 __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
532 spin_unlock_bh(&se_tpg->session_lock);
533}
534EXPORT_SYMBOL(transport_register_session);
535
536void transport_deregister_session_configfs(struct se_session *se_sess)
537{
538 struct se_node_acl *se_nacl;
539
540 /*
541 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
542 */
543 se_nacl = se_sess->se_node_acl;
544 if ((se_nacl)) {
545 spin_lock_irq(&se_nacl->nacl_sess_lock);
546 list_del(&se_sess->sess_acl_list);
547 /*
548 * If the session list is empty, then clear the pointer.
549 * Otherwise, set the struct se_session pointer from the tail
550 * element of the per struct se_node_acl active session list.
551 */
552 if (list_empty(&se_nacl->acl_sess_list))
553 se_nacl->nacl_sess = NULL;
554 else {
555 se_nacl->nacl_sess = container_of(
556 se_nacl->acl_sess_list.prev,
557 struct se_session, sess_acl_list);
558 }
559 spin_unlock_irq(&se_nacl->nacl_sess_lock);
560 }
561}
562EXPORT_SYMBOL(transport_deregister_session_configfs);
563
564void transport_free_session(struct se_session *se_sess)
565{
566 kmem_cache_free(se_sess_cache, se_sess);
567}
568EXPORT_SYMBOL(transport_free_session);
569
570void transport_deregister_session(struct se_session *se_sess)
571{
572 struct se_portal_group *se_tpg = se_sess->se_tpg;
573 struct se_node_acl *se_nacl;
574
575 if (!(se_tpg)) {
576 transport_free_session(se_sess);
577 return;
578 }
c66ac9db
NB		 579
580 spin_lock_bh(&se_tpg->session_lock);
581 list_del(&se_sess->sess_list);
582 se_sess->se_tpg = NULL;
583 se_sess->fabric_sess_ptr = NULL;
584 spin_unlock_bh(&se_tpg->session_lock);
585
586 /*
587 * Determine if we need to do extra work for this initiator node's
588 * struct se_node_acl if it had been previously dynamically generated.
589 */
590 se_nacl = se_sess->se_node_acl;
591 if ((se_nacl)) {
592 spin_lock_bh(&se_tpg->acl_node_lock);
593 if (se_nacl->dynamic_node_acl) {
594 if (!(TPG_TFO(se_tpg)->tpg_check_demo_mode_cache(
595 se_tpg))) {
596 list_del(&se_nacl->acl_list);
597 se_tpg->num_node_acls--;
598 spin_unlock_bh(&se_tpg->acl_node_lock);
599
600 core_tpg_wait_for_nacl_pr_ref(se_nacl);
c66ac9db
NB		 601 core_free_device_list_for_node(se_nacl, se_tpg);
602 TPG_TFO(se_tpg)->tpg_release_fabric_acl(se_tpg,
603 se_nacl);
604 spin_lock_bh(&se_tpg->acl_node_lock);
605 }
606 }
607 spin_unlock_bh(&se_tpg->acl_node_lock);
608 }
609
610 transport_free_session(se_sess);
611
612 printk(KERN_INFO "TARGET_CORE[%s]: Deregistered fabric_sess\n",
613 TPG_TFO(se_tpg)->get_fabric_name());
614}
615EXPORT_SYMBOL(transport_deregister_session);
616
617/*
618 * Called with T_TASK(cmd)->t_state_lock held.
619 */
620static void transport_all_task_dev_remove_state(struct se_cmd *cmd)
621{
622 struct se_device *dev;
623 struct se_task *task;
624 unsigned long flags;
625
626 if (!T_TASK(cmd))
627 return;
628
629 list_for_each_entry(task, &T_TASK(cmd)->t_task_list, t_list) {
630 dev = task->se_dev;
631 if (!(dev))
632 continue;
633
634 if (atomic_read(&task->task_active))
635 continue;
636
637 if (!(atomic_read(&task->task_state_active)))
638 continue;
639
640 spin_lock_irqsave(&dev->execute_task_lock, flags);
641 list_del(&task->t_state_list);
642 DEBUG_TSTATE("Removed ITT: 0x%08x dev: %p task[%p]\n",
643 CMD_TFO(cmd)->tfo_get_task_tag(cmd), dev, task);
644 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
645
646 atomic_set(&task->task_state_active, 0);
647 atomic_dec(&T_TASK(cmd)->t_task_cdbs_ex_left);
648 }
649}
650
651/* transport_cmd_check_stop():
652 *
653 * 'transport_off = 1' determines if t_transport_active should be cleared.
654 * 'transport_off = 2' determines if task_dev_state should be removed.
655 *
656 * A non-zero u8 t_state sets cmd->t_state.
657 * Returns 1 when command is stopped, else 0.
658 */
659static int transport_cmd_check_stop(
660 struct se_cmd *cmd,
661 int transport_off,
662 u8 t_state)
663{
664 unsigned long flags;
665
666 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
667 /*
668 * Determine if IOCTL context caller in requesting the stopping of this
669 * command for LUN shutdown purposes.
670 */
671 if (atomic_read(&T_TASK(cmd)->transport_lun_stop)) {
672 DEBUG_CS("%s:%d atomic_read(&T_TASK(cmd)->transport_lun_stop)"
673 " == TRUE for ITT: 0x%08x\n", __func__, __LINE__,
674 CMD_TFO(cmd)->get_task_tag(cmd));
675
676 cmd->deferred_t_state = cmd->t_state;
677 cmd->t_state = TRANSPORT_DEFERRED_CMD;
678 atomic_set(&T_TASK(cmd)->t_transport_active, 0);
679 if (transport_off == 2)
680 transport_all_task_dev_remove_state(cmd);
681 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
682
683 complete(&T_TASK(cmd)->transport_lun_stop_comp);
684 return 1;
685 }
686 /*
687 * Determine if frontend context caller is requesting the stopping of
688 * this command for frontend excpections.
689 */
690 if (atomic_read(&T_TASK(cmd)->t_transport_stop)) {
691 DEBUG_CS("%s:%d atomic_read(&T_TASK(cmd)->t_transport_stop) =="
692 " TRUE for ITT: 0x%08x\n", __func__, __LINE__,
693 CMD_TFO(cmd)->get_task_tag(cmd));
694
695 cmd->deferred_t_state = cmd->t_state;
696 cmd->t_state = TRANSPORT_DEFERRED_CMD;
697 if (transport_off == 2)
698 transport_all_task_dev_remove_state(cmd);
699
700 /*
701 * Clear struct se_cmd->se_lun before the transport_off == 2 handoff
702 * to FE.
703 */
704 if (transport_off == 2)
705 cmd->se_lun = NULL;
706 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
707
708 complete(&T_TASK(cmd)->t_transport_stop_comp);
709 return 1;
710 }
711 if (transport_off) {
712 atomic_set(&T_TASK(cmd)->t_transport_active, 0);
713 if (transport_off == 2) {
714 transport_all_task_dev_remove_state(cmd);
715 /*
716 * Clear struct se_cmd->se_lun before the transport_off == 2
717 * handoff to fabric module.
718 */
719 cmd->se_lun = NULL;
720 /*
721 * Some fabric modules like tcm_loop can release
25985edc 722 * their internally allocated I/O reference now and
c66ac9db
NB		 723 * struct se_cmd now.
724 */
725 if (CMD_TFO(cmd)->check_stop_free != NULL) {
726 spin_unlock_irqrestore(
727 &T_TASK(cmd)->t_state_lock, flags);
728
729 CMD_TFO(cmd)->check_stop_free(cmd);
730 return 1;
731 }
732 }
733 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
734
735 return 0;
736 } else if (t_state)
737 cmd->t_state = t_state;
738 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
739
740 return 0;
741}
742
743static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
744{
745 return transport_cmd_check_stop(cmd, 2, 0);
746}
747
748static void transport_lun_remove_cmd(struct se_cmd *cmd)
749{
750 struct se_lun *lun = SE_LUN(cmd);
751 unsigned long flags;
752
753 if (!lun)
754 return;
755
756 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
757 if (!(atomic_read(&T_TASK(cmd)->transport_dev_active))) {
758 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
759 goto check_lun;
760 }
761 atomic_set(&T_TASK(cmd)->transport_dev_active, 0);
762 transport_all_task_dev_remove_state(cmd);
763 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
764
c66ac9db
NB		 765
766check_lun:
767 spin_lock_irqsave(&lun->lun_cmd_lock, flags);
768 if (atomic_read(&T_TASK(cmd)->transport_lun_active)) {
769 list_del(&cmd->se_lun_list);
770 atomic_set(&T_TASK(cmd)->transport_lun_active, 0);
771#if 0
772 printk(KERN_INFO "Removed ITT: 0x%08x from LUN LIST[%d]\n"
773 CMD_TFO(cmd)->get_task_tag(cmd), lun->unpacked_lun);
774#endif
775 }
776 spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
777}
778
779void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
780{
781 transport_remove_cmd_from_queue(cmd, SE_DEV(cmd)->dev_queue_obj);
782 transport_lun_remove_cmd(cmd);
783
784 if (transport_cmd_check_stop_to_fabric(cmd))
785 return;
786 if (remove)
787 transport_generic_remove(cmd, 0, 0);
788}
789
790void transport_cmd_finish_abort_tmr(struct se_cmd *cmd)
791{
792 transport_remove_cmd_from_queue(cmd, SE_DEV(cmd)->dev_queue_obj);
793
794 if (transport_cmd_check_stop_to_fabric(cmd))
795 return;
796
797 transport_generic_remove(cmd, 0, 0);
798}
799
800static int transport_add_cmd_to_queue(
801 struct se_cmd *cmd,
802 int t_state)
803{
804 struct se_device *dev = cmd->se_dev;
805 struct se_queue_obj *qobj = dev->dev_queue_obj;
806 struct se_queue_req *qr;
807 unsigned long flags;
808
809 qr = kzalloc(sizeof(struct se_queue_req), GFP_ATOMIC);
810 if (!(qr)) {
811 printk(KERN_ERR "Unable to allocate memory for"
812 " struct se_queue_req\n");
813 return -1;
814 }
815 INIT_LIST_HEAD(&qr->qr_list);
816
817 qr->cmd = (void *)cmd;
818 qr->state = t_state;
819
820 if (t_state) {
821 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
822 cmd->t_state = t_state;
823 atomic_set(&T_TASK(cmd)->t_transport_active, 1);
824 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
825 }
826
827 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
828 list_add_tail(&qr->qr_list, &qobj->qobj_list);
829 atomic_inc(&T_TASK(cmd)->t_transport_queue_active);
830 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
831
832 atomic_inc(&qobj->queue_cnt);
833 wake_up_interruptible(&qobj->thread_wq);
834 return 0;
835}
836
837/*
838 * Called with struct se_queue_obj->cmd_queue_lock held.
839 */
840static struct se_queue_req *
841__transport_get_qr_from_queue(struct se_queue_obj *qobj)
842{
843 struct se_cmd *cmd;
844 struct se_queue_req *qr = NULL;
845
846 if (list_empty(&qobj->qobj_list))
847 return NULL;
848
849 list_for_each_entry(qr, &qobj->qobj_list, qr_list)
850 break;
851
852 if (qr->cmd) {
853 cmd = (struct se_cmd *)qr->cmd;
854 atomic_dec(&T_TASK(cmd)->t_transport_queue_active);
855 }
856 list_del(&qr->qr_list);
857 atomic_dec(&qobj->queue_cnt);
858
859 return qr;
860}
861
862static struct se_queue_req *
863transport_get_qr_from_queue(struct se_queue_obj *qobj)
864{
865 struct se_cmd *cmd;
866 struct se_queue_req *qr;
867 unsigned long flags;
868
869 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
870 if (list_empty(&qobj->qobj_list)) {
871 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
872 return NULL;
873 }
874
875 list_for_each_entry(qr, &qobj->qobj_list, qr_list)
876 break;
877
878 if (qr->cmd) {
879 cmd = (struct se_cmd *)qr->cmd;
880 atomic_dec(&T_TASK(cmd)->t_transport_queue_active);
881 }
882 list_del(&qr->qr_list);
883 atomic_dec(&qobj->queue_cnt);
884 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
885
886 return qr;
887}
888
889static void transport_remove_cmd_from_queue(struct se_cmd *cmd,
890 struct se_queue_obj *qobj)
891{
892 struct se_cmd *q_cmd;
893 struct se_queue_req *qr = NULL, *qr_p = NULL;
894 unsigned long flags;
895
896 spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
897 if (!(atomic_read(&T_TASK(cmd)->t_transport_queue_active))) {
898 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
899 return;
900 }
901
902 list_for_each_entry_safe(qr, qr_p, &qobj->qobj_list, qr_list) {
903 q_cmd = (struct se_cmd *)qr->cmd;
904 if (q_cmd != cmd)
905 continue;
906
907 atomic_dec(&T_TASK(q_cmd)->t_transport_queue_active);
908 atomic_dec(&qobj->queue_cnt);
909 list_del(&qr->qr_list);
910 kfree(qr);
911 }
912 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
913
914 if (atomic_read(&T_TASK(cmd)->t_transport_queue_active)) {
915 printk(KERN_ERR "ITT: 0x%08x t_transport_queue_active: %d\n",
916 CMD_TFO(cmd)->get_task_tag(cmd),
917 atomic_read(&T_TASK(cmd)->t_transport_queue_active));
918 }
919}
920
921/*
922 * Completion function used by TCM subsystem plugins (such as FILEIO)
923 * for queueing up response from struct se_subsystem_api->do_task()
924 */
925void transport_complete_sync_cache(struct se_cmd *cmd, int good)
926{
927 struct se_task *task = list_entry(T_TASK(cmd)->t_task_list.next,
928 struct se_task, t_list);
929
930 if (good) {
931 cmd->scsi_status = SAM_STAT_GOOD;
932 task->task_scsi_status = GOOD;
933 } else {
934 task->task_scsi_status = SAM_STAT_CHECK_CONDITION;
935 task->task_error_status = PYX_TRANSPORT_ILLEGAL_REQUEST;
936 TASK_CMD(task)->transport_error_status =
937 PYX_TRANSPORT_ILLEGAL_REQUEST;
938 }
939
940 transport_complete_task(task, good);
941}
942EXPORT_SYMBOL(transport_complete_sync_cache);
943
944/* transport_complete_task():
945 *
946 * Called from interrupt and non interrupt context depending
947 * on the transport plugin.
948 */
949void transport_complete_task(struct se_task *task, int success)
950{
951 struct se_cmd *cmd = TASK_CMD(task);
952 struct se_device *dev = task->se_dev;
953 int t_state;
954 unsigned long flags;
955#if 0
956 printk(KERN_INFO "task: %p CDB: 0x%02x obj_ptr: %p\n", task,
957 T_TASK(cmd)->t_task_cdb[0], dev);
958#endif
959 if (dev) {
960 spin_lock_irqsave(&SE_HBA(dev)->hba_queue_lock, flags);
961 atomic_inc(&dev->depth_left);
962 atomic_inc(&SE_HBA(dev)->left_queue_depth);
963 spin_unlock_irqrestore(&SE_HBA(dev)->hba_queue_lock, flags);
964 }
965
966 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
967 atomic_set(&task->task_active, 0);
968
969 /*
970 * See if any sense data exists, if so set the TASK_SENSE flag.
971 * Also check for any other post completion work that needs to be
972 * done by the plugins.
973 */
974 if (dev && dev->transport->transport_complete) {
975 if (dev->transport->transport_complete(task) != 0) {
976 cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
977 task->task_sense = 1;
978 success = 1;
979 }
980 }
981
982 /*
983 * See if we are waiting for outstanding struct se_task
984 * to complete for an exception condition
985 */
986 if (atomic_read(&task->task_stop)) {
987 /*
988 * Decrement T_TASK(cmd)->t_se_count if this task had
989 * previously thrown its timeout exception handler.
990 */
991 if (atomic_read(&task->task_timeout)) {
992 atomic_dec(&T_TASK(cmd)->t_se_count);
993 atomic_set(&task->task_timeout, 0);
994 }
995 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
996
997 complete(&task->task_stop_comp);
998 return;
999 }
1000 /*
1001 * If the task's timeout handler has fired, use the t_task_cdbs_timeout
1002 * left counter to determine when the struct se_cmd is ready to be queued to
1003 * the processing thread.
1004 */
1005 if (atomic_read(&task->task_timeout)) {
1006 if (!(atomic_dec_and_test(
1007 &T_TASK(cmd)->t_task_cdbs_timeout_left))) {
1008 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
1009 flags);
1010 return;
1011 }
1012 t_state = TRANSPORT_COMPLETE_TIMEOUT;
1013 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
1014
1015 transport_add_cmd_to_queue(cmd, t_state);
1016 return;
1017 }
1018 atomic_dec(&T_TASK(cmd)->t_task_cdbs_timeout_left);
1019
1020 /*
1021 * Decrement the outstanding t_task_cdbs_left count. The last
1022 * struct se_task from struct se_cmd will complete itself into the
1023 * device queue depending upon int success.
1024 */
1025 if (!(atomic_dec_and_test(&T_TASK(cmd)->t_task_cdbs_left))) {
1026 if (!success)
1027 T_TASK(cmd)->t_tasks_failed = 1;
1028
1029 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
1030 return;
1031 }
1032
1033 if (!success || T_TASK(cmd)->t_tasks_failed) {
1034 t_state = TRANSPORT_COMPLETE_FAILURE;
1035 if (!task->task_error_status) {
1036 task->task_error_status =
1037 PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
1038 cmd->transport_error_status =
1039 PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
1040 }
1041 } else {
1042 atomic_set(&T_TASK(cmd)->t_transport_complete, 1);
1043 t_state = TRANSPORT_COMPLETE_OK;
1044 }
1045 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
1046
1047 transport_add_cmd_to_queue(cmd, t_state);
1048}
1049EXPORT_SYMBOL(transport_complete_task);
1050
1051/*
1052 * Called by transport_add_tasks_from_cmd() once a struct se_cmd's
1053 * struct se_task list are ready to be added to the active execution list
1054 * struct se_device
1055
1056 * Called with se_dev_t->execute_task_lock called.
1057 */
1058static inline int transport_add_task_check_sam_attr(
1059 struct se_task *task,
1060 struct se_task *task_prev,
1061 struct se_device *dev)
1062{
1063 /*
1064 * No SAM Task attribute emulation enabled, add to tail of
1065 * execution queue
1066 */
1067 if (dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) {
1068 list_add_tail(&task->t_execute_list, &dev->execute_task_list);
1069 return 0;
1070 }
1071 /*
1072 * HEAD_OF_QUEUE attribute for received CDB, which means
1073 * the first task that is associated with a struct se_cmd goes to
1074 * head of the struct se_device->execute_task_list, and task_prev
1075 * after that for each subsequent task
1076 */
1077 if (task->task_se_cmd->sam_task_attr == TASK_ATTR_HOQ) {
1078 list_add(&task->t_execute_list,
1079 (task_prev != NULL) ?
1080 &task_prev->t_execute_list :
1081 &dev->execute_task_list);
1082
1083 DEBUG_STA("Set HEAD_OF_QUEUE for task CDB: 0x%02x"
1084 " in execution queue\n",
1085 T_TASK(task->task_se_cmd)->t_task_cdb[0]);
1086 return 1;
1087 }
1088 /*
1089 * For ORDERED, SIMPLE or UNTAGGED attribute tasks once they have been
1090 * transitioned from Dermant -> Active state, and are added to the end
1091 * of the struct se_device->execute_task_list
1092 */
1093 list_add_tail(&task->t_execute_list, &dev->execute_task_list);
1094 return 0;
1095}
1096
1097/* __transport_add_task_to_execute_queue():
1098 *
1099 * Called with se_dev_t->execute_task_lock called.
1100 */
1101static void __transport_add_task_to_execute_queue(
1102 struct se_task *task,
1103 struct se_task *task_prev,
1104 struct se_device *dev)
1105{
1106 int head_of_queue;
1107
1108 head_of_queue = transport_add_task_check_sam_attr(task, task_prev, dev);
1109 atomic_inc(&dev->execute_tasks);
1110
1111 if (atomic_read(&task->task_state_active))
1112 return;
1113 /*
1114 * Determine if this task needs to go to HEAD_OF_QUEUE for the
1115 * state list as well. Running with SAM Task Attribute emulation
1116 * will always return head_of_queue == 0 here
1117 */
1118 if (head_of_queue)
1119 list_add(&task->t_state_list, (task_prev) ?
1120 &task_prev->t_state_list :
1121 &dev->state_task_list);
1122 else
1123 list_add_tail(&task->t_state_list, &dev->state_task_list);
1124
1125 atomic_set(&task->task_state_active, 1);
1126
1127 DEBUG_TSTATE("Added ITT: 0x%08x task[%p] to dev: %p\n",
1128 CMD_TFO(task->task_se_cmd)->get_task_tag(task->task_se_cmd),
1129 task, dev);
1130}
1131
1132static void transport_add_tasks_to_state_queue(struct se_cmd *cmd)
1133{
1134 struct se_device *dev;
1135 struct se_task *task;
1136 unsigned long flags;
1137
1138 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
1139 list_for_each_entry(task, &T_TASK(cmd)->t_task_list, t_list) {
1140 dev = task->se_dev;
1141
1142 if (atomic_read(&task->task_state_active))
1143 continue;
1144
1145 spin_lock(&dev->execute_task_lock);
1146 list_add_tail(&task->t_state_list, &dev->state_task_list);
1147 atomic_set(&task->task_state_active, 1);
1148
1149 DEBUG_TSTATE("Added ITT: 0x%08x task[%p] to dev: %p\n",
1150 CMD_TFO(task->task_se_cmd)->get_task_tag(
1151 task->task_se_cmd), task, dev);
1152
1153 spin_unlock(&dev->execute_task_lock);
1154 }
1155 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
1156}
1157
1158static void transport_add_tasks_from_cmd(struct se_cmd *cmd)
1159{
1160 struct se_device *dev = SE_DEV(cmd);
1161 struct se_task *task, *task_prev = NULL;
1162 unsigned long flags;
1163
1164 spin_lock_irqsave(&dev->execute_task_lock, flags);
1165 list_for_each_entry(task, &T_TASK(cmd)->t_task_list, t_list) {
1166 if (atomic_read(&task->task_execute_queue))
1167 continue;
1168 /*
1169 * __transport_add_task_to_execute_queue() handles the
1170 * SAM Task Attribute emulation if enabled
1171 */
1172 __transport_add_task_to_execute_queue(task, task_prev, dev);
1173 atomic_set(&task->task_execute_queue, 1);
1174 task_prev = task;
1175 }
1176 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
1177
1178 return;
1179}
1180
1181/* transport_get_task_from_execute_queue():
1182 *
1183 * Called with dev->execute_task_lock held.
1184 */
1185static struct se_task *
1186transport_get_task_from_execute_queue(struct se_device *dev)
1187{
1188 struct se_task *task;
1189
1190 if (list_empty(&dev->execute_task_list))
1191 return NULL;
1192
1193 list_for_each_entry(task, &dev->execute_task_list, t_execute_list)
1194 break;
1195
1196 list_del(&task->t_execute_list);
1197 atomic_dec(&dev->execute_tasks);
1198
1199 return task;
1200}
1201
1202/* transport_remove_task_from_execute_queue():
1203 *
1204 *
1205 */
52208ae3 1206void transport_remove_task_from_execute_queue(
c66ac9db
NB		 1207 struct se_task *task,
1208 struct se_device *dev)
1209{
1210 unsigned long flags;
1211
1212 spin_lock_irqsave(&dev->execute_task_lock, flags);
1213 list_del(&task->t_execute_list);
1214 atomic_dec(&dev->execute_tasks);
1215 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
1216}
1217
1218unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
1219{
1220 switch (cmd->data_direction) {
1221 case DMA_NONE:
1222 return "NONE";
1223 case DMA_FROM_DEVICE:
1224 return "READ";
1225 case DMA_TO_DEVICE:
1226 return "WRITE";
1227 case DMA_BIDIRECTIONAL:
1228 return "BIDI";
1229 default:
1230 break;
1231 }
1232
1233 return "UNKNOWN";
1234}
1235
1236void transport_dump_dev_state(
1237 struct se_device *dev,
1238 char *b,
1239 int *bl)
1240{
1241 *bl += sprintf(b + *bl, "Status: ");
1242 switch (dev->dev_status) {
1243 case TRANSPORT_DEVICE_ACTIVATED:
1244 *bl += sprintf(b + *bl, "ACTIVATED");
1245 break;
1246 case TRANSPORT_DEVICE_DEACTIVATED:
1247 *bl += sprintf(b + *bl, "DEACTIVATED");
1248 break;
1249 case TRANSPORT_DEVICE_SHUTDOWN:
1250 *bl += sprintf(b + *bl, "SHUTDOWN");
1251 break;
1252 case TRANSPORT_DEVICE_OFFLINE_ACTIVATED:
1253 case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED:
1254 *bl += sprintf(b + *bl, "OFFLINE");
1255 break;
1256 default:
1257 *bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status);
1258 break;
1259 }
1260
1261 *bl += sprintf(b + *bl, " Execute/Left/Max Queue Depth: %d/%d/%d",
1262 atomic_read(&dev->execute_tasks), atomic_read(&dev->depth_left),
1263 dev->queue_depth);
1264 *bl += sprintf(b + *bl, " SectorSize: %u MaxSectors: %u\n",
1265 DEV_ATTRIB(dev)->block_size, DEV_ATTRIB(dev)->max_sectors);
1266 *bl += sprintf(b + *bl, " ");
1267}
1268
1269/* transport_release_all_cmds():
1270 *
1271 *
1272 */
1273static void transport_release_all_cmds(struct se_device *dev)
1274{
1275 struct se_cmd *cmd = NULL;
1276 struct se_queue_req *qr = NULL, *qr_p = NULL;
1277 int bug_out = 0, t_state;
1278 unsigned long flags;
1279
1280 spin_lock_irqsave(&dev->dev_queue_obj->cmd_queue_lock, flags);
1281 list_for_each_entry_safe(qr, qr_p, &dev->dev_queue_obj->qobj_list,
1282 qr_list) {
1283
1284 cmd = (struct se_cmd *)qr->cmd;
1285 t_state = qr->state;
1286 list_del(&qr->qr_list);
1287 kfree(qr);
1288 spin_unlock_irqrestore(&dev->dev_queue_obj->cmd_queue_lock,
1289 flags);
1290
1291 printk(KERN_ERR "Releasing ITT: 0x%08x, i_state: %u,"
1292 " t_state: %u directly\n",
1293 CMD_TFO(cmd)->get_task_tag(cmd),
1294 CMD_TFO(cmd)->get_cmd_state(cmd), t_state);
1295
1296 transport_release_fe_cmd(cmd);
1297 bug_out = 1;
1298
1299 spin_lock_irqsave(&dev->dev_queue_obj->cmd_queue_lock, flags);
1300 }
1301 spin_unlock_irqrestore(&dev->dev_queue_obj->cmd_queue_lock, flags);
1302#if 0
1303 if (bug_out)
1304 BUG();
1305#endif
1306}
1307
1308void transport_dump_vpd_proto_id(
1309 struct t10_vpd *vpd,
1310 unsigned char *p_buf,
1311 int p_buf_len)
1312{
1313 unsigned char buf[VPD_TMP_BUF_SIZE];
1314 int len;
1315
1316 memset(buf, 0, VPD_TMP_BUF_SIZE);
1317 len = sprintf(buf, "T10 VPD Protocol Identifier: ");
1318
1319 switch (vpd->protocol_identifier) {
1320 case 0x00:
1321 sprintf(buf+len, "Fibre Channel\n");
1322 break;
1323 case 0x10:
1324 sprintf(buf+len, "Parallel SCSI\n");
1325 break;
1326 case 0x20:
1327 sprintf(buf+len, "SSA\n");
1328 break;
1329 case 0x30:
1330 sprintf(buf+len, "IEEE 1394\n");
1331 break;
1332 case 0x40:
1333 sprintf(buf+len, "SCSI Remote Direct Memory Access"
1334 " Protocol\n");
1335 break;
1336 case 0x50:
1337 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
1338 break;
1339 case 0x60:
1340 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
1341 break;
1342 case 0x70:
1343 sprintf(buf+len, "Automation/Drive Interface Transport"
1344 " Protocol\n");
1345 break;
1346 case 0x80:
1347 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
1348 break;
1349 default:
1350 sprintf(buf+len, "Unknown 0x%02x\n",
1351 vpd->protocol_identifier);
1352 break;
1353 }
1354
1355 if (p_buf)
1356 strncpy(p_buf, buf, p_buf_len);
1357 else
1358 printk(KERN_INFO "%s", buf);
1359}
1360
1361void
1362transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
1363{
1364 /*
1365 * Check if the Protocol Identifier Valid (PIV) bit is set..
1366 *
1367 * from spc3r23.pdf section 7.5.1
1368 */
1369 if (page_83[1] & 0x80) {
1370 vpd->protocol_identifier = (page_83[0] & 0xf0);
1371 vpd->protocol_identifier_set = 1;
1372 transport_dump_vpd_proto_id(vpd, NULL, 0);
1373 }
1374}
1375EXPORT_SYMBOL(transport_set_vpd_proto_id);
1376
1377int transport_dump_vpd_assoc(
1378 struct t10_vpd *vpd,
1379 unsigned char *p_buf,
1380 int p_buf_len)
1381{
1382 unsigned char buf[VPD_TMP_BUF_SIZE];
1383 int ret = 0, len;
1384
1385 memset(buf, 0, VPD_TMP_BUF_SIZE);
1386 len = sprintf(buf, "T10 VPD Identifier Association: ");
1387
1388 switch (vpd->association) {
1389 case 0x00:
1390 sprintf(buf+len, "addressed logical unit\n");
1391 break;
1392 case 0x10:
1393 sprintf(buf+len, "target port\n");
1394 break;
1395 case 0x20:
1396 sprintf(buf+len, "SCSI target device\n");
1397 break;
1398 default:
1399 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1400 ret = -1;
1401 break;
1402 }
1403
1404 if (p_buf)
1405 strncpy(p_buf, buf, p_buf_len);
1406 else
1407 printk("%s", buf);
1408
1409 return ret;
1410}
1411
1412int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1413{
1414 /*
1415 * The VPD identification association..
1416 *
1417 * from spc3r23.pdf Section 7.6.3.1 Table 297
1418 */
1419 vpd->association = (page_83[1] & 0x30);
1420 return transport_dump_vpd_assoc(vpd, NULL, 0);
1421}
1422EXPORT_SYMBOL(transport_set_vpd_assoc);
1423
1424int transport_dump_vpd_ident_type(
1425 struct t10_vpd *vpd,
1426 unsigned char *p_buf,
1427 int p_buf_len)
1428{
1429 unsigned char buf[VPD_TMP_BUF_SIZE];
1430 int ret = 0, len;
1431
1432 memset(buf, 0, VPD_TMP_BUF_SIZE);
1433 len = sprintf(buf, "T10 VPD Identifier Type: ");
1434
1435 switch (vpd->device_identifier_type) {
1436 case 0x00:
1437 sprintf(buf+len, "Vendor specific\n");
1438 break;
1439 case 0x01:
1440 sprintf(buf+len, "T10 Vendor ID based\n");
1441 break;
1442 case 0x02:
1443 sprintf(buf+len, "EUI-64 based\n");
1444 break;
1445 case 0x03:
1446 sprintf(buf+len, "NAA\n");
1447 break;
1448 case 0x04:
1449 sprintf(buf+len, "Relative target port identifier\n");
1450 break;
1451 case 0x08:
1452 sprintf(buf+len, "SCSI name string\n");
1453 break;
1454 default:
1455 sprintf(buf+len, "Unsupported: 0x%02x\n",
1456 vpd->device_identifier_type);
1457 ret = -1;
1458 break;
1459 }
1460
1461 if (p_buf)
1462 strncpy(p_buf, buf, p_buf_len);
1463 else
1464 printk("%s", buf);
1465
1466 return ret;
1467}
1468
1469int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1470{
1471 /*
1472 * The VPD identifier type..
1473 *
1474 * from spc3r23.pdf Section 7.6.3.1 Table 298
1475 */
1476 vpd->device_identifier_type = (page_83[1] & 0x0f);
1477 return transport_dump_vpd_ident_type(vpd, NULL, 0);
1478}
1479EXPORT_SYMBOL(transport_set_vpd_ident_type);
1480
1481int transport_dump_vpd_ident(
1482 struct t10_vpd *vpd,
1483 unsigned char *p_buf,
1484 int p_buf_len)
1485{
1486 unsigned char buf[VPD_TMP_BUF_SIZE];
1487 int ret = 0;
1488
1489 memset(buf, 0, VPD_TMP_BUF_SIZE);
1490
1491 switch (vpd->device_identifier_code_set) {
1492 case 0x01: /* Binary */
1493 sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
1494 &vpd->device_identifier[0]);
1495 break;
1496 case 0x02: /* ASCII */
1497 sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
1498 &vpd->device_identifier[0]);
1499 break;
1500 case 0x03: /* UTF-8 */
1501 sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
1502 &vpd->device_identifier[0]);
1503 break;
1504 default:
1505 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1506 " 0x%02x", vpd->device_identifier_code_set);
1507 ret = -1;
1508 break;
1509 }
1510
1511 if (p_buf)
1512 strncpy(p_buf, buf, p_buf_len);
1513 else
1514 printk("%s", buf);
1515
1516 return ret;
1517}
1518
1519int
1520transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1521{
1522 static const char hex_str[] = "0123456789abcdef";
1523 int j = 0, i = 4; /* offset to start of the identifer */
1524
1525 /*
1526 * The VPD Code Set (encoding)
1527 *
1528 * from spc3r23.pdf Section 7.6.3.1 Table 296
1529 */
1530 vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1531 switch (vpd->device_identifier_code_set) {
1532 case 0x01: /* Binary */
1533 vpd->device_identifier[j++] =
1534 hex_str[vpd->device_identifier_type];
1535 while (i < (4 + page_83[3])) {
1536 vpd->device_identifier[j++] =
1537 hex_str[(page_83[i] & 0xf0) >> 4];
1538 vpd->device_identifier[j++] =
1539 hex_str[page_83[i] & 0x0f];
1540 i++;
1541 }
1542 break;
1543 case 0x02: /* ASCII */
1544 case 0x03: /* UTF-8 */
1545 while (i < (4 + page_83[3]))
1546 vpd->device_identifier[j++] = page_83[i++];
1547 break;
1548 default:
1549 break;
1550 }
1551
1552 return transport_dump_vpd_ident(vpd, NULL, 0);
1553}
1554EXPORT_SYMBOL(transport_set_vpd_ident);
1555
1556static void core_setup_task_attr_emulation(struct se_device *dev)
1557{
1558 /*
1559 * If this device is from Target_Core_Mod/pSCSI, disable the
1560 * SAM Task Attribute emulation.
1561 *
1562 * This is currently not available in upsream Linux/SCSI Target
1563 * mode code, and is assumed to be disabled while using TCM/pSCSI.
1564 */
1565 if (TRANSPORT(dev)->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) {
1566 dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH;
1567 return;
1568 }
1569
1570 dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED;
1571 DEBUG_STA("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x"
1572 " device\n", TRANSPORT(dev)->name,
1573 TRANSPORT(dev)->get_device_rev(dev));
1574}
1575
1576static void scsi_dump_inquiry(struct se_device *dev)
1577{
1578 struct t10_wwn *wwn = DEV_T10_WWN(dev);
1579 int i, device_type;
1580 /*
1581 * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer
1582 */
1583 printk(" Vendor: ");
1584 for (i = 0; i < 8; i++)
1585 if (wwn->vendor[i] >= 0x20)
1586 printk("%c", wwn->vendor[i]);
1587 else
1588 printk(" ");
1589
1590 printk(" Model: ");
1591 for (i = 0; i < 16; i++)
1592 if (wwn->model[i] >= 0x20)
1593 printk("%c", wwn->model[i]);
1594 else
1595 printk(" ");
1596
1597 printk(" Revision: ");
1598 for (i = 0; i < 4; i++)
1599 if (wwn->revision[i] >= 0x20)
1600 printk("%c", wwn->revision[i]);
1601 else
1602 printk(" ");
1603
1604 printk("\n");
1605
1606 device_type = TRANSPORT(dev)->get_device_type(dev);
1607 printk(" Type: %s ", scsi_device_type(device_type));
1608 printk(" ANSI SCSI revision: %02x\n",
1609 TRANSPORT(dev)->get_device_rev(dev));
1610}
1611
1612struct se_device *transport_add_device_to_core_hba(
1613 struct se_hba *hba,
1614 struct se_subsystem_api *transport,
1615 struct se_subsystem_dev *se_dev,
1616 u32 device_flags,
1617 void *transport_dev,
1618 struct se_dev_limits *dev_limits,
1619 const char *inquiry_prod,
1620 const char *inquiry_rev)
1621{
12a18bdc 1622 int force_pt;
c66ac9db
NB		 1623 struct se_device *dev;
1624
1625 dev = kzalloc(sizeof(struct se_device), GFP_KERNEL);
1626 if (!(dev)) {
1627 printk(KERN_ERR "Unable to allocate memory for se_dev_t\n");
1628 return NULL;
1629 }
1630 dev->dev_queue_obj = kzalloc(sizeof(struct se_queue_obj), GFP_KERNEL);
1631 if (!(dev->dev_queue_obj)) {
1632 printk(KERN_ERR "Unable to allocate memory for"
1633 " dev->dev_queue_obj\n");
1634 kfree(dev);
1635 return NULL;
1636 }
1637 transport_init_queue_obj(dev->dev_queue_obj);
1638
1639 dev->dev_status_queue_obj = kzalloc(sizeof(struct se_queue_obj),
1640 GFP_KERNEL);
1641 if (!(dev->dev_status_queue_obj)) {
1642 printk(KERN_ERR "Unable to allocate memory for"
1643 " dev->dev_status_queue_obj\n");
1644 kfree(dev->dev_queue_obj);
1645 kfree(dev);
1646 return NULL;
1647 }
1648 transport_init_queue_obj(dev->dev_status_queue_obj);
1649
1650 dev->dev_flags = device_flags;
1651 dev->dev_status |= TRANSPORT_DEVICE_DEACTIVATED;
1652 dev->dev_ptr = (void *) transport_dev;
1653 dev->se_hba = hba;
1654 dev->se_sub_dev = se_dev;
1655 dev->transport = transport;
1656 atomic_set(&dev->active_cmds, 0);
1657 INIT_LIST_HEAD(&dev->dev_list);
1658 INIT_LIST_HEAD(&dev->dev_sep_list);
1659 INIT_LIST_HEAD(&dev->dev_tmr_list);
1660 INIT_LIST_HEAD(&dev->execute_task_list);
1661 INIT_LIST_HEAD(&dev->delayed_cmd_list);
1662 INIT_LIST_HEAD(&dev->ordered_cmd_list);
1663 INIT_LIST_HEAD(&dev->state_task_list);
1664 spin_lock_init(&dev->execute_task_lock);
1665 spin_lock_init(&dev->delayed_cmd_lock);
1666 spin_lock_init(&dev->ordered_cmd_lock);
1667 spin_lock_init(&dev->state_task_lock);
1668 spin_lock_init(&dev->dev_alua_lock);
1669 spin_lock_init(&dev->dev_reservation_lock);
1670 spin_lock_init(&dev->dev_status_lock);
1671 spin_lock_init(&dev->dev_status_thr_lock);
1672 spin_lock_init(&dev->se_port_lock);
1673 spin_lock_init(&dev->se_tmr_lock);
1674
1675 dev->queue_depth = dev_limits->queue_depth;
1676 atomic_set(&dev->depth_left, dev->queue_depth);
1677 atomic_set(&dev->dev_ordered_id, 0);
1678
1679 se_dev_set_default_attribs(dev, dev_limits);
1680
1681 dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX);
1682 dev->creation_time = get_jiffies_64();
1683 spin_lock_init(&dev->stats_lock);
1684
1685 spin_lock(&hba->device_lock);
1686 list_add_tail(&dev->dev_list, &hba->hba_dev_list);
1687 hba->dev_count++;
1688 spin_unlock(&hba->device_lock);
1689 /*
1690 * Setup the SAM Task Attribute emulation for struct se_device
1691 */
1692 core_setup_task_attr_emulation(dev);
1693 /*
1694 * Force PR and ALUA passthrough emulation with internal object use.
1695 */
1696 force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE);
1697 /*
1698 * Setup the Reservations infrastructure for struct se_device
1699 */
1700 core_setup_reservations(dev, force_pt);
1701 /*
1702 * Setup the Asymmetric Logical Unit Assignment for struct se_device
1703 */
1704 if (core_setup_alua(dev, force_pt) < 0)
1705 goto out;
1706
1707 /*
1708 * Startup the struct se_device processing thread
1709 */
1710 dev->process_thread = kthread_run(transport_processing_thread, dev,
1711 "LIO_%s", TRANSPORT(dev)->name);
1712 if (IS_ERR(dev->process_thread)) {
1713 printk(KERN_ERR "Unable to create kthread: LIO_%s\n",
1714 TRANSPORT(dev)->name);
1715 goto out;
1716 }
1717
1718 /*
1719 * Preload the initial INQUIRY const values if we are doing
1720 * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI
1721 * passthrough because this is being provided by the backend LLD.
1722 * This is required so that transport_get_inquiry() copies these
1723 * originals once back into DEV_T10_WWN(dev) for the virtual device
1724 * setup.
1725 */
1726 if (TRANSPORT(dev)->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
1727 if (!(inquiry_prod) || !(inquiry_prod)) {
1728 printk(KERN_ERR "All non TCM/pSCSI plugins require"
1729 " INQUIRY consts\n");
1730 goto out;
1731 }
1732
1733 strncpy(&DEV_T10_WWN(dev)->vendor[0], "LIO-ORG", 8);
1734 strncpy(&DEV_T10_WWN(dev)->model[0], inquiry_prod, 16);
1735 strncpy(&DEV_T10_WWN(dev)->revision[0], inquiry_rev, 4);
1736 }
1737 scsi_dump_inquiry(dev);
1738
12a18bdc 1739 return dev;
c66ac9db 1740out:
c66ac9db
NB		 1741 kthread_stop(dev->process_thread);
1742
1743 spin_lock(&hba->device_lock);
1744 list_del(&dev->dev_list);
1745 hba->dev_count--;
1746 spin_unlock(&hba->device_lock);
1747
1748 se_release_vpd_for_dev(dev);
1749
1750 kfree(dev->dev_status_queue_obj);
1751 kfree(dev->dev_queue_obj);
1752 kfree(dev);
1753
1754 return NULL;
1755}
1756EXPORT_SYMBOL(transport_add_device_to_core_hba);
1757
1758/* transport_generic_prepare_cdb():
1759 *
1760 * Since the Initiator sees iSCSI devices as LUNs, the SCSI CDB will
1761 * contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2.
1762 * The point of this is since we are mapping iSCSI LUNs to
1763 * SCSI Target IDs having a non-zero LUN in the CDB will throw the
1764 * devices and HBAs for a loop.
1765 */
1766static inline void transport_generic_prepare_cdb(
1767 unsigned char *cdb)
1768{
1769 switch (cdb[0]) {
1770 case READ_10: /* SBC - RDProtect */
1771 case READ_12: /* SBC - RDProtect */
1772 case READ_16: /* SBC - RDProtect */
1773 case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */
1774 case VERIFY: /* SBC - VRProtect */
1775 case VERIFY_16: /* SBC - VRProtect */
1776 case WRITE_VERIFY: /* SBC - VRProtect */
1777 case WRITE_VERIFY_12: /* SBC - VRProtect */
1778 break;
1779 default:
1780 cdb[1] &= 0x1f; /* clear logical unit number */
1781 break;
1782 }
1783}
1784
1785static struct se_task *
1786transport_generic_get_task(struct se_cmd *cmd,
1787 enum dma_data_direction data_direction)
1788{
1789 struct se_task *task;
1790 struct se_device *dev = SE_DEV(cmd);
1791 unsigned long flags;
1792
1793 task = dev->transport->alloc_task(cmd);
1794 if (!task) {
1795 printk(KERN_ERR "Unable to allocate struct se_task\n");
1796 return NULL;
1797 }
1798
1799 INIT_LIST_HEAD(&task->t_list);
1800 INIT_LIST_HEAD(&task->t_execute_list);
1801 INIT_LIST_HEAD(&task->t_state_list);
1802 init_completion(&task->task_stop_comp);
1803 task->task_no = T_TASK(cmd)->t_tasks_no++;
1804 task->task_se_cmd = cmd;
1805 task->se_dev = dev;
1806 task->task_data_direction = data_direction;
1807
1808 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
1809 list_add_tail(&task->t_list, &T_TASK(cmd)->t_task_list);
1810 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
1811
1812 return task;
1813}
1814
1815static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *);
1816
1817void transport_device_setup_cmd(struct se_cmd *cmd)
1818{
1819 cmd->se_dev = SE_LUN(cmd)->lun_se_dev;
1820}
1821EXPORT_SYMBOL(transport_device_setup_cmd);
1822
1823/*
1824 * Used by fabric modules containing a local struct se_cmd within their
1825 * fabric dependent per I/O descriptor.
1826 */
1827void transport_init_se_cmd(
1828 struct se_cmd *cmd,
1829 struct target_core_fabric_ops *tfo,
1830 struct se_session *se_sess,
1831 u32 data_length,
1832 int data_direction,
1833 int task_attr,
1834 unsigned char *sense_buffer)
1835{
1836 INIT_LIST_HEAD(&cmd->se_lun_list);
1837 INIT_LIST_HEAD(&cmd->se_delayed_list);
1838 INIT_LIST_HEAD(&cmd->se_ordered_list);
1839 /*
1840 * Setup t_task pointer to t_task_backstore
1841 */
1842 cmd->t_task = &cmd->t_task_backstore;
1843
1844 INIT_LIST_HEAD(&T_TASK(cmd)->t_task_list);
1845 init_completion(&T_TASK(cmd)->transport_lun_fe_stop_comp);
1846 init_completion(&T_TASK(cmd)->transport_lun_stop_comp);
1847 init_completion(&T_TASK(cmd)->t_transport_stop_comp);
1848 spin_lock_init(&T_TASK(cmd)->t_state_lock);
1849 atomic_set(&T_TASK(cmd)->transport_dev_active, 1);
1850
1851 cmd->se_tfo = tfo;
1852 cmd->se_sess = se_sess;
1853 cmd->data_length = data_length;
1854 cmd->data_direction = data_direction;
1855 cmd->sam_task_attr = task_attr;
1856 cmd->sense_buffer = sense_buffer;
1857}
1858EXPORT_SYMBOL(transport_init_se_cmd);
1859
1860static int transport_check_alloc_task_attr(struct se_cmd *cmd)
1861{
1862 /*
1863 * Check if SAM Task Attribute emulation is enabled for this
1864 * struct se_device storage object
1865 */
1866 if (SE_DEV(cmd)->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
1867 return 0;
1868
1869 if (cmd->sam_task_attr == TASK_ATTR_ACA) {
1870 DEBUG_STA("SAM Task Attribute ACA"
1871 " emulation is not supported\n");
1872 return -1;
1873 }
1874 /*
1875 * Used to determine when ORDERED commands should go from
1876 * Dormant to Active status.
1877 */
1878 cmd->se_ordered_id = atomic_inc_return(&SE_DEV(cmd)->dev_ordered_id);
1879 smp_mb__after_atomic_inc();
1880 DEBUG_STA("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
1881 cmd->se_ordered_id, cmd->sam_task_attr,
1882 TRANSPORT(cmd->se_dev)->name);
1883 return 0;
1884}
1885
1886void transport_free_se_cmd(
1887 struct se_cmd *se_cmd)
1888{
1889 if (se_cmd->se_tmr_req)
1890 core_tmr_release_req(se_cmd->se_tmr_req);
1891 /*
1892 * Check and free any extended CDB buffer that was allocated
1893 */
1894 if (T_TASK(se_cmd)->t_task_cdb != T_TASK(se_cmd)->__t_task_cdb)
1895 kfree(T_TASK(se_cmd)->t_task_cdb);
1896}
1897EXPORT_SYMBOL(transport_free_se_cmd);
1898
1899static void transport_generic_wait_for_tasks(struct se_cmd *, int, int);
1900
1901/* transport_generic_allocate_tasks():
1902 *
1903 * Called from fabric RX Thread.
1904 */
1905int transport_generic_allocate_tasks(
1906 struct se_cmd *cmd,
1907 unsigned char *cdb)
1908{
1909 int ret;
1910
1911 transport_generic_prepare_cdb(cdb);
1912
1913 /*
1914 * This is needed for early exceptions.
1915 */
1916 cmd->transport_wait_for_tasks = &transport_generic_wait_for_tasks;
1917
1918 transport_device_setup_cmd(cmd);
1919 /*
1920 * Ensure that the received CDB is less than the max (252 + 8) bytes
1921 * for VARIABLE_LENGTH_CMD
1922 */
1923 if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1924 printk(KERN_ERR "Received SCSI CDB with command_size: %d that"
1925 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1926 scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1927 return -1;
1928 }
1929 /*
1930 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
1931 * allocate the additional extended CDB buffer now.. Otherwise
1932 * setup the pointer from __t_task_cdb to t_task_cdb.
1933 */
1934 if (scsi_command_size(cdb) > sizeof(T_TASK(cmd)->__t_task_cdb)) {
1935 T_TASK(cmd)->t_task_cdb = kzalloc(scsi_command_size(cdb),
1936 GFP_KERNEL);
1937 if (!(T_TASK(cmd)->t_task_cdb)) {
1938 printk(KERN_ERR "Unable to allocate T_TASK(cmd)->t_task_cdb"
1939 " %u > sizeof(T_TASK(cmd)->__t_task_cdb): %lu ops\n",
1940 scsi_command_size(cdb),
1941 (unsigned long)sizeof(T_TASK(cmd)->__t_task_cdb));
1942 return -1;
1943 }
1944 } else
1945 T_TASK(cmd)->t_task_cdb = &T_TASK(cmd)->__t_task_cdb[0];
1946 /*
1947 * Copy the original CDB into T_TASK(cmd).
1948 */
1949 memcpy(T_TASK(cmd)->t_task_cdb, cdb, scsi_command_size(cdb));
1950 /*
1951 * Setup the received CDB based on SCSI defined opcodes and
1952 * perform unit attention, persistent reservations and ALUA
1953 * checks for virtual device backends. The T_TASK(cmd)->t_task_cdb
1954 * pointer is expected to be setup before we reach this point.
1955 */
1956 ret = transport_generic_cmd_sequencer(cmd, cdb);
1957 if (ret < 0)
1958 return ret;
1959 /*
1960 * Check for SAM Task Attribute Emulation
1961 */
1962 if (transport_check_alloc_task_attr(cmd) < 0) {
1963 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1964 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
1965 return -2;
1966 }
1967 spin_lock(&cmd->se_lun->lun_sep_lock);
1968 if (cmd->se_lun->lun_sep)
1969 cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
1970 spin_unlock(&cmd->se_lun->lun_sep_lock);
1971 return 0;
1972}
1973EXPORT_SYMBOL(transport_generic_allocate_tasks);
1974
1975/*
1976 * Used by fabric module frontends not defining a TFO->new_cmd_map()
1977 * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD statis
1978 */
1979int transport_generic_handle_cdb(
1980 struct se_cmd *cmd)
1981{
1982 if (!SE_LUN(cmd)) {
1983 dump_stack();
1984 printk(KERN_ERR "SE_LUN(cmd) is NULL\n");
1985 return -1;
1986 }
1987
1988 transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD);
1989 return 0;
1990}
1991EXPORT_SYMBOL(transport_generic_handle_cdb);
1992
1993/*
1994 * Used by fabric module frontends defining a TFO->new_cmd_map() caller
1995 * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to
1996 * complete setup in TCM process context w/ TFO->new_cmd_map().
1997 */
1998int transport_generic_handle_cdb_map(
1999 struct se_cmd *cmd)
2000{
2001 if (!SE_LUN(cmd)) {
2002 dump_stack();
2003 printk(KERN_ERR "SE_LUN(cmd) is NULL\n");
2004 return -1;
2005 }
2006
2007 transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP);
2008 return 0;
2009}
2010EXPORT_SYMBOL(transport_generic_handle_cdb_map);
2011
2012/* transport_generic_handle_data():
2013 *
2014 *
2015 */
2016int transport_generic_handle_data(
2017 struct se_cmd *cmd)
2018{
2019 /*
2020 * For the software fabric case, then we assume the nexus is being
2021 * failed/shutdown when signals are pending from the kthread context
2022 * caller, so we return a failure. For the HW target mode case running
2023 * in interrupt code, the signal_pending() check is skipped.
2024 */
2025 if (!in_interrupt() && signal_pending(current))
2026 return -1;
2027 /*
2028 * If the received CDB has aleady been ABORTED by the generic
2029 * target engine, we now call transport_check_aborted_status()
2030 * to queue any delated TASK_ABORTED status for the received CDB to the
25985edc 2031 * fabric module as we are expecting no further incoming DATA OUT
c66ac9db
NB		 2032 * sequences at this point.
2033 */
2034 if (transport_check_aborted_status(cmd, 1) != 0)
2035 return 0;
2036
2037 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE);
2038 return 0;
2039}
2040EXPORT_SYMBOL(transport_generic_handle_data);
2041
2042/* transport_generic_handle_tmr():
2043 *
2044 *
2045 */
2046int transport_generic_handle_tmr(
2047 struct se_cmd *cmd)
2048{
2049 /*
2050 * This is needed for early exceptions.
2051 */
2052 cmd->transport_wait_for_tasks = &transport_generic_wait_for_tasks;
2053 transport_device_setup_cmd(cmd);
2054
2055 transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR);
2056 return 0;
2057}
2058EXPORT_SYMBOL(transport_generic_handle_tmr);
2059
f4366772
NB		 2060void transport_generic_free_cmd_intr(
2061 struct se_cmd *cmd)
2062{
2063 transport_add_cmd_to_queue(cmd, TRANSPORT_FREE_CMD_INTR);
2064}
2065EXPORT_SYMBOL(transport_generic_free_cmd_intr);
2066
c66ac9db
NB		 2067static int transport_stop_tasks_for_cmd(struct se_cmd *cmd)
2068{
2069 struct se_task *task, *task_tmp;
2070 unsigned long flags;
2071 int ret = 0;
2072
2073 DEBUG_TS("ITT[0x%08x] - Stopping tasks\n",
2074 CMD_TFO(cmd)->get_task_tag(cmd));
2075
2076 /*
2077 * No tasks remain in the execution queue
2078 */
2079 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2080 list_for_each_entry_safe(task, task_tmp,
2081 &T_TASK(cmd)->t_task_list, t_list) {
2082 DEBUG_TS("task_no[%d] - Processing task %p\n",
2083 task->task_no, task);
2084 /*
2085 * If the struct se_task has not been sent and is not active,
2086 * remove the struct se_task from the execution queue.
2087 */
2088 if (!atomic_read(&task->task_sent) &&
2089 !atomic_read(&task->task_active)) {
2090 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
2091 flags);
2092 transport_remove_task_from_execute_queue(task,
2093 task->se_dev);
2094
2095 DEBUG_TS("task_no[%d] - Removed from execute queue\n",
2096 task->task_no);
2097 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2098 continue;
2099 }
2100
2101 /*
2102 * If the struct se_task is active, sleep until it is returned
2103 * from the plugin.
2104 */
2105 if (atomic_read(&task->task_active)) {
2106 atomic_set(&task->task_stop, 1);
2107 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
2108 flags);
2109
2110 DEBUG_TS("task_no[%d] - Waiting to complete\n",
2111 task->task_no);
2112 wait_for_completion(&task->task_stop_comp);
2113 DEBUG_TS("task_no[%d] - Stopped successfully\n",
2114 task->task_no);
2115
2116 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2117 atomic_dec(&T_TASK(cmd)->t_task_cdbs_left);
2118
2119 atomic_set(&task->task_active, 0);
2120 atomic_set(&task->task_stop, 0);
2121 } else {
2122 DEBUG_TS("task_no[%d] - Did nothing\n", task->task_no);
2123 ret++;
2124 }
2125
2126 __transport_stop_task_timer(task, &flags);
2127 }
2128 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2129
2130 return ret;
2131}
2132
2133static void transport_failure_reset_queue_depth(struct se_device *dev)
2134{
2135 unsigned long flags;
2136
6eab04a8 2137 spin_lock_irqsave(&SE_HBA(dev)->hba_queue_lock, flags);
c66ac9db
NB		 2138 atomic_inc(&dev->depth_left);
2139 atomic_inc(&SE_HBA(dev)->left_queue_depth);
2140 spin_unlock_irqrestore(&SE_HBA(dev)->hba_queue_lock, flags);
2141}
2142
2143/*
2144 * Handle SAM-esque emulation for generic transport request failures.
2145 */
2146static void transport_generic_request_failure(
2147 struct se_cmd *cmd,
2148 struct se_device *dev,
2149 int complete,
2150 int sc)
2151{
2152 DEBUG_GRF("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
2153 " CDB: 0x%02x\n", cmd, CMD_TFO(cmd)->get_task_tag(cmd),
2154 T_TASK(cmd)->t_task_cdb[0]);
2155 DEBUG_GRF("-----[ i_state: %d t_state/def_t_state:"
2156 " %d/%d transport_error_status: %d\n",
2157 CMD_TFO(cmd)->get_cmd_state(cmd),
2158 cmd->t_state, cmd->deferred_t_state,
2159 cmd->transport_error_status);
2160 DEBUG_GRF("-----[ t_task_cdbs: %d t_task_cdbs_left: %d"
2161 " t_task_cdbs_sent: %d t_task_cdbs_ex_left: %d --"
2162 " t_transport_active: %d t_transport_stop: %d"
2163 " t_transport_sent: %d\n", T_TASK(cmd)->t_task_cdbs,
2164 atomic_read(&T_TASK(cmd)->t_task_cdbs_left),
2165 atomic_read(&T_TASK(cmd)->t_task_cdbs_sent),
2166 atomic_read(&T_TASK(cmd)->t_task_cdbs_ex_left),
2167 atomic_read(&T_TASK(cmd)->t_transport_active),
2168 atomic_read(&T_TASK(cmd)->t_transport_stop),
2169 atomic_read(&T_TASK(cmd)->t_transport_sent));
2170
2171 transport_stop_all_task_timers(cmd);
2172
2173 if (dev)
2174 transport_failure_reset_queue_depth(dev);
2175 /*
2176 * For SAM Task Attribute emulation for failed struct se_cmd
2177 */
2178 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
2179 transport_complete_task_attr(cmd);
2180
2181 if (complete) {
2182 transport_direct_request_timeout(cmd);
2183 cmd->transport_error_status = PYX_TRANSPORT_LU_COMM_FAILURE;
2184 }
2185
2186 switch (cmd->transport_error_status) {
2187 case PYX_TRANSPORT_UNKNOWN_SAM_OPCODE:
2188 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
2189 break;
2190 case PYX_TRANSPORT_REQ_TOO_MANY_SECTORS:
2191 cmd->scsi_sense_reason = TCM_SECTOR_COUNT_TOO_MANY;
2192 break;
2193 case PYX_TRANSPORT_INVALID_CDB_FIELD:
2194 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
2195 break;
2196 case PYX_TRANSPORT_INVALID_PARAMETER_LIST:
2197 cmd->scsi_sense_reason = TCM_INVALID_PARAMETER_LIST;
2198 break;
2199 case PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES:
2200 if (!sc)
2201 transport_new_cmd_failure(cmd);
2202 /*
2203 * Currently for PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES,
2204 * we force this session to fall back to session
2205 * recovery.
2206 */
2207 CMD_TFO(cmd)->fall_back_to_erl0(cmd->se_sess);
2208 CMD_TFO(cmd)->stop_session(cmd->se_sess, 0, 0);
2209
2210 goto check_stop;
2211 case PYX_TRANSPORT_LU_COMM_FAILURE:
2212 case PYX_TRANSPORT_ILLEGAL_REQUEST:
2213 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2214 break;
2215 case PYX_TRANSPORT_UNKNOWN_MODE_PAGE:
2216 cmd->scsi_sense_reason = TCM_UNKNOWN_MODE_PAGE;
2217 break;
2218 case PYX_TRANSPORT_WRITE_PROTECTED:
2219 cmd->scsi_sense_reason = TCM_WRITE_PROTECTED;
2220 break;
2221 case PYX_TRANSPORT_RESERVATION_CONFLICT:
2222 /*
2223 * No SENSE Data payload for this case, set SCSI Status
2224 * and queue the response to $FABRIC_MOD.
2225 *
2226 * Uses linux/include/scsi/scsi.h SAM status codes defs
2227 */
2228 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
2229 /*
2230 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
2231 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
2232 * CONFLICT STATUS.
2233 *
2234 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
2235 */
2236 if (SE_SESS(cmd) &&
2237 DEV_ATTRIB(cmd->se_dev)->emulate_ua_intlck_ctrl == 2)
2238 core_scsi3_ua_allocate(SE_SESS(cmd)->se_node_acl,
2239 cmd->orig_fe_lun, 0x2C,
2240 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
2241
2242 CMD_TFO(cmd)->queue_status(cmd);
2243 goto check_stop;
2244 case PYX_TRANSPORT_USE_SENSE_REASON:
2245 /*
2246 * struct se_cmd->scsi_sense_reason already set
2247 */
2248 break;
2249 default:
2250 printk(KERN_ERR "Unknown transport error for CDB 0x%02x: %d\n",
2251 T_TASK(cmd)->t_task_cdb[0],
2252 cmd->transport_error_status);
2253 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
2254 break;
2255 }
2256
2257 if (!sc)
2258 transport_new_cmd_failure(cmd);
2259 else
2260 transport_send_check_condition_and_sense(cmd,
2261 cmd->scsi_sense_reason, 0);
2262check_stop:
2263 transport_lun_remove_cmd(cmd);
2264 if (!(transport_cmd_check_stop_to_fabric(cmd)))
2265 ;
2266}
2267
2268static void transport_direct_request_timeout(struct se_cmd *cmd)
2269{
2270 unsigned long flags;
2271
2272 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2273 if (!(atomic_read(&T_TASK(cmd)->t_transport_timeout))) {
2274 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2275 return;
2276 }
2277 if (atomic_read(&T_TASK(cmd)->t_task_cdbs_timeout_left)) {
2278 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2279 return;
2280 }
2281
2282 atomic_sub(atomic_read(&T_TASK(cmd)->t_transport_timeout),
2283 &T_TASK(cmd)->t_se_count);
2284 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2285}
2286
2287static void transport_generic_request_timeout(struct se_cmd *cmd)
2288{
2289 unsigned long flags;
2290
2291 /*
2292 * Reset T_TASK(cmd)->t_se_count to allow transport_generic_remove()
2293 * to allow last call to free memory resources.
2294 */
2295 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2296 if (atomic_read(&T_TASK(cmd)->t_transport_timeout) > 1) {
2297 int tmp = (atomic_read(&T_TASK(cmd)->t_transport_timeout) - 1);
2298
2299 atomic_sub(tmp, &T_TASK(cmd)->t_se_count);
2300 }
2301 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2302
2303 transport_generic_remove(cmd, 0, 0);
2304}
2305
2306static int
2307transport_generic_allocate_buf(struct se_cmd *cmd, u32 data_length)
2308{
2309 unsigned char *buf;
2310
2311 buf = kzalloc(data_length, GFP_KERNEL);
2312 if (!(buf)) {
2313 printk(KERN_ERR "Unable to allocate memory for buffer\n");
2314 return -1;
2315 }
2316
2317 T_TASK(cmd)->t_tasks_se_num = 0;
2318 T_TASK(cmd)->t_task_buf = buf;
2319
2320 return 0;
2321}
2322
2323static inline u32 transport_lba_21(unsigned char *cdb)
2324{
2325 return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3];
2326}
2327
2328static inline u32 transport_lba_32(unsigned char *cdb)
2329{
2330 return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
2331}
2332
2333static inline unsigned long long transport_lba_64(unsigned char *cdb)
2334{
2335 unsigned int __v1, __v2;
2336
2337 __v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
2338 __v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
2339
2340 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
2341}
2342
2343/*
2344 * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs
2345 */
2346static inline unsigned long long transport_lba_64_ext(unsigned char *cdb)
2347{
2348 unsigned int __v1, __v2;
2349
2350 __v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15];
2351 __v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19];
2352
2353 return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
2354}
2355
2356static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd)
2357{
2358 unsigned long flags;
2359
2360 spin_lock_irqsave(&T_TASK(se_cmd)->t_state_lock, flags);
2361 se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
2362 spin_unlock_irqrestore(&T_TASK(se_cmd)->t_state_lock, flags);
2363}
2364
2365/*
2366 * Called from interrupt context.
2367 */
2368static void transport_task_timeout_handler(unsigned long data)
2369{
2370 struct se_task *task = (struct se_task *)data;
2371 struct se_cmd *cmd = TASK_CMD(task);
2372 unsigned long flags;
2373
2374 DEBUG_TT("transport task timeout fired! task: %p cmd: %p\n", task, cmd);
2375
2376 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2377 if (task->task_flags & TF_STOP) {
2378 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2379 return;
2380 }
2381 task->task_flags &= ~TF_RUNNING;
2382
2383 /*
2384 * Determine if transport_complete_task() has already been called.
2385 */
2386 if (!(atomic_read(&task->task_active))) {
2387 DEBUG_TT("transport task: %p cmd: %p timeout task_active"
2388 " == 0\n", task, cmd);
2389 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2390 return;
2391 }
2392
2393 atomic_inc(&T_TASK(cmd)->t_se_count);
2394 atomic_inc(&T_TASK(cmd)->t_transport_timeout);
2395 T_TASK(cmd)->t_tasks_failed = 1;
2396
2397 atomic_set(&task->task_timeout, 1);
2398 task->task_error_status = PYX_TRANSPORT_TASK_TIMEOUT;
2399 task->task_scsi_status = 1;
2400
2401 if (atomic_read(&task->task_stop)) {
2402 DEBUG_TT("transport task: %p cmd: %p timeout task_stop"
2403 " == 1\n", task, cmd);
2404 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2405 complete(&task->task_stop_comp);
2406 return;
2407 }
2408
2409 if (!(atomic_dec_and_test(&T_TASK(cmd)->t_task_cdbs_left))) {
2410 DEBUG_TT("transport task: %p cmd: %p timeout non zero"
2411 " t_task_cdbs_left\n", task, cmd);
2412 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2413 return;
2414 }
2415 DEBUG_TT("transport task: %p cmd: %p timeout ZERO t_task_cdbs_left\n",
2416 task, cmd);
2417
2418 cmd->t_state = TRANSPORT_COMPLETE_FAILURE;
2419 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2420
2421 transport_add_cmd_to_queue(cmd, TRANSPORT_COMPLETE_FAILURE);
2422}
2423
2424/*
2425 * Called with T_TASK(cmd)->t_state_lock held.
2426 */
2427static void transport_start_task_timer(struct se_task *task)
2428{
2429 struct se_device *dev = task->se_dev;
2430 int timeout;
2431
2432 if (task->task_flags & TF_RUNNING)
2433 return;
2434 /*
2435 * If the task_timeout is disabled, exit now.
2436 */
2437 timeout = DEV_ATTRIB(dev)->task_timeout;
2438 if (!(timeout))
2439 return;
2440
2441 init_timer(&task->task_timer);
2442 task->task_timer.expires = (get_jiffies_64() + timeout * HZ);
2443 task->task_timer.data = (unsigned long) task;
2444 task->task_timer.function = transport_task_timeout_handler;
2445
2446 task->task_flags |= TF_RUNNING;
2447 add_timer(&task->task_timer);
2448#if 0
2449 printk(KERN_INFO "Starting task timer for cmd: %p task: %p seconds:"
2450 " %d\n", task->task_se_cmd, task, timeout);
2451#endif
2452}
2453
2454/*
2455 * Called with spin_lock_irq(&T_TASK(cmd)->t_state_lock) held.
2456 */
2457void __transport_stop_task_timer(struct se_task *task, unsigned long *flags)
2458{
2459 struct se_cmd *cmd = TASK_CMD(task);
2460
2461 if (!(task->task_flags & TF_RUNNING))
2462 return;
2463
2464 task->task_flags |= TF_STOP;
2465 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, *flags);
2466
2467 del_timer_sync(&task->task_timer);
2468
2469 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, *flags);
2470 task->task_flags &= ~TF_RUNNING;
2471 task->task_flags &= ~TF_STOP;
2472}
2473
2474static void transport_stop_all_task_timers(struct se_cmd *cmd)
2475{
2476 struct se_task *task = NULL, *task_tmp;
2477 unsigned long flags;
2478
2479 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2480 list_for_each_entry_safe(task, task_tmp,
2481 &T_TASK(cmd)->t_task_list, t_list)
2482 __transport_stop_task_timer(task, &flags);
2483 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2484}
2485
2486static inline int transport_tcq_window_closed(struct se_device *dev)
2487{
2488 if (dev->dev_tcq_window_closed++ <
2489 PYX_TRANSPORT_WINDOW_CLOSED_THRESHOLD) {
2490 msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_SHORT);
2491 } else
2492 msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_LONG);
2493
2494 wake_up_interruptible(&dev->dev_queue_obj->thread_wq);
2495 return 0;
2496}
2497
2498/*
2499 * Called from Fabric Module context from transport_execute_tasks()
2500 *
2501 * The return of this function determins if the tasks from struct se_cmd
2502 * get added to the execution queue in transport_execute_tasks(),
2503 * or are added to the delayed or ordered lists here.
2504 */
2505static inline int transport_execute_task_attr(struct se_cmd *cmd)
2506{
2507 if (SE_DEV(cmd)->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
2508 return 1;
2509 /*
25985edc 2510 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
c66ac9db
NB		 2511 * to allow the passed struct se_cmd list of tasks to the front of the list.
2512 */
2513 if (cmd->sam_task_attr == TASK_ATTR_HOQ) {
2514 atomic_inc(&SE_DEV(cmd)->dev_hoq_count);
2515 smp_mb__after_atomic_inc();
2516 DEBUG_STA("Added HEAD_OF_QUEUE for CDB:"
2517 " 0x%02x, se_ordered_id: %u\n",
2518 T_TASK(cmd)->t_task_cdb[0],
2519 cmd->se_ordered_id);
2520 return 1;
2521 } else if (cmd->sam_task_attr == TASK_ATTR_ORDERED) {
2522 spin_lock(&SE_DEV(cmd)->ordered_cmd_lock);
2523 list_add_tail(&cmd->se_ordered_list,
2524 &SE_DEV(cmd)->ordered_cmd_list);
2525 spin_unlock(&SE_DEV(cmd)->ordered_cmd_lock);
2526
2527 atomic_inc(&SE_DEV(cmd)->dev_ordered_sync);
2528 smp_mb__after_atomic_inc();
2529
2530 DEBUG_STA("Added ORDERED for CDB: 0x%02x to ordered"
2531 " list, se_ordered_id: %u\n",
2532 T_TASK(cmd)->t_task_cdb[0],
2533 cmd->se_ordered_id);
2534 /*
2535 * Add ORDERED command to tail of execution queue if
2536 * no other older commands exist that need to be
2537 * completed first.
2538 */
2539 if (!(atomic_read(&SE_DEV(cmd)->simple_cmds)))
2540 return 1;
2541 } else {
2542 /*
2543 * For SIMPLE and UNTAGGED Task Attribute commands
2544 */
2545 atomic_inc(&SE_DEV(cmd)->simple_cmds);
2546 smp_mb__after_atomic_inc();
2547 }
2548 /*
2549 * Otherwise if one or more outstanding ORDERED task attribute exist,
2550 * add the dormant task(s) built for the passed struct se_cmd to the
2551 * execution queue and become in Active state for this struct se_device.
2552 */
2553 if (atomic_read(&SE_DEV(cmd)->dev_ordered_sync) != 0) {
2554 /*
2555 * Otherwise, add cmd w/ tasks to delayed cmd queue that
25985edc 2556 * will be drained upon completion of HEAD_OF_QUEUE task.
c66ac9db
NB		 2557 */
2558 spin_lock(&SE_DEV(cmd)->delayed_cmd_lock);
2559 cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR;
2560 list_add_tail(&cmd->se_delayed_list,
2561 &SE_DEV(cmd)->delayed_cmd_list);
2562 spin_unlock(&SE_DEV(cmd)->delayed_cmd_lock);
2563
2564 DEBUG_STA("Added CDB: 0x%02x Task Attr: 0x%02x to"
2565 " delayed CMD list, se_ordered_id: %u\n",
2566 T_TASK(cmd)->t_task_cdb[0], cmd->sam_task_attr,
2567 cmd->se_ordered_id);
2568 /*
2569 * Return zero to let transport_execute_tasks() know
2570 * not to add the delayed tasks to the execution list.
2571 */
2572 return 0;
2573 }
2574 /*
2575 * Otherwise, no ORDERED task attributes exist..
2576 */
2577 return 1;
2578}
2579
2580/*
2581 * Called from fabric module context in transport_generic_new_cmd() and
2582 * transport_generic_process_write()
2583 */
2584static int transport_execute_tasks(struct se_cmd *cmd)
2585{
2586 int add_tasks;
2587
2588 if (!(cmd->se_cmd_flags & SCF_SE_DISABLE_ONLINE_CHECK)) {
2589 if (se_dev_check_online(cmd->se_orig_obj_ptr) != 0) {
2590 cmd->transport_error_status =
2591 PYX_TRANSPORT_LU_COMM_FAILURE;
2592 transport_generic_request_failure(cmd, NULL, 0, 1);
2593 return 0;
2594 }
2595 }
2596 /*
2597 * Call transport_cmd_check_stop() to see if a fabric exception
25985edc 2598 * has occurred that prevents execution.
c66ac9db
NB		 2599 */
2600 if (!(transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING))) {
2601 /*
2602 * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE
2603 * attribute for the tasks of the received struct se_cmd CDB
2604 */
2605 add_tasks = transport_execute_task_attr(cmd);
2606 if (add_tasks == 0)
2607 goto execute_tasks;
2608 /*
2609 * This calls transport_add_tasks_from_cmd() to handle
2610 * HEAD_OF_QUEUE ordering for SAM Task Attribute emulation
2611 * (if enabled) in __transport_add_task_to_execute_queue() and
2612 * transport_add_task_check_sam_attr().
2613 */
2614 transport_add_tasks_from_cmd(cmd);
2615 }
2616 /*
2617 * Kick the execution queue for the cmd associated struct se_device
2618 * storage object.
2619 */
2620execute_tasks:
2621 __transport_execute_tasks(SE_DEV(cmd));
2622 return 0;
2623}
2624
2625/*
2626 * Called to check struct se_device tcq depth window, and once open pull struct se_task
2627 * from struct se_device->execute_task_list and
2628 *
2629 * Called from transport_processing_thread()
2630 */
2631static int __transport_execute_tasks(struct se_device *dev)
2632{
2633 int error;
2634 struct se_cmd *cmd = NULL;
2635 struct se_task *task;
2636 unsigned long flags;
2637
2638 /*
2639 * Check if there is enough room in the device and HBA queue to send
2640 * struct se_transport_task's to the selected transport.
2641 */
2642check_depth:
2643 spin_lock_irqsave(&SE_HBA(dev)->hba_queue_lock, flags);
2644 if (!(atomic_read(&dev->depth_left)) ||
2645 !(atomic_read(&SE_HBA(dev)->left_queue_depth))) {
2646 spin_unlock_irqrestore(&SE_HBA(dev)->hba_queue_lock, flags);
2647 return transport_tcq_window_closed(dev);
2648 }
2649 dev->dev_tcq_window_closed = 0;
2650
2651 spin_lock(&dev->execute_task_lock);
2652 task = transport_get_task_from_execute_queue(dev);
2653 spin_unlock(&dev->execute_task_lock);
2654
2655 if (!task) {
2656 spin_unlock_irqrestore(&SE_HBA(dev)->hba_queue_lock, flags);
2657 return 0;
2658 }
2659
2660 atomic_dec(&dev->depth_left);
2661 atomic_dec(&SE_HBA(dev)->left_queue_depth);
2662 spin_unlock_irqrestore(&SE_HBA(dev)->hba_queue_lock, flags);
2663
2664 cmd = TASK_CMD(task);
2665
2666 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2667 atomic_set(&task->task_active, 1);
2668 atomic_set(&task->task_sent, 1);
2669 atomic_inc(&T_TASK(cmd)->t_task_cdbs_sent);
2670
2671 if (atomic_read(&T_TASK(cmd)->t_task_cdbs_sent) ==
2672 T_TASK(cmd)->t_task_cdbs)
2673 atomic_set(&cmd->transport_sent, 1);
2674
2675 transport_start_task_timer(task);
2676 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2677 /*
2678 * The struct se_cmd->transport_emulate_cdb() function pointer is used
2679 * to grab REPORT_LUNS CDBs before they hit the
2680 * struct se_subsystem_api->do_task() caller below.
2681 */
2682 if (cmd->transport_emulate_cdb) {
2683 error = cmd->transport_emulate_cdb(cmd);
2684 if (error != 0) {
2685 cmd->transport_error_status = error;
2686 atomic_set(&task->task_active, 0);
2687 atomic_set(&cmd->transport_sent, 0);
2688 transport_stop_tasks_for_cmd(cmd);
2689 transport_generic_request_failure(cmd, dev, 0, 1);
2690 goto check_depth;
2691 }
2692 /*
2693 * Handle the successful completion for transport_emulate_cdb()
2694 * for synchronous operation, following SCF_EMULATE_CDB_ASYNC
2695 * Otherwise the caller is expected to complete the task with
2696 * proper status.
2697 */
2698 if (!(cmd->se_cmd_flags & SCF_EMULATE_CDB_ASYNC)) {
2699 cmd->scsi_status = SAM_STAT_GOOD;
2700 task->task_scsi_status = GOOD;
2701 transport_complete_task(task, 1);
2702 }
2703 } else {
2704 /*
2705 * Currently for all virtual TCM plugins including IBLOCK, FILEIO and
2706 * RAMDISK we use the internal transport_emulate_control_cdb() logic
2707 * with struct se_subsystem_api callers for the primary SPC-3 TYPE_DISK
2708 * LUN emulation code.
2709 *
2710 * For TCM/pSCSI and all other SCF_SCSI_DATA_SG_IO_CDB I/O tasks we
2711 * call ->do_task() directly and let the underlying TCM subsystem plugin
2712 * code handle the CDB emulation.
2713 */
2714 if ((TRANSPORT(dev)->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) &&
2715 (!(TASK_CMD(task)->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)))
2716 error = transport_emulate_control_cdb(task);
2717 else
2718 error = TRANSPORT(dev)->do_task(task);
2719
2720 if (error != 0) {
2721 cmd->transport_error_status = error;
2722 atomic_set(&task->task_active, 0);
2723 atomic_set(&cmd->transport_sent, 0);
2724 transport_stop_tasks_for_cmd(cmd);
2725 transport_generic_request_failure(cmd, dev, 0, 1);
2726 }
2727 }
2728
2729 goto check_depth;
2730
2731 return 0;
2732}
2733
2734void transport_new_cmd_failure(struct se_cmd *se_cmd)
2735{
2736 unsigned long flags;
2737 /*
2738 * Any unsolicited data will get dumped for failed command inside of
2739 * the fabric plugin
2740 */
2741 spin_lock_irqsave(&T_TASK(se_cmd)->t_state_lock, flags);
2742 se_cmd->se_cmd_flags |= SCF_SE_CMD_FAILED;
2743 se_cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2744 spin_unlock_irqrestore(&T_TASK(se_cmd)->t_state_lock, flags);
2745
2746 CMD_TFO(se_cmd)->new_cmd_failure(se_cmd);
2747}
2748
2749static void transport_nop_wait_for_tasks(struct se_cmd *, int, int);
2750
2751static inline u32 transport_get_sectors_6(
2752 unsigned char *cdb,
2753 struct se_cmd *cmd,
2754 int *ret)
2755{
2756 struct se_device *dev = SE_LUN(cmd)->lun_se_dev;
2757
2758 /*
2759 * Assume TYPE_DISK for non struct se_device objects.
2760 * Use 8-bit sector value.
2761 */
2762 if (!dev)
2763 goto type_disk;
2764
2765 /*
2766 * Use 24-bit allocation length for TYPE_TAPE.
2767 */
2768 if (TRANSPORT(dev)->get_device_type(dev) == TYPE_TAPE)
2769 return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4];
2770
2771 /*
2772 * Everything else assume TYPE_DISK Sector CDB location.
2773 * Use 8-bit sector value.
2774 */
2775type_disk:
2776 return (u32)cdb[4];
2777}
2778
2779static inline u32 transport_get_sectors_10(
2780 unsigned char *cdb,
2781 struct se_cmd *cmd,
2782 int *ret)
2783{
2784 struct se_device *dev = SE_LUN(cmd)->lun_se_dev;
2785
2786 /*
2787 * Assume TYPE_DISK for non struct se_device objects.
2788 * Use 16-bit sector value.
2789 */
2790 if (!dev)
2791 goto type_disk;
2792
2793 /*
2794 * XXX_10 is not defined in SSC, throw an exception
2795 */
2796 if (TRANSPORT(dev)->get_device_type(dev) == TYPE_TAPE) {
2797 *ret = -1;
2798 return 0;
2799 }
2800
2801 /*
2802 * Everything else assume TYPE_DISK Sector CDB location.
2803 * Use 16-bit sector value.
2804 */
2805type_disk:
2806 return (u32)(cdb[7] << 8) + cdb[8];
2807}
2808
2809static inline u32 transport_get_sectors_12(
2810 unsigned char *cdb,
2811 struct se_cmd *cmd,
2812 int *ret)
2813{
2814 struct se_device *dev = SE_LUN(cmd)->lun_se_dev;
2815
2816 /*
2817 * Assume TYPE_DISK for non struct se_device objects.
2818 * Use 32-bit sector value.
2819 */
2820 if (!dev)
2821 goto type_disk;
2822
2823 /*
2824 * XXX_12 is not defined in SSC, throw an exception
2825 */
2826 if (TRANSPORT(dev)->get_device_type(dev) == TYPE_TAPE) {
2827 *ret = -1;
2828 return 0;
2829 }
2830
2831 /*
2832 * Everything else assume TYPE_DISK Sector CDB location.
2833 * Use 32-bit sector value.
2834 */
2835type_disk:
2836 return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9];
2837}
2838
2839static inline u32 transport_get_sectors_16(
2840 unsigned char *cdb,
2841 struct se_cmd *cmd,
2842 int *ret)
2843{
2844 struct se_device *dev = SE_LUN(cmd)->lun_se_dev;
2845
2846 /*
2847 * Assume TYPE_DISK for non struct se_device objects.
2848 * Use 32-bit sector value.
2849 */
2850 if (!dev)
2851 goto type_disk;
2852
2853 /*
2854 * Use 24-bit allocation length for TYPE_TAPE.
2855 */
2856 if (TRANSPORT(dev)->get_device_type(dev) == TYPE_TAPE)
2857 return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14];
2858
2859type_disk:
2860 return (u32)(cdb[10] << 24) + (cdb[11] << 16) +
2861 (cdb[12] << 8) + cdb[13];
2862}
2863
2864/*
2865 * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants
2866 */
2867static inline u32 transport_get_sectors_32(
2868 unsigned char *cdb,
2869 struct se_cmd *cmd,
2870 int *ret)
2871{
2872 /*
2873 * Assume TYPE_DISK for non struct se_device objects.
2874 * Use 32-bit sector value.
2875 */
2876 return (u32)(cdb[28] << 24) + (cdb[29] << 16) +
2877 (cdb[30] << 8) + cdb[31];
2878
2879}
2880
2881static inline u32 transport_get_size(
2882 u32 sectors,
2883 unsigned char *cdb,
2884 struct se_cmd *cmd)
2885{
2886 struct se_device *dev = SE_DEV(cmd);
2887
2888 if (TRANSPORT(dev)->get_device_type(dev) == TYPE_TAPE) {
2889 if (cdb[1] & 1) { /* sectors */
2890 return DEV_ATTRIB(dev)->block_size * sectors;
2891 } else /* bytes */
2892 return sectors;
2893 }
2894#if 0
2895 printk(KERN_INFO "Returning block_size: %u, sectors: %u == %u for"
2896 " %s object\n", DEV_ATTRIB(dev)->block_size, sectors,
2897 DEV_ATTRIB(dev)->block_size * sectors,
2898 TRANSPORT(dev)->name);
2899#endif
2900 return DEV_ATTRIB(dev)->block_size * sectors;
2901}
2902
2903unsigned char transport_asciihex_to_binaryhex(unsigned char val[2])
2904{
2905 unsigned char result = 0;
2906 /*
2907 * MSB
2908 */
2909 if ((val[0] >= 'a') && (val[0] <= 'f'))
2910 result = ((val[0] - 'a' + 10) & 0xf) << 4;
2911 else
2912 if ((val[0] >= 'A') && (val[0] <= 'F'))
2913 result = ((val[0] - 'A' + 10) & 0xf) << 4;
2914 else /* digit */
2915 result = ((val[0] - '0') & 0xf) << 4;
2916 /*
2917 * LSB
2918 */
2919 if ((val[1] >= 'a') && (val[1] <= 'f'))
2920 result |= ((val[1] - 'a' + 10) & 0xf);
2921 else
2922 if ((val[1] >= 'A') && (val[1] <= 'F'))
2923 result |= ((val[1] - 'A' + 10) & 0xf);
2924 else /* digit */
2925 result |= ((val[1] - '0') & 0xf);
2926
2927 return result;
2928}
2929EXPORT_SYMBOL(transport_asciihex_to_binaryhex);
2930
2931static void transport_xor_callback(struct se_cmd *cmd)
2932{
2933 unsigned char *buf, *addr;
2934 struct se_mem *se_mem;
2935 unsigned int offset;
2936 int i;
2937 /*
2938 * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command
2939 *
2940 * 1) read the specified logical block(s);
2941 * 2) transfer logical blocks from the data-out buffer;
2942 * 3) XOR the logical blocks transferred from the data-out buffer with
2943 * the logical blocks read, storing the resulting XOR data in a buffer;
2944 * 4) if the DISABLE WRITE bit is set to zero, then write the logical
2945 * blocks transferred from the data-out buffer; and
2946 * 5) transfer the resulting XOR data to the data-in buffer.
2947 */
2948 buf = kmalloc(cmd->data_length, GFP_KERNEL);
2949 if (!(buf)) {
2950 printk(KERN_ERR "Unable to allocate xor_callback buf\n");
2951 return;
2952 }
2953 /*
2954 * Copy the scatterlist WRITE buffer located at T_TASK(cmd)->t_mem_list
2955 * into the locally allocated *buf
2956 */
2957 transport_memcpy_se_mem_read_contig(cmd, buf, T_TASK(cmd)->t_mem_list);
2958 /*
2959 * Now perform the XOR against the BIDI read memory located at
2960 * T_TASK(cmd)->t_mem_bidi_list
2961 */
2962
2963 offset = 0;
2964 list_for_each_entry(se_mem, T_TASK(cmd)->t_mem_bidi_list, se_list) {
2965 addr = (unsigned char *)kmap_atomic(se_mem->se_page, KM_USER0);
2966 if (!(addr))
2967 goto out;
2968
2969 for (i = 0; i < se_mem->se_len; i++)
2970 *(addr + se_mem->se_off + i) ^= *(buf + offset + i);
2971
2972 offset += se_mem->se_len;
2973 kunmap_atomic(addr, KM_USER0);
2974 }
2975out:
2976 kfree(buf);
2977}
2978
2979/*
2980 * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd
2981 */
2982static int transport_get_sense_data(struct se_cmd *cmd)
2983{
2984 unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL;
2985 struct se_device *dev;
2986 struct se_task *task = NULL, *task_tmp;
2987 unsigned long flags;
2988 u32 offset = 0;
2989
2990 if (!SE_LUN(cmd)) {
2991 printk(KERN_ERR "SE_LUN(cmd) is NULL\n");
2992 return -1;
2993 }
2994 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
2995 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
2996 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
2997 return 0;
2998 }
2999
3000 list_for_each_entry_safe(task, task_tmp,
3001 &T_TASK(cmd)->t_task_list, t_list) {
3002
3003 if (!task->task_sense)
3004 continue;
3005
3006 dev = task->se_dev;
3007 if (!(dev))
3008 continue;
3009
3010 if (!TRANSPORT(dev)->get_sense_buffer) {
3011 printk(KERN_ERR "TRANSPORT(dev)->get_sense_buffer"
3012 " is NULL\n");
3013 continue;
3014 }
3015
3016 sense_buffer = TRANSPORT(dev)->get_sense_buffer(task);
3017 if (!(sense_buffer)) {
3018 printk(KERN_ERR "ITT[0x%08x]_TASK[%d]: Unable to locate"
3019 " sense buffer for task with sense\n",
3020 CMD_TFO(cmd)->get_task_tag(cmd), task->task_no);
3021 continue;
3022 }
3023 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
3024
3025 offset = CMD_TFO(cmd)->set_fabric_sense_len(cmd,
3026 TRANSPORT_SENSE_BUFFER);
3027
3028 memcpy((void *)&buffer[offset], (void *)sense_buffer,
3029 TRANSPORT_SENSE_BUFFER);
3030 cmd->scsi_status = task->task_scsi_status;
3031 /* Automatically padded */
3032 cmd->scsi_sense_length =
3033 (TRANSPORT_SENSE_BUFFER + offset);
3034
3035 printk(KERN_INFO "HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x"
3036 " and sense\n",
3037 dev->se_hba->hba_id, TRANSPORT(dev)->name,
3038 cmd->scsi_status);
3039 return 0;
3040 }
3041 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
3042
3043 return -1;
3044}
3045
3046static int transport_allocate_resources(struct se_cmd *cmd)
3047{
3048 u32 length = cmd->data_length;
3049
3050 if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
3051 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB))
3052 return transport_generic_get_mem(cmd, length, PAGE_SIZE);
3053 else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB)
3054 return transport_generic_allocate_buf(cmd, length);
3055 else
3056 return 0;
3057}
3058
3059static int
3060transport_handle_reservation_conflict(struct se_cmd *cmd)
3061{
3062 cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
3063 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3064 cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT;
3065 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
3066 /*
3067 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
3068 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
3069 * CONFLICT STATUS.
3070 *
3071 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
3072 */
3073 if (SE_SESS(cmd) &&
3074 DEV_ATTRIB(cmd->se_dev)->emulate_ua_intlck_ctrl == 2)
3075 core_scsi3_ua_allocate(SE_SESS(cmd)->se_node_acl,
3076 cmd->orig_fe_lun, 0x2C,
3077 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
3078 return -2;
3079}
3080
3081/* transport_generic_cmd_sequencer():
3082 *
3083 * Generic Command Sequencer that should work for most DAS transport
3084 * drivers.
3085 *
3086 * Called from transport_generic_allocate_tasks() in the $FABRIC_MOD
3087 * RX Thread.
3088 *
3089 * FIXME: Need to support other SCSI OPCODES where as well.
3090 */
3091static int transport_generic_cmd_sequencer(
3092 struct se_cmd *cmd,
3093 unsigned char *cdb)
3094{
3095 struct se_device *dev = SE_DEV(cmd);
3096 struct se_subsystem_dev *su_dev = dev->se_sub_dev;
3097 int ret = 0, sector_ret = 0, passthrough;
3098 u32 sectors = 0, size = 0, pr_reg_type = 0;
3099 u16 service_action;
3100 u8 alua_ascq = 0;
3101 /*
3102 * Check for an existing UNIT ATTENTION condition
3103 */
3104 if (core_scsi3_ua_check(cmd, cdb) < 0) {
3105 cmd->transport_wait_for_tasks =
3106 &transport_nop_wait_for_tasks;
3107 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3108 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION;
3109 return -2;
3110 }
3111 /*
3112 * Check status of Asymmetric Logical Unit Assignment port
3113 */
3114 ret = T10_ALUA(su_dev)->alua_state_check(cmd, cdb, &alua_ascq);
3115 if (ret != 0) {
3116 cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
3117 /*
25985edc 3118 * Set SCSI additional sense code (ASC) to 'LUN Not Accessible';
c66ac9db
NB		 3119 * The ALUA additional sense code qualifier (ASCQ) is determined
3120 * by the ALUA primary or secondary access state..
3121 */
3122 if (ret > 0) {
3123#if 0
3124 printk(KERN_INFO "[%s]: ALUA TG Port not available,"
3125 " SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n",
3126 CMD_TFO(cmd)->get_fabric_name(), alua_ascq);
3127#endif
3128 transport_set_sense_codes(cmd, 0x04, alua_ascq);
3129 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3130 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY;
3131 return -2;
3132 }
3133 goto out_invalid_cdb_field;
3134 }
3135 /*
3136 * Check status for SPC-3 Persistent Reservations
3137 */
3138 if (T10_PR_OPS(su_dev)->t10_reservation_check(cmd, &pr_reg_type) != 0) {
3139 if (T10_PR_OPS(su_dev)->t10_seq_non_holder(
3140 cmd, cdb, pr_reg_type) != 0)
3141 return transport_handle_reservation_conflict(cmd);
3142 /*
3143 * This means the CDB is allowed for the SCSI Initiator port
3144 * when said port is *NOT* holding the legacy SPC-2 or
3145 * SPC-3 Persistent Reservation.
3146 */
3147 }
3148
3149 switch (cdb[0]) {
3150 case READ_6:
3151 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
3152 if (sector_ret)
3153 goto out_unsupported_cdb;
3154 size = transport_get_size(sectors, cdb, cmd);
3155 cmd->transport_split_cdb = &split_cdb_XX_6;
3156 T_TASK(cmd)->t_task_lba = transport_lba_21(cdb);
3157 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3158 break;
3159 case READ_10:
3160 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3161 if (sector_ret)
3162 goto out_unsupported_cdb;
3163 size = transport_get_size(sectors, cdb, cmd);
3164 cmd->transport_split_cdb = &split_cdb_XX_10;
3165 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3166 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3167 break;
3168 case READ_12:
3169 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
3170 if (sector_ret)
3171 goto out_unsupported_cdb;
3172 size = transport_get_size(sectors, cdb, cmd);
3173 cmd->transport_split_cdb = &split_cdb_XX_12;
3174 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3175 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3176 break;
3177 case READ_16:
3178 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3179 if (sector_ret)
3180 goto out_unsupported_cdb;
3181 size = transport_get_size(sectors, cdb, cmd);
3182 cmd->transport_split_cdb = &split_cdb_XX_16;
3183 T_TASK(cmd)->t_task_lba = transport_lba_64(cdb);
3184 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3185 break;
3186 case WRITE_6:
3187 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
3188 if (sector_ret)
3189 goto out_unsupported_cdb;
3190 size = transport_get_size(sectors, cdb, cmd);
3191 cmd->transport_split_cdb = &split_cdb_XX_6;
3192 T_TASK(cmd)->t_task_lba = transport_lba_21(cdb);
3193 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3194 break;
3195 case WRITE_10:
3196 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3197 if (sector_ret)
3198 goto out_unsupported_cdb;
3199 size = transport_get_size(sectors, cdb, cmd);
3200 cmd->transport_split_cdb = &split_cdb_XX_10;
3201 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3202 T_TASK(cmd)->t_tasks_fua = (cdb[1] & 0x8);
3203 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3204 break;
3205 case WRITE_12:
3206 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
3207 if (sector_ret)
3208 goto out_unsupported_cdb;
3209 size = transport_get_size(sectors, cdb, cmd);
3210 cmd->transport_split_cdb = &split_cdb_XX_12;
3211 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3212 T_TASK(cmd)->t_tasks_fua = (cdb[1] & 0x8);
3213 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3214 break;
3215 case WRITE_16:
3216 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3217 if (sector_ret)
3218 goto out_unsupported_cdb;
3219 size = transport_get_size(sectors, cdb, cmd);
3220 cmd->transport_split_cdb = &split_cdb_XX_16;
3221 T_TASK(cmd)->t_task_lba = transport_lba_64(cdb);
3222 T_TASK(cmd)->t_tasks_fua = (cdb[1] & 0x8);
3223 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3224 break;
3225 case XDWRITEREAD_10:
3226 if ((cmd->data_direction != DMA_TO_DEVICE) ||
3227 !(T_TASK(cmd)->t_tasks_bidi))
3228 goto out_invalid_cdb_field;
3229 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3230 if (sector_ret)
3231 goto out_unsupported_cdb;
3232 size = transport_get_size(sectors, cdb, cmd);
3233 cmd->transport_split_cdb = &split_cdb_XX_10;
3234 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3235 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3236 passthrough = (TRANSPORT(dev)->transport_type ==
3237 TRANSPORT_PLUGIN_PHBA_PDEV);
3238 /*
3239 * Skip the remaining assignments for TCM/PSCSI passthrough
3240 */
3241 if (passthrough)
3242 break;
3243 /*
3244 * Setup BIDI XOR callback to be run during transport_generic_complete_ok()
3245 */
3246 cmd->transport_complete_callback = &transport_xor_callback;
3247 T_TASK(cmd)->t_tasks_fua = (cdb[1] & 0x8);
3248 break;
3249 case VARIABLE_LENGTH_CMD:
3250 service_action = get_unaligned_be16(&cdb[8]);
3251 /*
3252 * Determine if this is TCM/PSCSI device and we should disable
3253 * internal emulation for this CDB.
3254 */
3255 passthrough = (TRANSPORT(dev)->transport_type ==
3256 TRANSPORT_PLUGIN_PHBA_PDEV);
3257
3258 switch (service_action) {
3259 case XDWRITEREAD_32:
3260 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
3261 if (sector_ret)
3262 goto out_unsupported_cdb;
3263 size = transport_get_size(sectors, cdb, cmd);
3264 /*
3265 * Use WRITE_32 and READ_32 opcodes for the emulated
3266 * XDWRITE_READ_32 logic.
3267 */
3268 cmd->transport_split_cdb = &split_cdb_XX_32;
3269 T_TASK(cmd)->t_task_lba = transport_lba_64_ext(cdb);
3270 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3271
3272 /*
3273 * Skip the remaining assignments for TCM/PSCSI passthrough
3274 */
3275 if (passthrough)
3276 break;
3277
3278 /*
3279 * Setup BIDI XOR callback to be run during
3280 * transport_generic_complete_ok()
3281 */
3282 cmd->transport_complete_callback = &transport_xor_callback;
3283 T_TASK(cmd)->t_tasks_fua = (cdb[10] & 0x8);
3284 break;
3285 case WRITE_SAME_32:
3286 sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
3287 if (sector_ret)
3288 goto out_unsupported_cdb;
3289 size = transport_get_size(sectors, cdb, cmd);
3290 T_TASK(cmd)->t_task_lba = get_unaligned_be64(&cdb[12]);
3291 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3292
3293 /*
3294 * Skip the remaining assignments for TCM/PSCSI passthrough
3295 */
3296 if (passthrough)
3297 break;
3298
3299 if ((cdb[10] & 0x04) || (cdb[10] & 0x02)) {
3300 printk(KERN_ERR "WRITE_SAME PBDATA and LBDATA"
3301 " bits not supported for Block Discard"
3302 " Emulation\n");
3303 goto out_invalid_cdb_field;
3304 }
3305 /*
3306 * Currently for the emulated case we only accept
3307 * tpws with the UNMAP=1 bit set.
3308 */
3309 if (!(cdb[10] & 0x08)) {
3310 printk(KERN_ERR "WRITE_SAME w/o UNMAP bit not"
3311 " supported for Block Discard Emulation\n");
3312 goto out_invalid_cdb_field;
3313 }
3314 break;
3315 default:
3316 printk(KERN_ERR "VARIABLE_LENGTH_CMD service action"
3317 " 0x%04x not supported\n", service_action);
3318 goto out_unsupported_cdb;
3319 }
3320 break;
3321 case 0xa3:
3322 if (TRANSPORT(dev)->get_device_type(dev) != TYPE_ROM) {
3323 /* MAINTENANCE_IN from SCC-2 */
3324 /*
3325 * Check for emulated MI_REPORT_TARGET_PGS.
3326 */
3327 if (cdb[1] == MI_REPORT_TARGET_PGS) {
3328 cmd->transport_emulate_cdb =
3329 (T10_ALUA(su_dev)->alua_type ==
3330 SPC3_ALUA_EMULATED) ?
3331 &core_emulate_report_target_port_groups :
3332 NULL;
3333 }
3334 size = (cdb[6] << 24) | (cdb[7] << 16) |
3335 (cdb[8] << 8) | cdb[9];
3336 } else {
3337 /* GPCMD_SEND_KEY from multi media commands */
3338 size = (cdb[8] << 8) + cdb[9];
3339 }
3340 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3341 break;
3342 case MODE_SELECT:
3343 size = cdb[4];
3344 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3345 break;
3346 case MODE_SELECT_10:
3347 size = (cdb[7] << 8) + cdb[8];
3348 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3349 break;
3350 case MODE_SENSE:
3351 size = cdb[4];
3352 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3353 break;
3354 case MODE_SENSE_10:
3355 case GPCMD_READ_BUFFER_CAPACITY:
3356 case GPCMD_SEND_OPC:
3357 case LOG_SELECT:
3358 case LOG_SENSE:
3359 size = (cdb[7] << 8) + cdb[8];
3360 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3361 break;
3362 case READ_BLOCK_LIMITS:
3363 size = READ_BLOCK_LEN;
3364 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3365 break;
3366 case GPCMD_GET_CONFIGURATION:
3367 case GPCMD_READ_FORMAT_CAPACITIES:
3368 case GPCMD_READ_DISC_INFO:
3369 case GPCMD_READ_TRACK_RZONE_INFO:
3370 size = (cdb[7] << 8) + cdb[8];
3371 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3372 break;
3373 case PERSISTENT_RESERVE_IN:
3374 case PERSISTENT_RESERVE_OUT:
3375 cmd->transport_emulate_cdb =
3376 (T10_RES(su_dev)->res_type ==
3377 SPC3_PERSISTENT_RESERVATIONS) ?
3378 &core_scsi3_emulate_pr : NULL;
3379 size = (cdb[7] << 8) + cdb[8];
3380 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3381 break;
3382 case GPCMD_MECHANISM_STATUS:
3383 case GPCMD_READ_DVD_STRUCTURE:
3384 size = (cdb[8] << 8) + cdb[9];
3385 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3386 break;
3387 case READ_POSITION:
3388 size = READ_POSITION_LEN;
3389 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3390 break;
3391 case 0xa4:
3392 if (TRANSPORT(dev)->get_device_type(dev) != TYPE_ROM) {
3393 /* MAINTENANCE_OUT from SCC-2
3394 *
3395 * Check for emulated MO_SET_TARGET_PGS.
3396 */
3397 if (cdb[1] == MO_SET_TARGET_PGS) {
3398 cmd->transport_emulate_cdb =
3399 (T10_ALUA(su_dev)->alua_type ==
3400 SPC3_ALUA_EMULATED) ?
3401 &core_emulate_set_target_port_groups :
3402 NULL;
3403 }
3404
3405 size = (cdb[6] << 24) | (cdb[7] << 16) |
3406 (cdb[8] << 8) | cdb[9];
3407 } else {
3408 /* GPCMD_REPORT_KEY from multi media commands */
3409 size = (cdb[8] << 8) + cdb[9];
3410 }
3411 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3412 break;
3413 case INQUIRY:
3414 size = (cdb[3] << 8) + cdb[4];
3415 /*
3416 * Do implict HEAD_OF_QUEUE processing for INQUIRY.
3417 * See spc4r17 section 5.3
3418 */
3419 if (SE_DEV(cmd)->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3420 cmd->sam_task_attr = TASK_ATTR_HOQ;
3421 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3422 break;
3423 case READ_BUFFER:
3424 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3425 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3426 break;
3427 case READ_CAPACITY:
3428 size = READ_CAP_LEN;
3429 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3430 break;
3431 case READ_MEDIA_SERIAL_NUMBER:
3432 case SECURITY_PROTOCOL_IN:
3433 case SECURITY_PROTOCOL_OUT:
3434 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3435 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3436 break;
3437 case SERVICE_ACTION_IN:
3438 case ACCESS_CONTROL_IN:
3439 case ACCESS_CONTROL_OUT:
3440 case EXTENDED_COPY:
3441 case READ_ATTRIBUTE:
3442 case RECEIVE_COPY_RESULTS:
3443 case WRITE_ATTRIBUTE:
3444 size = (cdb[10] << 24) | (cdb[11] << 16) |
3445 (cdb[12] << 8) | cdb[13];
3446 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3447 break;
3448 case RECEIVE_DIAGNOSTIC:
3449 case SEND_DIAGNOSTIC:
3450 size = (cdb[3] << 8) | cdb[4];
3451 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3452 break;
3453/* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */
3454#if 0
3455 case GPCMD_READ_CD:
3456 sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3457 size = (2336 * sectors);
3458 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3459 break;
3460#endif
3461 case READ_TOC:
3462 size = cdb[8];
3463 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3464 break;
3465 case REQUEST_SENSE:
3466 size = cdb[4];
3467 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3468 break;
3469 case READ_ELEMENT_STATUS:
3470 size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9];
3471 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3472 break;
3473 case WRITE_BUFFER:
3474 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3475 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3476 break;
3477 case RESERVE:
3478 case RESERVE_10:
3479 /*
3480 * The SPC-2 RESERVE does not contain a size in the SCSI CDB.
3481 * Assume the passthrough or $FABRIC_MOD will tell us about it.
3482 */
3483 if (cdb[0] == RESERVE_10)
3484 size = (cdb[7] << 8) | cdb[8];
3485 else
3486 size = cmd->data_length;
3487
3488 /*
3489 * Setup the legacy emulated handler for SPC-2 and
3490 * >= SPC-3 compatible reservation handling (CRH=1)
3491 * Otherwise, we assume the underlying SCSI logic is
3492 * is running in SPC_PASSTHROUGH, and wants reservations
3493 * emulation disabled.
3494 */
3495 cmd->transport_emulate_cdb =
3496 (T10_RES(su_dev)->res_type !=
3497 SPC_PASSTHROUGH) ?
3498 &core_scsi2_emulate_crh : NULL;
3499 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3500 break;
3501 case RELEASE:
3502 case RELEASE_10:
3503 /*
3504 * The SPC-2 RELEASE does not contain a size in the SCSI CDB.
3505 * Assume the passthrough or $FABRIC_MOD will tell us about it.
3506 */
3507 if (cdb[0] == RELEASE_10)
3508 size = (cdb[7] << 8) | cdb[8];
3509 else
3510 size = cmd->data_length;
3511
3512 cmd->transport_emulate_cdb =
3513 (T10_RES(su_dev)->res_type !=
3514 SPC_PASSTHROUGH) ?
3515 &core_scsi2_emulate_crh : NULL;
3516 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3517 break;
3518 case SYNCHRONIZE_CACHE:
3519 case 0x91: /* SYNCHRONIZE_CACHE_16: */
3520 /*
3521 * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE
3522 */
3523 if (cdb[0] == SYNCHRONIZE_CACHE) {
3524 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3525 T_TASK(cmd)->t_task_lba = transport_lba_32(cdb);
3526 } else {
3527 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3528 T_TASK(cmd)->t_task_lba = transport_lba_64(cdb);
3529 }
3530 if (sector_ret)
3531 goto out_unsupported_cdb;
3532
3533 size = transport_get_size(sectors, cdb, cmd);
3534 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3535
3536 /*
3537 * For TCM/pSCSI passthrough, skip cmd->transport_emulate_cdb()
3538 */
3539 if (TRANSPORT(dev)->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
3540 break;
3541 /*
3542 * Set SCF_EMULATE_CDB_ASYNC to ensure asynchronous operation
3543 * for SYNCHRONIZE_CACHE* Immed=1 case in __transport_execute_tasks()
3544 */
3545 cmd->se_cmd_flags |= SCF_EMULATE_CDB_ASYNC;
3546 /*
3547 * Check to ensure that LBA + Range does not exceed past end of
3548 * device.
3549 */
3550 if (transport_get_sectors(cmd) < 0)
3551 goto out_invalid_cdb_field;
3552 break;
3553 case UNMAP:
3554 size = get_unaligned_be16(&cdb[7]);
3555 passthrough = (TRANSPORT(dev)->transport_type ==
3556 TRANSPORT_PLUGIN_PHBA_PDEV);
3557 /*
3558 * Determine if the received UNMAP used to for direct passthrough
3559 * into Linux/SCSI with struct request via TCM/pSCSI or we are
3560 * signaling the use of internal transport_generic_unmap() emulation
3561 * for UNMAP -> Linux/BLOCK disbard with TCM/IBLOCK and TCM/FILEIO
3562 * subsystem plugin backstores.
3563 */
3564 if (!(passthrough))
3565 cmd->se_cmd_flags |= SCF_EMULATE_SYNC_UNMAP;
3566
3567 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3568 break;
3569 case WRITE_SAME_16:
3570 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3571 if (sector_ret)
3572 goto out_unsupported_cdb;
3573 size = transport_get_size(sectors, cdb, cmd);
3574 T_TASK(cmd)->t_task_lba = get_unaligned_be16(&cdb[2]);
3575 passthrough = (TRANSPORT(dev)->transport_type ==
3576 TRANSPORT_PLUGIN_PHBA_PDEV);
3577 /*
3578 * Determine if the received WRITE_SAME_16 is used to for direct
3579 * passthrough into Linux/SCSI with struct request via TCM/pSCSI
3580 * or we are signaling the use of internal WRITE_SAME + UNMAP=1
3581 * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK and
3582 * TCM/FILEIO subsystem plugin backstores.
3583 */
3584 if (!(passthrough)) {
3585 if ((cdb[1] & 0x04) || (cdb[1] & 0x02)) {
3586 printk(KERN_ERR "WRITE_SAME PBDATA and LBDATA"
3587 " bits not supported for Block Discard"
3588 " Emulation\n");
3589 goto out_invalid_cdb_field;
3590 }
3591 /*
3592 * Currently for the emulated case we only accept
3593 * tpws with the UNMAP=1 bit set.
3594 */
3595 if (!(cdb[1] & 0x08)) {
3596 printk(KERN_ERR "WRITE_SAME w/o UNMAP bit not "
3597 " supported for Block Discard Emulation\n");
3598 goto out_invalid_cdb_field;
3599 }
3600 }
3601 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3602 break;
3603 case ALLOW_MEDIUM_REMOVAL:
3604 case GPCMD_CLOSE_TRACK:
3605 case ERASE:
3606 case INITIALIZE_ELEMENT_STATUS:
3607 case GPCMD_LOAD_UNLOAD:
3608 case REZERO_UNIT:
3609 case SEEK_10:
3610 case GPCMD_SET_SPEED:
3611 case SPACE:
3612 case START_STOP:
3613 case TEST_UNIT_READY:
3614 case VERIFY:
3615 case WRITE_FILEMARKS:
3616 case MOVE_MEDIUM:
3617 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3618 break;
3619 case REPORT_LUNS:
3620 cmd->transport_emulate_cdb =
3621 &transport_core_report_lun_response;
3622 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3623 /*
3624 * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS
3625 * See spc4r17 section 5.3
3626 */
3627 if (SE_DEV(cmd)->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3628 cmd->sam_task_attr = TASK_ATTR_HOQ;
3629 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3630 break;
3631 default:
3632 printk(KERN_WARNING "TARGET_CORE[%s]: Unsupported SCSI Opcode"
3633 " 0x%02x, sending CHECK_CONDITION.\n",
3634 CMD_TFO(cmd)->get_fabric_name(), cdb[0]);
3635 cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
3636 goto out_unsupported_cdb;
3637 }
3638
3639 if (size != cmd->data_length) {
3640 printk(KERN_WARNING "TARGET_CORE[%s]: Expected Transfer Length:"
3641 " %u does not match SCSI CDB Length: %u for SAM Opcode:"
3642 " 0x%02x\n", CMD_TFO(cmd)->get_fabric_name(),
3643 cmd->data_length, size, cdb[0]);
3644
3645 cmd->cmd_spdtl = size;
3646
3647 if (cmd->data_direction == DMA_TO_DEVICE) {
3648 printk(KERN_ERR "Rejecting underflow/overflow"
3649 " WRITE data\n");
3650 goto out_invalid_cdb_field;
3651 }
3652 /*
3653 * Reject READ_* or WRITE_* with overflow/underflow for
3654 * type SCF_SCSI_DATA_SG_IO_CDB.
3655 */
3656 if (!(ret) && (DEV_ATTRIB(dev)->block_size != 512)) {
3657 printk(KERN_ERR "Failing OVERFLOW/UNDERFLOW for LBA op"
3658 " CDB on non 512-byte sector setup subsystem"
3659 " plugin: %s\n", TRANSPORT(dev)->name);
3660 /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
3661 goto out_invalid_cdb_field;
3662 }
3663
3664 if (size > cmd->data_length) {
3665 cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
3666 cmd->residual_count = (size - cmd->data_length);
3667 } else {
3668 cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
3669 cmd->residual_count = (cmd->data_length - size);
3670 }
3671 cmd->data_length = size;
3672 }
3673
3674 transport_set_supported_SAM_opcode(cmd);
3675 return ret;
3676
3677out_unsupported_cdb:
3678 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3679 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
3680 return -2;
3681out_invalid_cdb_field:
3682 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3683 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
3684 return -2;
3685}
3686
3687static inline void transport_release_tasks(struct se_cmd *);
3688
3689/*
3690 * This function will copy a contiguous *src buffer into a destination
3691 * struct scatterlist array.
3692 */
3693static void transport_memcpy_write_contig(
3694 struct se_cmd *cmd,
3695 struct scatterlist *sg_d,
3696 unsigned char *src)
3697{
3698 u32 i = 0, length = 0, total_length = cmd->data_length;
3699 void *dst;
3700
3701 while (total_length) {
3702 length = sg_d[i].length;
3703
3704 if (length > total_length)
3705 length = total_length;
3706
3707 dst = sg_virt(&sg_d[i]);
3708
3709 memcpy(dst, src, length);
3710
3711 if (!(total_length -= length))
3712 return;
3713
3714 src += length;
3715 i++;
3716 }
3717}
3718
3719/*
3720 * This function will copy a struct scatterlist array *sg_s into a destination
3721 * contiguous *dst buffer.
3722 */
3723static void transport_memcpy_read_contig(
3724 struct se_cmd *cmd,
3725 unsigned char *dst,
3726 struct scatterlist *sg_s)
3727{
3728 u32 i = 0, length = 0, total_length = cmd->data_length;
3729 void *src;
3730
3731 while (total_length) {
3732 length = sg_s[i].length;
3733
3734 if (length > total_length)
3735 length = total_length;
3736
3737 src = sg_virt(&sg_s[i]);
3738
3739 memcpy(dst, src, length);
3740
3741 if (!(total_length -= length))
3742 return;
3743
3744 dst += length;
3745 i++;
3746 }
3747}
3748
3749static void transport_memcpy_se_mem_read_contig(
3750 struct se_cmd *cmd,
3751 unsigned char *dst,
3752 struct list_head *se_mem_list)
3753{
3754 struct se_mem *se_mem;
3755 void *src;
3756 u32 length = 0, total_length = cmd->data_length;
3757
3758 list_for_each_entry(se_mem, se_mem_list, se_list) {
3759 length = se_mem->se_len;
3760
3761 if (length > total_length)
3762 length = total_length;
3763
3764 src = page_address(se_mem->se_page) + se_mem->se_off;
3765
3766 memcpy(dst, src, length);
3767
3768 if (!(total_length -= length))
3769 return;
3770
3771 dst += length;
3772 }
3773}
3774
3775/*
3776 * Called from transport_generic_complete_ok() and
3777 * transport_generic_request_failure() to determine which dormant/delayed
3778 * and ordered cmds need to have their tasks added to the execution queue.
3779 */
3780static void transport_complete_task_attr(struct se_cmd *cmd)
3781{
3782 struct se_device *dev = SE_DEV(cmd);
3783 struct se_cmd *cmd_p, *cmd_tmp;
3784 int new_active_tasks = 0;
3785
3786 if (cmd->sam_task_attr == TASK_ATTR_SIMPLE) {
3787 atomic_dec(&dev->simple_cmds);
3788 smp_mb__after_atomic_dec();
3789 dev->dev_cur_ordered_id++;
3790 DEBUG_STA("Incremented dev->dev_cur_ordered_id: %u for"
3791 " SIMPLE: %u\n", dev->dev_cur_ordered_id,
3792 cmd->se_ordered_id);
3793 } else if (cmd->sam_task_attr == TASK_ATTR_HOQ) {
3794 atomic_dec(&dev->dev_hoq_count);
3795 smp_mb__after_atomic_dec();
3796 dev->dev_cur_ordered_id++;
3797 DEBUG_STA("Incremented dev_cur_ordered_id: %u for"
3798 " HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
3799 cmd->se_ordered_id);
3800 } else if (cmd->sam_task_attr == TASK_ATTR_ORDERED) {
3801 spin_lock(&dev->ordered_cmd_lock);
3802 list_del(&cmd->se_ordered_list);
3803 atomic_dec(&dev->dev_ordered_sync);
3804 smp_mb__after_atomic_dec();
3805 spin_unlock(&dev->ordered_cmd_lock);
3806
3807 dev->dev_cur_ordered_id++;
3808 DEBUG_STA("Incremented dev_cur_ordered_id: %u for ORDERED:"
3809 " %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
3810 }
3811 /*
3812 * Process all commands up to the last received
3813 * ORDERED task attribute which requires another blocking
3814 * boundary
3815 */
3816 spin_lock(&dev->delayed_cmd_lock);
3817 list_for_each_entry_safe(cmd_p, cmd_tmp,
3818 &dev->delayed_cmd_list, se_delayed_list) {
3819
3820 list_del(&cmd_p->se_delayed_list);
3821 spin_unlock(&dev->delayed_cmd_lock);
3822
3823 DEBUG_STA("Calling add_tasks() for"
3824 " cmd_p: 0x%02x Task Attr: 0x%02x"
3825 " Dormant -> Active, se_ordered_id: %u\n",
3826 T_TASK(cmd_p)->t_task_cdb[0],
3827 cmd_p->sam_task_attr, cmd_p->se_ordered_id);
3828
3829 transport_add_tasks_from_cmd(cmd_p);
3830 new_active_tasks++;
3831
3832 spin_lock(&dev->delayed_cmd_lock);
3833 if (cmd_p->sam_task_attr == TASK_ATTR_ORDERED)
3834 break;
3835 }
3836 spin_unlock(&dev->delayed_cmd_lock);
3837 /*
3838 * If new tasks have become active, wake up the transport thread
3839 * to do the processing of the Active tasks.
3840 */
3841 if (new_active_tasks != 0)
3842 wake_up_interruptible(&dev->dev_queue_obj->thread_wq);
3843}
3844
3845static void transport_generic_complete_ok(struct se_cmd *cmd)
3846{
3847 int reason = 0;
3848 /*
3849 * Check if we need to move delayed/dormant tasks from cmds on the
3850 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
3851 * Attribute.
3852 */
3853 if (SE_DEV(cmd)->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3854 transport_complete_task_attr(cmd);
3855 /*
3856 * Check if we need to retrieve a sense buffer from
3857 * the struct se_cmd in question.
3858 */
3859 if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3860 if (transport_get_sense_data(cmd) < 0)
3861 reason = TCM_NON_EXISTENT_LUN;
3862
3863 /*
3864 * Only set when an struct se_task->task_scsi_status returned
3865 * a non GOOD status.
3866 */
3867 if (cmd->scsi_status) {
3868 transport_send_check_condition_and_sense(
3869 cmd, reason, 1);
3870 transport_lun_remove_cmd(cmd);
3871 transport_cmd_check_stop_to_fabric(cmd);
3872 return;
3873 }
3874 }
3875 /*
25985edc 3876 * Check for a callback, used by amongst other things
c66ac9db
NB		 3877 * XDWRITE_READ_10 emulation.
3878 */
3879 if (cmd->transport_complete_callback)
3880 cmd->transport_complete_callback(cmd);
3881
3882 switch (cmd->data_direction) {
3883 case DMA_FROM_DEVICE:
3884 spin_lock(&cmd->se_lun->lun_sep_lock);
3885 if (SE_LUN(cmd)->lun_sep) {
3886 SE_LUN(cmd)->lun_sep->sep_stats.tx_data_octets +=
3887 cmd->data_length;
3888 }
3889 spin_unlock(&cmd->se_lun->lun_sep_lock);
3890 /*
3891 * If enabled by TCM fabirc module pre-registered SGL
3892 * memory, perform the memcpy() from the TCM internal
3893 * contigious buffer back to the original SGL.
3894 */
3895 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_CONTIG_TO_SG)
3896 transport_memcpy_write_contig(cmd,
3897 T_TASK(cmd)->t_task_pt_sgl,
3898 T_TASK(cmd)->t_task_buf);
3899
3900 CMD_TFO(cmd)->queue_data_in(cmd);
3901 break;
3902 case DMA_TO_DEVICE:
3903 spin_lock(&cmd->se_lun->lun_sep_lock);
3904 if (SE_LUN(cmd)->lun_sep) {
3905 SE_LUN(cmd)->lun_sep->sep_stats.rx_data_octets +=
3906 cmd->data_length;
3907 }
3908 spin_unlock(&cmd->se_lun->lun_sep_lock);
3909 /*
3910 * Check if we need to send READ payload for BIDI-COMMAND
3911 */
3912 if (T_TASK(cmd)->t_mem_bidi_list != NULL) {
3913 spin_lock(&cmd->se_lun->lun_sep_lock);
3914 if (SE_LUN(cmd)->lun_sep) {
3915 SE_LUN(cmd)->lun_sep->sep_stats.tx_data_octets +=
3916 cmd->data_length;
3917 }
3918 spin_unlock(&cmd->se_lun->lun_sep_lock);
3919 CMD_TFO(cmd)->queue_data_in(cmd);
3920 break;
3921 }
3922 /* Fall through for DMA_TO_DEVICE */
3923 case DMA_NONE:
3924 CMD_TFO(cmd)->queue_status(cmd);
3925 break;
3926 default:
3927 break;
3928 }
3929
3930 transport_lun_remove_cmd(cmd);
3931 transport_cmd_check_stop_to_fabric(cmd);
3932}
3933
3934static void transport_free_dev_tasks(struct se_cmd *cmd)
3935{
3936 struct se_task *task, *task_tmp;
3937 unsigned long flags;
3938
3939 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
3940 list_for_each_entry_safe(task, task_tmp,
3941 &T_TASK(cmd)->t_task_list, t_list) {
3942 if (atomic_read(&task->task_active))
3943 continue;
3944
3945 kfree(task->task_sg_bidi);
3946 kfree(task->task_sg);
3947
3948 list_del(&task->t_list);
3949
3950 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
3951 if (task->se_dev)
3952 TRANSPORT(task->se_dev)->free_task(task);
3953 else
3954 printk(KERN_ERR "task[%u] - task->se_dev is NULL\n",
3955 task->task_no);
3956 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
3957 }
3958 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
3959}
3960
3961static inline void transport_free_pages(struct se_cmd *cmd)
3962{
3963 struct se_mem *se_mem, *se_mem_tmp;
3964 int free_page = 1;
3965
3966 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
3967 free_page = 0;
3968 if (cmd->se_dev->transport->do_se_mem_map)
3969 free_page = 0;
3970
3971 if (T_TASK(cmd)->t_task_buf) {
3972 kfree(T_TASK(cmd)->t_task_buf);
3973 T_TASK(cmd)->t_task_buf = NULL;
3974 return;
3975 }
3976
3977 /*
3978 * Caller will handle releasing of struct se_mem.
3979 */
3980 if (cmd->se_cmd_flags & SCF_CMD_PASSTHROUGH_NOALLOC)
3981 return;
3982
3983 if (!(T_TASK(cmd)->t_tasks_se_num))
3984 return;
3985
3986 list_for_each_entry_safe(se_mem, se_mem_tmp,
3987 T_TASK(cmd)->t_mem_list, se_list) {
3988 /*
3989 * We only release call __free_page(struct se_mem->se_page) when
3990 * SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC is NOT in use,
3991 */
3992 if (free_page)
3993 __free_page(se_mem->se_page);
3994
3995 list_del(&se_mem->se_list);
3996 kmem_cache_free(se_mem_cache, se_mem);
3997 }
3998
3999 if (T_TASK(cmd)->t_mem_bidi_list && T_TASK(cmd)->t_tasks_se_bidi_num) {
4000 list_for_each_entry_safe(se_mem, se_mem_tmp,
4001 T_TASK(cmd)->t_mem_bidi_list, se_list) {
4002 /*
4003 * We only release call __free_page(struct se_mem->se_page) when
4004 * SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC is NOT in use,
4005 */
4006 if (free_page)
4007 __free_page(se_mem->se_page);
4008
4009 list_del(&se_mem->se_list);
4010 kmem_cache_free(se_mem_cache, se_mem);
4011 }
4012 }
4013
4014 kfree(T_TASK(cmd)->t_mem_bidi_list);
4015 T_TASK(cmd)->t_mem_bidi_list = NULL;
4016 kfree(T_TASK(cmd)->t_mem_list);
4017 T_TASK(cmd)->t_mem_list = NULL;
4018 T_TASK(cmd)->t_tasks_se_num = 0;
4019}
4020
4021static inline void transport_release_tasks(struct se_cmd *cmd)
4022{
4023 transport_free_dev_tasks(cmd);
4024}
4025
4026static inline int transport_dec_and_check(struct se_cmd *cmd)
4027{
4028 unsigned long flags;
4029
4030 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
4031 if (atomic_read(&T_TASK(cmd)->t_fe_count)) {
4032 if (!(atomic_dec_and_test(&T_TASK(cmd)->t_fe_count))) {
4033 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
4034 flags);
4035 return 1;
4036 }
4037 }
4038
4039 if (atomic_read(&T_TASK(cmd)->t_se_count)) {
4040 if (!(atomic_dec_and_test(&T_TASK(cmd)->t_se_count))) {
4041 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
4042 flags);
4043 return 1;
4044 }
4045 }
4046 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
4047
4048 return 0;
4049}
4050
4051static void transport_release_fe_cmd(struct se_cmd *cmd)
4052{
4053 unsigned long flags;
4054
4055 if (transport_dec_and_check(cmd))
4056 return;
4057
4058 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
4059 if (!(atomic_read(&T_TASK(cmd)->transport_dev_active))) {
4060 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
4061 goto free_pages;
4062 }
4063 atomic_set(&T_TASK(cmd)->transport_dev_active, 0);
4064 transport_all_task_dev_remove_state(cmd);
4065 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
4066
4067 transport_release_tasks(cmd);
4068free_pages:
4069 transport_free_pages(cmd);
4070 transport_free_se_cmd(cmd);
4071 CMD_TFO(cmd)->release_cmd_direct(cmd);
4072}
4073
4074static int transport_generic_remove(
4075 struct se_cmd *cmd,
4076 int release_to_pool,
4077 int session_reinstatement)
4078{
4079 unsigned long flags;
4080
4081 if (!(T_TASK(cmd)))
4082 goto release_cmd;
4083
4084 if (transport_dec_and_check(cmd)) {
4085 if (session_reinstatement) {
4086 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
4087 transport_all_task_dev_remove_state(cmd);
4088 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
4089 flags);
4090 }
4091 return 1;
4092 }
4093
4094 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
4095 if (!(atomic_read(&T_TASK(cmd)->transport_dev_active))) {
4096 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
4097 goto free_pages;
4098 }
4099 atomic_set(&T_TASK(cmd)->transport_dev_active, 0);
4100 transport_all_task_dev_remove_state(cmd);
4101 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
4102
4103 transport_release_tasks(cmd);
4104free_pages:
4105 transport_free_pages(cmd);
4106
4107release_cmd:
4108 if (release_to_pool) {
4109 transport_release_cmd_to_pool(cmd);
4110 } else {
4111 transport_free_se_cmd(cmd);
4112 CMD_TFO(cmd)->release_cmd_direct(cmd);
4113 }
4114
4115 return 0;
4116}
4117
4118/*
4119 * transport_generic_map_mem_to_cmd - Perform SGL -> struct se_mem map
4120 * @cmd: Associated se_cmd descriptor
4121 * @mem: SGL style memory for TCM WRITE / READ
4122 * @sg_mem_num: Number of SGL elements
4123 * @mem_bidi_in: SGL style memory for TCM BIDI READ
4124 * @sg_mem_bidi_num: Number of BIDI READ SGL elements
4125 *
4126 * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage
4127 * of parameters.
4128 */
4129int transport_generic_map_mem_to_cmd(
4130 struct se_cmd *cmd,
4131 struct scatterlist *mem,
4132 u32 sg_mem_num,
4133 struct scatterlist *mem_bidi_in,
4134 u32 sg_mem_bidi_num)
4135{
4136 u32 se_mem_cnt_out = 0;
4137 int ret;
4138
4139 if (!(mem) || !(sg_mem_num))
4140 return 0;
4141 /*
4142 * Passed *mem will contain a list_head containing preformatted
4143 * struct se_mem elements...
4144 */
4145 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM)) {
4146 if ((mem_bidi_in) || (sg_mem_bidi_num)) {
4147 printk(KERN_ERR "SCF_CMD_PASSTHROUGH_NOALLOC not supported"
4148 " with BIDI-COMMAND\n");
4149 return -ENOSYS;
4150 }
4151
4152 T_TASK(cmd)->t_mem_list = (struct list_head *)mem;
4153 T_TASK(cmd)->t_tasks_se_num = sg_mem_num;
4154 cmd->se_cmd_flags |= SCF_CMD_PASSTHROUGH_NOALLOC;
4155 return 0;
4156 }
4157 /*
4158 * Otherwise, assume the caller is passing a struct scatterlist
4159 * array from include/linux/scatterlist.h
4160 */
4161 if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
4162 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) {
4163 /*
4164 * For CDB using TCM struct se_mem linked list scatterlist memory
4165 * processed into a TCM struct se_subsystem_dev, we do the mapping
4166 * from the passed physical memory to struct se_mem->se_page here.
4167 */
4168 T_TASK(cmd)->t_mem_list = transport_init_se_mem_list();
4169 if (!(T_TASK(cmd)->t_mem_list))
4170 return -ENOMEM;
4171
4172 ret = transport_map_sg_to_mem(cmd,
4173 T_TASK(cmd)->t_mem_list, mem, &se_mem_cnt_out);
4174 if (ret < 0)
4175 return -ENOMEM;
4176
4177 T_TASK(cmd)->t_tasks_se_num = se_mem_cnt_out;
4178 /*
4179 * Setup BIDI READ list of struct se_mem elements
4180 */
4181 if ((mem_bidi_in) && (sg_mem_bidi_num)) {
4182 T_TASK(cmd)->t_mem_bidi_list = transport_init_se_mem_list();
4183 if (!(T_TASK(cmd)->t_mem_bidi_list)) {
4184 kfree(T_TASK(cmd)->t_mem_list);
4185 return -ENOMEM;
4186 }
4187 se_mem_cnt_out = 0;
4188
4189 ret = transport_map_sg_to_mem(cmd,
4190 T_TASK(cmd)->t_mem_bidi_list, mem_bidi_in,
4191 &se_mem_cnt_out);
4192 if (ret < 0) {
4193 kfree(T_TASK(cmd)->t_mem_list);
4194 return -ENOMEM;
4195 }
4196
4197 T_TASK(cmd)->t_tasks_se_bidi_num = se_mem_cnt_out;
4198 }
4199 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
4200
4201 } else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB) {
4202 if (mem_bidi_in || sg_mem_bidi_num) {
4203 printk(KERN_ERR "BIDI-Commands not supported using "
4204 "SCF_SCSI_CONTROL_NONSG_IO_CDB\n");
4205 return -ENOSYS;
4206 }
4207 /*
4208 * For incoming CDBs using a contiguous buffer internall with TCM,
4209 * save the passed struct scatterlist memory. After TCM storage object
4210 * processing has completed for this struct se_cmd, TCM core will call
4211 * transport_memcpy_[write,read]_contig() as necessary from
4212 * transport_generic_complete_ok() and transport_write_pending() in order
4213 * to copy the TCM buffer to/from the original passed *mem in SGL ->
4214 * struct scatterlist format.
4215 */
4216 cmd->se_cmd_flags |= SCF_PASSTHROUGH_CONTIG_TO_SG;
4217 T_TASK(cmd)->t_task_pt_sgl = mem;
4218 }
4219
4220 return 0;
4221}
4222EXPORT_SYMBOL(transport_generic_map_mem_to_cmd);
4223
4224
4225static inline long long transport_dev_end_lba(struct se_device *dev)
4226{
4227 return dev->transport->get_blocks(dev) + 1;
4228}
4229
4230static int transport_get_sectors(struct se_cmd *cmd)
4231{
4232 struct se_device *dev = SE_DEV(cmd);
4233
4234 T_TASK(cmd)->t_tasks_sectors =
4235 (cmd->data_length / DEV_ATTRIB(dev)->block_size);
4236 if (!(T_TASK(cmd)->t_tasks_sectors))
4237 T_TASK(cmd)->t_tasks_sectors = 1;
4238
4239 if (TRANSPORT(dev)->get_device_type(dev) != TYPE_DISK)
4240 return 0;
4241
4242 if ((T_TASK(cmd)->t_task_lba + T_TASK(cmd)->t_tasks_sectors) >
4243 transport_dev_end_lba(dev)) {
4244 printk(KERN_ERR "LBA: %llu Sectors: %u exceeds"
4245 " transport_dev_end_lba(): %llu\n",
4246 T_TASK(cmd)->t_task_lba, T_TASK(cmd)->t_tasks_sectors,
4247 transport_dev_end_lba(dev));
4248 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4249 cmd->scsi_sense_reason = TCM_SECTOR_COUNT_TOO_MANY;
4250 return PYX_TRANSPORT_REQ_TOO_MANY_SECTORS;
4251 }
4252
4253 return 0;
4254}
4255
4256static int transport_new_cmd_obj(struct se_cmd *cmd)
4257{
4258 struct se_device *dev = SE_DEV(cmd);
4259 u32 task_cdbs = 0, rc;
4260
4261 if (!(cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)) {
4262 task_cdbs++;
4263 T_TASK(cmd)->t_task_cdbs++;
4264 } else {
4265 int set_counts = 1;
4266
4267 /*
4268 * Setup any BIDI READ tasks and memory from
4269 * T_TASK(cmd)->t_mem_bidi_list so the READ struct se_tasks
4270 * are queued first for the non pSCSI passthrough case.
4271 */
4272 if ((T_TASK(cmd)->t_mem_bidi_list != NULL) &&
4273 (TRANSPORT(dev)->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV)) {
4274 rc = transport_generic_get_cdb_count(cmd,
4275 T_TASK(cmd)->t_task_lba,
4276 T_TASK(cmd)->t_tasks_sectors,
4277 DMA_FROM_DEVICE, T_TASK(cmd)->t_mem_bidi_list,
4278 set_counts);
4279 if (!(rc)) {
4280 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4281 cmd->scsi_sense_reason =
4282 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
4283 return PYX_TRANSPORT_LU_COMM_FAILURE;
4284 }
4285 set_counts = 0;
4286 }
4287 /*
4288 * Setup the tasks and memory from T_TASK(cmd)->t_mem_list
4289 * Note for BIDI transfers this will contain the WRITE payload
4290 */
4291 task_cdbs = transport_generic_get_cdb_count(cmd,
4292 T_TASK(cmd)->t_task_lba,
4293 T_TASK(cmd)->t_tasks_sectors,
4294 cmd->data_direction, T_TASK(cmd)->t_mem_list,
4295 set_counts);
4296 if (!(task_cdbs)) {
4297 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4298 cmd->scsi_sense_reason =
4299 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
4300 return PYX_TRANSPORT_LU_COMM_FAILURE;
4301 }
4302 T_TASK(cmd)->t_task_cdbs += task_cdbs;
4303
4304#if 0
4305 printk(KERN_INFO "data_length: %u, LBA: %llu t_tasks_sectors:"
4306 " %u, t_task_cdbs: %u\n", obj_ptr, cmd->data_length,
4307 T_TASK(cmd)->t_task_lba, T_TASK(cmd)->t_tasks_sectors,
4308 T_TASK(cmd)->t_task_cdbs);
4309#endif
4310 }
4311
4312 atomic_set(&T_TASK(cmd)->t_task_cdbs_left, task_cdbs);
4313 atomic_set(&T_TASK(cmd)->t_task_cdbs_ex_left, task_cdbs);
4314 atomic_set(&T_TASK(cmd)->t_task_cdbs_timeout_left, task_cdbs);
4315 return 0;
4316}
4317
4318static struct list_head *transport_init_se_mem_list(void)
4319{
4320 struct list_head *se_mem_list;
4321
4322 se_mem_list = kzalloc(sizeof(struct list_head), GFP_KERNEL);
4323 if (!(se_mem_list)) {
4324 printk(KERN_ERR "Unable to allocate memory for se_mem_list\n");
4325 return NULL;
4326 }
4327 INIT_LIST_HEAD(se_mem_list);
4328
4329 return se_mem_list;
4330}
4331
4332static int
4333transport_generic_get_mem(struct se_cmd *cmd, u32 length, u32 dma_size)
4334{
4335 unsigned char *buf;
4336 struct se_mem *se_mem;
4337
4338 T_TASK(cmd)->t_mem_list = transport_init_se_mem_list();
4339 if (!(T_TASK(cmd)->t_mem_list))
4340 return -ENOMEM;
4341
4342 /*
4343 * If the device uses memory mapping this is enough.
4344 */
4345 if (cmd->se_dev->transport->do_se_mem_map)
4346 return 0;
4347
4348 /*
4349 * Setup BIDI-COMMAND READ list of struct se_mem elements
4350 */
4351 if (T_TASK(cmd)->t_tasks_bidi) {
4352 T_TASK(cmd)->t_mem_bidi_list = transport_init_se_mem_list();
4353 if (!(T_TASK(cmd)->t_mem_bidi_list)) {
4354 kfree(T_TASK(cmd)->t_mem_list);
4355 return -ENOMEM;
4356 }
4357 }
4358
4359 while (length) {
4360 se_mem = kmem_cache_zalloc(se_mem_cache, GFP_KERNEL);
4361 if (!(se_mem)) {
4362 printk(KERN_ERR "Unable to allocate struct se_mem\n");
4363 goto out;
4364 }
c66ac9db
NB		 4365
4366/* #warning FIXME Allocate contigous pages for struct se_mem elements */
87210568 4367 se_mem->se_page = alloc_pages(GFP_KERNEL, 0);
c66ac9db
NB		 4368 if (!(se_mem->se_page)) {
4369 printk(KERN_ERR "alloc_pages() failed\n");
4370 goto out;
4371 }
4372
4373 buf = kmap_atomic(se_mem->se_page, KM_IRQ0);
4374 if (!(buf)) {
4375 printk(KERN_ERR "kmap_atomic() failed\n");
4376 goto out;
4377 }
87210568
JJ		 4378 INIT_LIST_HEAD(&se_mem->se_list);
4379 se_mem->se_len = (length > dma_size) ? dma_size : length;
c66ac9db
NB		 4380 memset(buf, 0, se_mem->se_len);
4381 kunmap_atomic(buf, KM_IRQ0);
4382
4383 list_add_tail(&se_mem->se_list, T_TASK(cmd)->t_mem_list);
4384 T_TASK(cmd)->t_tasks_se_num++;
4385
4386 DEBUG_MEM("Allocated struct se_mem page(%p) Length(%u)"
4387 " Offset(%u)\n", se_mem->se_page, se_mem->se_len,
4388 se_mem->se_off);
4389
4390 length -= se_mem->se_len;
4391 }
4392
4393 DEBUG_MEM("Allocated total struct se_mem elements(%u)\n",
4394 T_TASK(cmd)->t_tasks_se_num);
4395
4396 return 0;
4397out:
87210568
JJ		 4398 if (se_mem)
4399 __free_pages(se_mem->se_page, 0);
4400 kmem_cache_free(se_mem_cache, se_mem);
c66ac9db
NB		 4401 return -1;
4402}
4403
5dd7ed2e 4404u32 transport_calc_sg_num(
c66ac9db
NB		 4405 struct se_task *task,
4406 struct se_mem *in_se_mem,
4407 u32 task_offset)
4408{
4409 struct se_cmd *se_cmd = task->task_se_cmd;
4410 struct se_device *se_dev = SE_DEV(se_cmd);
4411 struct se_mem *se_mem = in_se_mem;
4412 struct target_core_fabric_ops *tfo = CMD_TFO(se_cmd);
4413 u32 sg_length, task_size = task->task_size, task_sg_num_padded;
4414
4415 while (task_size != 0) {
4416 DEBUG_SC("se_mem->se_page(%p) se_mem->se_len(%u)"
4417 " se_mem->se_off(%u) task_offset(%u)\n",
4418 se_mem->se_page, se_mem->se_len,
4419 se_mem->se_off, task_offset);
4420
4421 if (task_offset == 0) {
4422 if (task_size >= se_mem->se_len) {
4423 sg_length = se_mem->se_len;
4424
4425 if (!(list_is_last(&se_mem->se_list,
4426 T_TASK(se_cmd)->t_mem_list)))
4427 se_mem = list_entry(se_mem->se_list.next,
4428 struct se_mem, se_list);
4429 } else {
4430 sg_length = task_size;
4431 task_size -= sg_length;
4432 goto next;
4433 }
4434
4435 DEBUG_SC("sg_length(%u) task_size(%u)\n",
4436 sg_length, task_size);
4437 } else {
4438 if ((se_mem->se_len - task_offset) > task_size) {
4439 sg_length = task_size;
4440 task_size -= sg_length;
4441 goto next;
4442 } else {
4443 sg_length = (se_mem->se_len - task_offset);
4444
4445 if (!(list_is_last(&se_mem->se_list,
4446 T_TASK(se_cmd)->t_mem_list)))
4447 se_mem = list_entry(se_mem->se_list.next,
4448 struct se_mem, se_list);
4449 }
4450
4451 DEBUG_SC("sg_length(%u) task_size(%u)\n",
4452 sg_length, task_size);
4453
4454 task_offset = 0;
4455 }
4456 task_size -= sg_length;
4457next:
4458 DEBUG_SC("task[%u] - Reducing task_size to(%u)\n",
4459 task->task_no, task_size);
4460
4461 task->task_sg_num++;
4462 }
4463 /*
4464 * Check if the fabric module driver is requesting that all
4465 * struct se_task->task_sg[] be chained together.. If so,
4466 * then allocate an extra padding SG entry for linking and
4467 * marking the end of the chained SGL.
4468 */
4469 if (tfo->task_sg_chaining) {
4470 task_sg_num_padded = (task->task_sg_num + 1);
4471 task->task_padded_sg = 1;
4472 } else
4473 task_sg_num_padded = task->task_sg_num;
4474
4475 task->task_sg = kzalloc(task_sg_num_padded *
4476 sizeof(struct scatterlist), GFP_KERNEL);
4477 if (!(task->task_sg)) {
4478 printk(KERN_ERR "Unable to allocate memory for"
4479 " task->task_sg\n");
4480 return 0;
4481 }
4482 sg_init_table(&task->task_sg[0], task_sg_num_padded);
4483 /*
4484 * Setup task->task_sg_bidi for SCSI READ payload for
4485 * TCM/pSCSI passthrough if present for BIDI-COMMAND
4486 */
4487 if ((T_TASK(se_cmd)->t_mem_bidi_list != NULL) &&
4488 (TRANSPORT(se_dev)->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)) {
4489 task->task_sg_bidi = kzalloc(task_sg_num_padded *
4490 sizeof(struct scatterlist), GFP_KERNEL);
4491 if (!(task->task_sg_bidi)) {
4492 printk(KERN_ERR "Unable to allocate memory for"
4493 " task->task_sg_bidi\n");
4494 return 0;
4495 }
4496 sg_init_table(&task->task_sg_bidi[0], task_sg_num_padded);
4497 }
4498 /*
4499 * For the chaining case, setup the proper end of SGL for the
4500 * initial submission struct task into struct se_subsystem_api.
4501 * This will be cleared later by transport_do_task_sg_chain()
4502 */
4503 if (task->task_padded_sg) {
4504 sg_mark_end(&task->task_sg[task->task_sg_num - 1]);
4505 /*
4506 * Added the 'if' check before marking end of bi-directional
4507 * scatterlist (which gets created only in case of request
4508 * (RD + WR).
4509 */
4510 if (task->task_sg_bidi)
4511 sg_mark_end(&task->task_sg_bidi[task->task_sg_num - 1]);
4512 }
4513
4514 DEBUG_SC("Successfully allocated task->task_sg_num(%u),"
4515 " task_sg_num_padded(%u)\n", task->task_sg_num,
4516 task_sg_num_padded);
4517
4518 return task->task_sg_num;
4519}
4520
4521static inline int transport_set_tasks_sectors_disk(
4522 struct se_task *task,
4523 struct se_device *dev,
4524 unsigned long long lba,
4525 u32 sectors,
4526 int *max_sectors_set)
4527{
4528 if ((lba + sectors) > transport_dev_end_lba(dev)) {
4529 task->task_sectors = ((transport_dev_end_lba(dev) - lba) + 1);
4530
4531 if (task->task_sectors > DEV_ATTRIB(dev)->max_sectors) {
4532 task->task_sectors = DEV_ATTRIB(dev)->max_sectors;
4533 *max_sectors_set = 1;
4534 }
4535 } else {
4536 if (sectors > DEV_ATTRIB(dev)->max_sectors) {
4537 task->task_sectors = DEV_ATTRIB(dev)->max_sectors;
4538 *max_sectors_set = 1;
4539 } else
4540 task->task_sectors = sectors;
4541 }
4542
4543 return 0;
4544}
4545
4546static inline int transport_set_tasks_sectors_non_disk(
4547 struct se_task *task,
4548 struct se_device *dev,
4549 unsigned long long lba,
4550 u32 sectors,
4551 int *max_sectors_set)
4552{
4553 if (sectors > DEV_ATTRIB(dev)->max_sectors) {
4554 task->task_sectors = DEV_ATTRIB(dev)->max_sectors;
4555 *max_sectors_set = 1;
4556 } else
4557 task->task_sectors = sectors;
4558
4559 return 0;
4560}
4561
4562static inline int transport_set_tasks_sectors(
4563 struct se_task *task,
4564 struct se_device *dev,
4565 unsigned long long lba,
4566 u32 sectors,
4567 int *max_sectors_set)
4568{
4569 return (TRANSPORT(dev)->get_device_type(dev) == TYPE_DISK) ?
4570 transport_set_tasks_sectors_disk(task, dev, lba, sectors,
4571 max_sectors_set) :
4572 transport_set_tasks_sectors_non_disk(task, dev, lba, sectors,
4573 max_sectors_set);
4574}
4575
4576static int transport_map_sg_to_mem(
4577 struct se_cmd *cmd,
4578 struct list_head *se_mem_list,
4579 void *in_mem,
4580 u32 *se_mem_cnt)
4581{
4582 struct se_mem *se_mem;
4583 struct scatterlist *sg;
4584 u32 sg_count = 1, cmd_size = cmd->data_length;
4585
4586 if (!in_mem) {
4587 printk(KERN_ERR "No source scatterlist\n");
4588 return -1;
4589 }
4590 sg = (struct scatterlist *)in_mem;
4591
4592 while (cmd_size) {
4593 se_mem = kmem_cache_zalloc(se_mem_cache, GFP_KERNEL);
4594 if (!(se_mem)) {
4595 printk(KERN_ERR "Unable to allocate struct se_mem\n");
4596 return -1;
4597 }
4598 INIT_LIST_HEAD(&se_mem->se_list);
4599 DEBUG_MEM("sg_to_mem: Starting loop with cmd_size: %u"
4600 " sg_page: %p offset: %d length: %d\n", cmd_size,
4601 sg_page(sg), sg->offset, sg->length);
4602
4603 se_mem->se_page = sg_page(sg);
4604 se_mem->se_off = sg->offset;
4605
4606 if (cmd_size > sg->length) {
4607 se_mem->se_len = sg->length;
4608 sg = sg_next(sg);
4609 sg_count++;
4610 } else
4611 se_mem->se_len = cmd_size;
4612
4613 cmd_size -= se_mem->se_len;
4614
4615 DEBUG_MEM("sg_to_mem: *se_mem_cnt: %u cmd_size: %u\n",
4616 *se_mem_cnt, cmd_size);
4617 DEBUG_MEM("sg_to_mem: Final se_page: %p se_off: %d se_len: %d\n",
4618 se_mem->se_page, se_mem->se_off, se_mem->se_len);
4619
4620 list_add_tail(&se_mem->se_list, se_mem_list);
4621 (*se_mem_cnt)++;
4622 }
4623
4624 DEBUG_MEM("task[0] - Mapped(%u) struct scatterlist segments to(%u)"
4625 " struct se_mem\n", sg_count, *se_mem_cnt);
4626
4627 if (sg_count != *se_mem_cnt)
4628 BUG();
4629
4630 return 0;
4631}
4632
4633/* transport_map_mem_to_sg():
4634 *
4635 *
4636 */
4637int transport_map_mem_to_sg(
4638 struct se_task *task,
4639 struct list_head *se_mem_list,
4640 void *in_mem,
4641 struct se_mem *in_se_mem,
4642 struct se_mem **out_se_mem,
4643 u32 *se_mem_cnt,
4644 u32 *task_offset)
4645{
4646 struct se_cmd *se_cmd = task->task_se_cmd;
4647 struct se_mem *se_mem = in_se_mem;
4648 struct scatterlist *sg = (struct scatterlist *)in_mem;
4649 u32 task_size = task->task_size, sg_no = 0;
4650
4651 if (!sg) {
4652 printk(KERN_ERR "Unable to locate valid struct"
4653 " scatterlist pointer\n");
4654 return -1;
4655 }
4656
4657 while (task_size != 0) {
4658 /*
4659 * Setup the contigious array of scatterlists for
4660 * this struct se_task.
4661 */
4662 sg_assign_page(sg, se_mem->se_page);
4663
4664 if (*task_offset == 0) {
4665 sg->offset = se_mem->se_off;
4666
4667 if (task_size >= se_mem->se_len) {
4668 sg->length = se_mem->se_len;
4669
4670 if (!(list_is_last(&se_mem->se_list,
4671 T_TASK(se_cmd)->t_mem_list))) {
4672 se_mem = list_entry(se_mem->se_list.next,
4673 struct se_mem, se_list);
4674 (*se_mem_cnt)++;
4675 }
4676 } else {
4677 sg->length = task_size;
4678 /*
4679 * Determine if we need to calculate an offset
4680 * into the struct se_mem on the next go around..
4681 */
4682 task_size -= sg->length;
4683 if (!(task_size))
4684 *task_offset = sg->length;
4685
4686 goto next;
4687 }
4688
4689 } else {
4690 sg->offset = (*task_offset + se_mem->se_off);
4691
4692 if ((se_mem->se_len - *task_offset) > task_size) {
4693 sg->length = task_size;
4694 /*
4695 * Determine if we need to calculate an offset
4696 * into the struct se_mem on the next go around..
4697 */
4698 task_size -= sg->length;
4699 if (!(task_size))
4700 *task_offset += sg->length;
4701
4702 goto next;
4703 } else {
4704 sg->length = (se_mem->se_len - *task_offset);
4705
4706 if (!(list_is_last(&se_mem->se_list,
4707 T_TASK(se_cmd)->t_mem_list))) {
4708 se_mem = list_entry(se_mem->se_list.next,
4709 struct se_mem, se_list);
4710 (*se_mem_cnt)++;
4711 }
4712 }
4713
4714 *task_offset = 0;
4715 }
4716 task_size -= sg->length;
4717next:
4718 DEBUG_MEM("task[%u] mem_to_sg - sg[%u](%p)(%u)(%u) - Reducing"
4719 " task_size to(%u), task_offset: %u\n", task->task_no, sg_no,
4720 sg_page(sg), sg->length, sg->offset, task_size, *task_offset);
4721
4722 sg_no++;
4723 if (!(task_size))
4724 break;
4725
4726 sg = sg_next(sg);
4727
4728 if (task_size > se_cmd->data_length)
4729 BUG();
4730 }
4731 *out_se_mem = se_mem;
4732
4733 DEBUG_MEM("task[%u] - Mapped(%u) struct se_mem segments to total(%u)"
4734 " SGs\n", task->task_no, *se_mem_cnt, sg_no);
4735
4736 return 0;
4737}
4738
4739/*
4740 * This function can be used by HW target mode drivers to create a linked
4741 * scatterlist from all contiguously allocated struct se_task->task_sg[].
4742 * This is intended to be called during the completion path by TCM Core
4743 * when struct target_core_fabric_ops->check_task_sg_chaining is enabled.
4744 */
4745void transport_do_task_sg_chain(struct se_cmd *cmd)
4746{
4747 struct scatterlist *sg_head = NULL, *sg_link = NULL, *sg_first = NULL;
4748 struct scatterlist *sg_head_cur = NULL, *sg_link_cur = NULL;
4749 struct scatterlist *sg, *sg_end = NULL, *sg_end_cur = NULL;
4750 struct se_task *task;
4751 struct target_core_fabric_ops *tfo = CMD_TFO(cmd);
4752 u32 task_sg_num = 0, sg_count = 0;
4753 int i;
4754
4755 if (tfo->task_sg_chaining == 0) {
4756 printk(KERN_ERR "task_sg_chaining is diabled for fabric module:"
4757 " %s\n", tfo->get_fabric_name());
4758 dump_stack();
4759 return;
4760 }
4761 /*
4762 * Walk the struct se_task list and setup scatterlist chains
4763 * for each contiguosly allocated struct se_task->task_sg[].
4764 */
4765 list_for_each_entry(task, &T_TASK(cmd)->t_task_list, t_list) {
4766 if (!(task->task_sg) || !(task->task_padded_sg))
4767 continue;
4768
4769 if (sg_head && sg_link) {
4770 sg_head_cur = &task->task_sg[0];
4771 sg_link_cur = &task->task_sg[task->task_sg_num];
4772 /*
4773 * Either add chain or mark end of scatterlist
4774 */
4775 if (!(list_is_last(&task->t_list,
4776 &T_TASK(cmd)->t_task_list))) {
4777 /*
4778 * Clear existing SGL termination bit set in
4779 * transport_calc_sg_num(), see sg_mark_end()
4780 */
4781 sg_end_cur = &task->task_sg[task->task_sg_num - 1];
4782 sg_end_cur->page_link &= ~0x02;
4783
4784 sg_chain(sg_head, task_sg_num, sg_head_cur);
c66ac9db 4785 sg_count += task->task_sg_num;
97868c89
NB		 4786 task_sg_num = (task->task_sg_num + 1);
4787 } else {
4788 sg_chain(sg_head, task_sg_num, sg_head_cur);
4789 sg_count += task->task_sg_num;
4790 task_sg_num = task->task_sg_num;
4791 }
c66ac9db
NB		 4792
4793 sg_head = sg_head_cur;
4794 sg_link = sg_link_cur;
c66ac9db
NB		 4795 continue;
4796 }
4797 sg_head = sg_first = &task->task_sg[0];
4798 sg_link = &task->task_sg[task->task_sg_num];
c66ac9db
NB		 4799 /*
4800 * Check for single task..
4801 */
4802 if (!(list_is_last(&task->t_list, &T_TASK(cmd)->t_task_list))) {
4803 /*
4804 * Clear existing SGL termination bit set in
4805 * transport_calc_sg_num(), see sg_mark_end()
4806 */
4807 sg_end = &task->task_sg[task->task_sg_num - 1];
4808 sg_end->page_link &= ~0x02;
c66ac9db 4809 sg_count += task->task_sg_num;
97868c89
NB		 4810 task_sg_num = (task->task_sg_num + 1);
4811 } else {
4812 sg_count += task->task_sg_num;
4813 task_sg_num = task->task_sg_num;
4814 }
c66ac9db
NB		 4815 }
4816 /*
4817 * Setup the starting pointer and total t_tasks_sg_linked_no including
4818 * padding SGs for linking and to mark the end.
4819 */
4820 T_TASK(cmd)->t_tasks_sg_chained = sg_first;
4821 T_TASK(cmd)->t_tasks_sg_chained_no = sg_count;
4822
97868c89
NB		 4823 DEBUG_CMD_M("Setup cmd: %p T_TASK(cmd)->t_tasks_sg_chained: %p and"
4824 " t_tasks_sg_chained_no: %u\n", cmd, T_TASK(cmd)->t_tasks_sg_chained,
c66ac9db
NB		 4825 T_TASK(cmd)->t_tasks_sg_chained_no);
4826
4827 for_each_sg(T_TASK(cmd)->t_tasks_sg_chained, sg,
4828 T_TASK(cmd)->t_tasks_sg_chained_no, i) {
4829
97868c89
NB		 4830 DEBUG_CMD_M("SG[%d]: %p page: %p length: %d offset: %d, magic: 0x%08x\n",
4831 i, sg, sg_page(sg), sg->length, sg->offset, sg->sg_magic);
c66ac9db
NB		 4832 if (sg_is_chain(sg))
4833 DEBUG_CMD_M("SG: %p sg_is_chain=1\n", sg);
4834 if (sg_is_last(sg))
4835 DEBUG_CMD_M("SG: %p sg_is_last=1\n", sg);
4836 }
c66ac9db
NB		 4837}
4838EXPORT_SYMBOL(transport_do_task_sg_chain);
4839
4840static int transport_do_se_mem_map(
4841 struct se_device *dev,
4842 struct se_task *task,
4843 struct list_head *se_mem_list,
4844 void *in_mem,
4845 struct se_mem *in_se_mem,
4846 struct se_mem **out_se_mem,
4847 u32 *se_mem_cnt,
4848 u32 *task_offset_in)
4849{
4850 u32 task_offset = *task_offset_in;
4851 int ret = 0;
4852 /*
4853 * se_subsystem_api_t->do_se_mem_map is used when internal allocation
4854 * has been done by the transport plugin.
4855 */
4856 if (TRANSPORT(dev)->do_se_mem_map) {
4857 ret = TRANSPORT(dev)->do_se_mem_map(task, se_mem_list,
4858 in_mem, in_se_mem, out_se_mem, se_mem_cnt,
4859 task_offset_in);
4860 if (ret == 0)
4861 T_TASK(task->task_se_cmd)->t_tasks_se_num += *se_mem_cnt;
4862
4863 return ret;
4864 }
e63af958
NB		 4865
4866 BUG_ON(list_empty(se_mem_list));
c66ac9db
NB		 4867 /*
4868 * This is the normal path for all normal non BIDI and BIDI-COMMAND
4869 * WRITE payloads.. If we need to do BIDI READ passthrough for
4870 * TCM/pSCSI the first call to transport_do_se_mem_map ->
4871 * transport_calc_sg_num() -> transport_map_mem_to_sg() will do the
4872 * allocation for task->task_sg_bidi, and the subsequent call to
4873 * transport_do_se_mem_map() from transport_generic_get_cdb_count()
4874 */
4875 if (!(task->task_sg_bidi)) {
4876 /*
4877 * Assume default that transport plugin speaks preallocated
4878 * scatterlists.
4879 */
4880 if (!(transport_calc_sg_num(task, in_se_mem, task_offset)))
4881 return -1;
4882 /*
4883 * struct se_task->task_sg now contains the struct scatterlist array.
4884 */
4885 return transport_map_mem_to_sg(task, se_mem_list, task->task_sg,
4886 in_se_mem, out_se_mem, se_mem_cnt,
4887 task_offset_in);
4888 }
4889 /*
4890 * Handle the se_mem_list -> struct task->task_sg_bidi
4891 * memory map for the extra BIDI READ payload
4892 */
4893 return transport_map_mem_to_sg(task, se_mem_list, task->task_sg_bidi,
4894 in_se_mem, out_se_mem, se_mem_cnt,
4895 task_offset_in);
4896}
4897
4898static u32 transport_generic_get_cdb_count(
4899 struct se_cmd *cmd,
4900 unsigned long long lba,
4901 u32 sectors,
4902 enum dma_data_direction data_direction,
4903 struct list_head *mem_list,
4904 int set_counts)
4905{
4906 unsigned char *cdb = NULL;
4907 struct se_task *task;
4908 struct se_mem *se_mem = NULL, *se_mem_lout = NULL;
4909 struct se_mem *se_mem_bidi = NULL, *se_mem_bidi_lout = NULL;
4910 struct se_device *dev = SE_DEV(cmd);
4911 int max_sectors_set = 0, ret;
4912 u32 task_offset_in = 0, se_mem_cnt = 0, se_mem_bidi_cnt = 0, task_cdbs = 0;
4913
4914 if (!mem_list) {
4915 printk(KERN_ERR "mem_list is NULL in transport_generic_get"
4916 "_cdb_count()\n");
4917 return 0;
4918 }
4919 /*
4920 * While using RAMDISK_DR backstores is the only case where
4921 * mem_list will ever be empty at this point.
4922 */
4923 if (!(list_empty(mem_list)))
4924 se_mem = list_entry(mem_list->next, struct se_mem, se_list);
4925 /*
4926 * Check for extra se_mem_bidi mapping for BIDI-COMMANDs to
4927 * struct se_task->task_sg_bidi for TCM/pSCSI passthrough operation
4928 */
4929 if ((T_TASK(cmd)->t_mem_bidi_list != NULL) &&
4930 !(list_empty(T_TASK(cmd)->t_mem_bidi_list)) &&
4931 (TRANSPORT(dev)->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV))
4932 se_mem_bidi = list_entry(T_TASK(cmd)->t_mem_bidi_list->next,
4933 struct se_mem, se_list);
4934
4935 while (sectors) {
4936 DEBUG_VOL("ITT[0x%08x] LBA(%llu) SectorsLeft(%u) EOBJ(%llu)\n",
4937 CMD_TFO(cmd)->get_task_tag(cmd), lba, sectors,
4938 transport_dev_end_lba(dev));
4939
4940 task = transport_generic_get_task(cmd, data_direction);
4941 if (!(task))
4942 goto out;
4943
4944 transport_set_tasks_sectors(task, dev, lba, sectors,
4945 &max_sectors_set);
4946
4947 task->task_lba = lba;
4948 lba += task->task_sectors;
4949 sectors -= task->task_sectors;
4950 task->task_size = (task->task_sectors *
4951 DEV_ATTRIB(dev)->block_size);
4952
4953 cdb = TRANSPORT(dev)->get_cdb(task);
4954 if ((cdb)) {
4955 memcpy(cdb, T_TASK(cmd)->t_task_cdb,
4956 scsi_command_size(T_TASK(cmd)->t_task_cdb));
4957 cmd->transport_split_cdb(task->task_lba,
4958 &task->task_sectors, cdb);
4959 }
4960
4961 /*
4962 * Perform the SE OBJ plugin and/or Transport plugin specific
4963 * mapping for T_TASK(cmd)->t_mem_list. And setup the
4964 * task->task_sg and if necessary task->task_sg_bidi
4965 */
4966 ret = transport_do_se_mem_map(dev, task, mem_list,
4967 NULL, se_mem, &se_mem_lout, &se_mem_cnt,
4968 &task_offset_in);
4969 if (ret < 0)
4970 goto out;
4971
4972 se_mem = se_mem_lout;
4973 /*
4974 * Setup the T_TASK(cmd)->t_mem_bidi_list -> task->task_sg_bidi
4975 * mapping for SCSI READ for BIDI-COMMAND passthrough with TCM/pSCSI
4976 *
4977 * Note that the first call to transport_do_se_mem_map() above will
4978 * allocate struct se_task->task_sg_bidi in transport_do_se_mem_map()
4979 * -> transport_calc_sg_num(), and the second here will do the
4980 * mapping for SCSI READ for BIDI-COMMAND passthrough with TCM/pSCSI.
4981 */
4982 if (task->task_sg_bidi != NULL) {
4983 ret = transport_do_se_mem_map(dev, task,
4984 T_TASK(cmd)->t_mem_bidi_list, NULL,
4985 se_mem_bidi, &se_mem_bidi_lout, &se_mem_bidi_cnt,
4986 &task_offset_in);
4987 if (ret < 0)
4988 goto out;
4989
4990 se_mem_bidi = se_mem_bidi_lout;
4991 }
4992 task_cdbs++;
4993
4994 DEBUG_VOL("Incremented task_cdbs(%u) task->task_sg_num(%u)\n",
4995 task_cdbs, task->task_sg_num);
4996
4997 if (max_sectors_set) {
4998 max_sectors_set = 0;
4999 continue;
5000 }
5001
5002 if (!sectors)
5003 break;
5004 }
5005
5006 if (set_counts) {
5007 atomic_inc(&T_TASK(cmd)->t_fe_count);
5008 atomic_inc(&T_TASK(cmd)->t_se_count);
5009 }
5010
5011 DEBUG_VOL("ITT[0x%08x] total %s cdbs(%u)\n",
5012 CMD_TFO(cmd)->get_task_tag(cmd), (data_direction == DMA_TO_DEVICE)
5013 ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE", task_cdbs);
5014
5015 return task_cdbs;
5016out:
5017 return 0;
5018}
5019
5020static int
5021transport_map_control_cmd_to_task(struct se_cmd *cmd)
5022{
5023 struct se_device *dev = SE_DEV(cmd);
5024 unsigned char *cdb;
5025 struct se_task *task;
5026 int ret;
5027
5028 task = transport_generic_get_task(cmd, cmd->data_direction);
5029 if (!task)
5030 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
5031
5032 cdb = TRANSPORT(dev)->get_cdb(task);
5033 if (cdb)
5034 memcpy(cdb, cmd->t_task->t_task_cdb,
5035 scsi_command_size(cmd->t_task->t_task_cdb));
5036
5037 task->task_size = cmd->data_length;
5038 task->task_sg_num =
5039 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) ? 1 : 0;
5040
5041 atomic_inc(&cmd->t_task->t_fe_count);
5042 atomic_inc(&cmd->t_task->t_se_count);
5043
5044 if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) {
5045 struct se_mem *se_mem = NULL, *se_mem_lout = NULL;
5046 u32 se_mem_cnt = 0, task_offset = 0;
5047
e63af958
NB		 5048 if (!list_empty(T_TASK(cmd)->t_mem_list))
5049 se_mem = list_entry(T_TASK(cmd)->t_mem_list->next,
5050 struct se_mem, se_list);
c66ac9db
NB		 5051
5052 ret = transport_do_se_mem_map(dev, task,
5053 cmd->t_task->t_mem_list, NULL, se_mem,
5054 &se_mem_lout, &se_mem_cnt, &task_offset);
5055 if (ret < 0)
5056 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
5057
5058 if (dev->transport->map_task_SG)
5059 return dev->transport->map_task_SG(task);
5060 return 0;
5061 } else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB) {
5062 if (dev->transport->map_task_non_SG)
5063 return dev->transport->map_task_non_SG(task);
5064 return 0;
5065 } else if (cmd->se_cmd_flags & SCF_SCSI_NON_DATA_CDB) {
5066 if (dev->transport->cdb_none)
5067 return dev->transport->cdb_none(task);
5068 return 0;
5069 } else {
5070 BUG();
5071 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
5072 }
5073}
5074
5075/* transport_generic_new_cmd(): Called from transport_processing_thread()
5076 *
5077 * Allocate storage transport resources from a set of values predefined
5078 * by transport_generic_cmd_sequencer() from the iSCSI Target RX process.
5079 * Any non zero return here is treated as an "out of resource' op here.
5080 */
5081 /*
5082 * Generate struct se_task(s) and/or their payloads for this CDB.
5083 */
5084static int transport_generic_new_cmd(struct se_cmd *cmd)
5085{
5086 struct se_portal_group *se_tpg;
5087 struct se_task *task;
5088 struct se_device *dev = SE_DEV(cmd);
5089 int ret = 0;
5090
5091 /*
5092 * Determine is the TCM fabric module has already allocated physical
5093 * memory, and is directly calling transport_generic_map_mem_to_cmd()
5094 * to setup beforehand the linked list of physical memory at
5095 * T_TASK(cmd)->t_mem_list of struct se_mem->se_page
5096 */
5097 if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)) {
5098 ret = transport_allocate_resources(cmd);
5099 if (ret < 0)
5100 return ret;
5101 }
5102
5103 ret = transport_get_sectors(cmd);
5104 if (ret < 0)
5105 return ret;
5106
5107 ret = transport_new_cmd_obj(cmd);
5108 if (ret < 0)
5109 return ret;
5110
5111 /*
5112 * Determine if the calling TCM fabric module is talking to
5113 * Linux/NET via kernel sockets and needs to allocate a
5114 * struct iovec array to complete the struct se_cmd
5115 */
5116 se_tpg = SE_LUN(cmd)->lun_sep->sep_tpg;
5117 if (TPG_TFO(se_tpg)->alloc_cmd_iovecs != NULL) {
5118 ret = TPG_TFO(se_tpg)->alloc_cmd_iovecs(cmd);
5119 if (ret < 0)
5120 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
5121 }
5122
5123 if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) {
5124 list_for_each_entry(task, &T_TASK(cmd)->t_task_list, t_list) {
5125 if (atomic_read(&task->task_sent))
5126 continue;
5127 if (!dev->transport->map_task_SG)
5128 continue;
5129
5130 ret = dev->transport->map_task_SG(task);
5131 if (ret < 0)
5132 return ret;
5133 }
5134 } else {
5135 ret = transport_map_control_cmd_to_task(cmd);
5136 if (ret < 0)
5137 return ret;
5138 }
5139
5140 /*
5141 * For WRITEs, let the iSCSI Target RX Thread know its buffer is ready..
5142 * This WRITE struct se_cmd (and all of its associated struct se_task's)
5143 * will be added to the struct se_device execution queue after its WRITE
5144 * data has arrived. (ie: It gets handled by the transport processing
5145 * thread a second time)
5146 */
5147 if (cmd->data_direction == DMA_TO_DEVICE) {
5148 transport_add_tasks_to_state_queue(cmd);
5149 return transport_generic_write_pending(cmd);
5150 }
5151 /*
5152 * Everything else but a WRITE, add the struct se_cmd's struct se_task's
5153 * to the execution queue.
5154 */
5155 transport_execute_tasks(cmd);
5156 return 0;
5157}
5158
5159/* transport_generic_process_write():
5160 *
5161 *
5162 */
5163void transport_generic_process_write(struct se_cmd *cmd)
5164{
5165#if 0
5166 /*
5167 * Copy SCSI Presented DTL sector(s) from received buffers allocated to
5168 * original EDTL
5169 */
5170 if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
5171 if (!T_TASK(cmd)->t_tasks_se_num) {
5172 unsigned char *dst, *buf =
5173 (unsigned char *)T_TASK(cmd)->t_task_buf;
5174
5175 dst = kzalloc(cmd->cmd_spdtl), GFP_KERNEL);
5176 if (!(dst)) {
5177 printk(KERN_ERR "Unable to allocate memory for"
5178 " WRITE underflow\n");
5179 transport_generic_request_failure(cmd, NULL,
5180 PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
5181 return;
5182 }
5183 memcpy(dst, buf, cmd->cmd_spdtl);
5184
5185 kfree(T_TASK(cmd)->t_task_buf);
5186 T_TASK(cmd)->t_task_buf = dst;
5187 } else {
5188 struct scatterlist *sg =
5189 (struct scatterlist *sg)T_TASK(cmd)->t_task_buf;
5190 struct scatterlist *orig_sg;
5191
5192 orig_sg = kzalloc(sizeof(struct scatterlist) *
5193 T_TASK(cmd)->t_tasks_se_num,
5194 GFP_KERNEL))) {
5195 if (!(orig_sg)) {
5196 printk(KERN_ERR "Unable to allocate memory"
5197 " for WRITE underflow\n");
5198 transport_generic_request_failure(cmd, NULL,
5199 PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
5200 return;
5201 }
5202
5203 memcpy(orig_sg, T_TASK(cmd)->t_task_buf,
5204 sizeof(struct scatterlist) *
5205 T_TASK(cmd)->t_tasks_se_num);
5206
5207 cmd->data_length = cmd->cmd_spdtl;
5208 /*
5209 * FIXME, clear out original struct se_task and state
5210 * information.
5211 */
5212 if (transport_generic_new_cmd(cmd) < 0) {
5213 transport_generic_request_failure(cmd, NULL,
5214 PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
5215 kfree(orig_sg);
5216 return;
5217 }
5218
5219 transport_memcpy_write_sg(cmd, orig_sg);
5220 }
5221 }
5222#endif
5223 transport_execute_tasks(cmd);
5224}
5225EXPORT_SYMBOL(transport_generic_process_write);
5226
5227/* transport_generic_write_pending():
5228 *
5229 *
5230 */
5231static int transport_generic_write_pending(struct se_cmd *cmd)
5232{
5233 unsigned long flags;
5234 int ret;
5235
5236 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5237 cmd->t_state = TRANSPORT_WRITE_PENDING;
5238 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5239 /*
5240 * For the TCM control CDBs using a contiguous buffer, do the memcpy
5241 * from the passed Linux/SCSI struct scatterlist located at
5242 * T_TASK(se_cmd)->t_task_pt_buf to the contiguous buffer at
5243 * T_TASK(se_cmd)->t_task_buf.
5244 */
5245 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_CONTIG_TO_SG)
5246 transport_memcpy_read_contig(cmd,
5247 T_TASK(cmd)->t_task_buf,
5248 T_TASK(cmd)->t_task_pt_sgl);
5249 /*
5250 * Clear the se_cmd for WRITE_PENDING status in order to set
5251 * T_TASK(cmd)->t_transport_active=0 so that transport_generic_handle_data
5252 * can be called from HW target mode interrupt code. This is safe
5253 * to be called with transport_off=1 before the CMD_TFO(cmd)->write_pending
5254 * because the se_cmd->se_lun pointer is not being cleared.
5255 */
5256 transport_cmd_check_stop(cmd, 1, 0);
5257
5258 /*
5259 * Call the fabric write_pending function here to let the
5260 * frontend know that WRITE buffers are ready.
5261 */
5262 ret = CMD_TFO(cmd)->write_pending(cmd);
5263 if (ret < 0)
5264 return ret;
5265
5266 return PYX_TRANSPORT_WRITE_PENDING;
5267}
5268
5269/* transport_release_cmd_to_pool():
5270 *
5271 *
5272 */
5273void transport_release_cmd_to_pool(struct se_cmd *cmd)
5274{
5275 BUG_ON(!T_TASK(cmd));
5276 BUG_ON(!CMD_TFO(cmd));
5277
5278 transport_free_se_cmd(cmd);
5279 CMD_TFO(cmd)->release_cmd_to_pool(cmd);
5280}
5281EXPORT_SYMBOL(transport_release_cmd_to_pool);
5282
5283/* transport_generic_free_cmd():
5284 *
5285 * Called from processing frontend to release storage engine resources
5286 */
5287void transport_generic_free_cmd(
5288 struct se_cmd *cmd,
5289 int wait_for_tasks,
5290 int release_to_pool,
5291 int session_reinstatement)
5292{
5293 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) || !T_TASK(cmd))
5294 transport_release_cmd_to_pool(cmd);
5295 else {
5296 core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);
5297
5298 if (SE_LUN(cmd)) {
5299#if 0
5300 printk(KERN_INFO "cmd: %p ITT: 0x%08x contains"
5301 " SE_LUN(cmd)\n", cmd,
5302 CMD_TFO(cmd)->get_task_tag(cmd));
5303#endif
5304 transport_lun_remove_cmd(cmd);
5305 }
5306
5307 if (wait_for_tasks && cmd->transport_wait_for_tasks)
5308 cmd->transport_wait_for_tasks(cmd, 0, 0);
5309
f4366772
NB		 5310 transport_free_dev_tasks(cmd);
5311
c66ac9db
NB		 5312 transport_generic_remove(cmd, release_to_pool,
5313 session_reinstatement);
5314 }
5315}
5316EXPORT_SYMBOL(transport_generic_free_cmd);
5317
5318static void transport_nop_wait_for_tasks(
5319 struct se_cmd *cmd,
5320 int remove_cmd,
5321 int session_reinstatement)
5322{
5323 return;
5324}
5325
5326/* transport_lun_wait_for_tasks():
5327 *
5328 * Called from ConfigFS context to stop the passed struct se_cmd to allow
5329 * an struct se_lun to be successfully shutdown.
5330 */
5331static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
5332{
5333 unsigned long flags;
5334 int ret;
5335 /*
5336 * If the frontend has already requested this struct se_cmd to
5337 * be stopped, we can safely ignore this struct se_cmd.
5338 */
5339 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5340 if (atomic_read(&T_TASK(cmd)->t_transport_stop)) {
5341 atomic_set(&T_TASK(cmd)->transport_lun_stop, 0);
5342 DEBUG_TRANSPORT_S("ConfigFS ITT[0x%08x] - t_transport_stop =="
5343 " TRUE, skipping\n", CMD_TFO(cmd)->get_task_tag(cmd));
5344 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5345 transport_cmd_check_stop(cmd, 1, 0);
5346 return -1;
5347 }
5348 atomic_set(&T_TASK(cmd)->transport_lun_fe_stop, 1);
5349 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5350
5351 wake_up_interruptible(&SE_DEV(cmd)->dev_queue_obj->thread_wq);
5352
5353 ret = transport_stop_tasks_for_cmd(cmd);
5354
5355 DEBUG_TRANSPORT_S("ConfigFS: cmd: %p t_task_cdbs: %d stop tasks ret:"
5356 " %d\n", cmd, T_TASK(cmd)->t_task_cdbs, ret);
5357 if (!ret) {
5358 DEBUG_TRANSPORT_S("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
5359 CMD_TFO(cmd)->get_task_tag(cmd));
5360 wait_for_completion(&T_TASK(cmd)->transport_lun_stop_comp);
5361 DEBUG_TRANSPORT_S("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
5362 CMD_TFO(cmd)->get_task_tag(cmd));
5363 }
5364 transport_remove_cmd_from_queue(cmd, SE_DEV(cmd)->dev_queue_obj);
5365
5366 return 0;
5367}
5368
5369/* #define DEBUG_CLEAR_LUN */
5370#ifdef DEBUG_CLEAR_LUN
5371#define DEBUG_CLEAR_L(x...) printk(KERN_INFO x)
5372#else
5373#define DEBUG_CLEAR_L(x...)
5374#endif
5375
5376static void __transport_clear_lun_from_sessions(struct se_lun *lun)
5377{
5378 struct se_cmd *cmd = NULL;
5379 unsigned long lun_flags, cmd_flags;
5380 /*
5381 * Do exception processing and return CHECK_CONDITION status to the
5382 * Initiator Port.
5383 */
5384 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5385 while (!list_empty_careful(&lun->lun_cmd_list)) {
5386 cmd = list_entry(lun->lun_cmd_list.next,
5387 struct se_cmd, se_lun_list);
5388 list_del(&cmd->se_lun_list);
5389
5390 if (!(T_TASK(cmd))) {
5391 printk(KERN_ERR "ITT: 0x%08x, T_TASK(cmd) = NULL"
5392 "[i,t]_state: %u/%u\n",
5393 CMD_TFO(cmd)->get_task_tag(cmd),
5394 CMD_TFO(cmd)->get_cmd_state(cmd), cmd->t_state);
5395 BUG();
5396 }
5397 atomic_set(&T_TASK(cmd)->transport_lun_active, 0);
5398 /*
5399 * This will notify iscsi_target_transport.c:
5400 * transport_cmd_check_stop() that a LUN shutdown is in
5401 * progress for the iscsi_cmd_t.
5402 */
5403 spin_lock(&T_TASK(cmd)->t_state_lock);
5404 DEBUG_CLEAR_L("SE_LUN[%d] - Setting T_TASK(cmd)->transport"
5405 "_lun_stop for ITT: 0x%08x\n",
5406 SE_LUN(cmd)->unpacked_lun,
5407 CMD_TFO(cmd)->get_task_tag(cmd));
5408 atomic_set(&T_TASK(cmd)->transport_lun_stop, 1);
5409 spin_unlock(&T_TASK(cmd)->t_state_lock);
5410
5411 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
5412
5413 if (!(SE_LUN(cmd))) {
5414 printk(KERN_ERR "ITT: 0x%08x, [i,t]_state: %u/%u\n",
5415 CMD_TFO(cmd)->get_task_tag(cmd),
5416 CMD_TFO(cmd)->get_cmd_state(cmd), cmd->t_state);
5417 BUG();
5418 }
5419 /*
5420 * If the Storage engine still owns the iscsi_cmd_t, determine
5421 * and/or stop its context.
5422 */
5423 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x before transport"
5424 "_lun_wait_for_tasks()\n", SE_LUN(cmd)->unpacked_lun,
5425 CMD_TFO(cmd)->get_task_tag(cmd));
5426
5427 if (transport_lun_wait_for_tasks(cmd, SE_LUN(cmd)) < 0) {
5428 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5429 continue;
5430 }
5431
5432 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
5433 "_wait_for_tasks(): SUCCESS\n",
5434 SE_LUN(cmd)->unpacked_lun,
5435 CMD_TFO(cmd)->get_task_tag(cmd));
5436
5437 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, cmd_flags);
5438 if (!(atomic_read(&T_TASK(cmd)->transport_dev_active))) {
5439 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, cmd_flags);
5440 goto check_cond;
5441 }
5442 atomic_set(&T_TASK(cmd)->transport_dev_active, 0);
5443 transport_all_task_dev_remove_state(cmd);
5444 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, cmd_flags);
5445
5446 transport_free_dev_tasks(cmd);
5447 /*
5448 * The Storage engine stopped this struct se_cmd before it was
5449 * send to the fabric frontend for delivery back to the
5450 * Initiator Node. Return this SCSI CDB back with an
5451 * CHECK_CONDITION status.
5452 */
5453check_cond:
5454 transport_send_check_condition_and_sense(cmd,
5455 TCM_NON_EXISTENT_LUN, 0);
5456 /*
5457 * If the fabric frontend is waiting for this iscsi_cmd_t to
5458 * be released, notify the waiting thread now that LU has
5459 * finished accessing it.
5460 */
5461 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, cmd_flags);
5462 if (atomic_read(&T_TASK(cmd)->transport_lun_fe_stop)) {
5463 DEBUG_CLEAR_L("SE_LUN[%d] - Detected FE stop for"
5464 " struct se_cmd: %p ITT: 0x%08x\n",
5465 lun->unpacked_lun,
5466 cmd, CMD_TFO(cmd)->get_task_tag(cmd));
5467
5468 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock,
5469 cmd_flags);
5470 transport_cmd_check_stop(cmd, 1, 0);
5471 complete(&T_TASK(cmd)->transport_lun_fe_stop_comp);
5472 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5473 continue;
5474 }
5475 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
5476 lun->unpacked_lun, CMD_TFO(cmd)->get_task_tag(cmd));
5477
5478 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, cmd_flags);
5479 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5480 }
5481 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
5482}
5483
5484static int transport_clear_lun_thread(void *p)
5485{
5486 struct se_lun *lun = (struct se_lun *)p;
5487
5488 __transport_clear_lun_from_sessions(lun);
5489 complete(&lun->lun_shutdown_comp);
5490
5491 return 0;
5492}
5493
5494int transport_clear_lun_from_sessions(struct se_lun *lun)
5495{
5496 struct task_struct *kt;
5497
5498 kt = kthread_run(transport_clear_lun_thread, (void *)lun,
5499 "tcm_cl_%u", lun->unpacked_lun);
5500 if (IS_ERR(kt)) {
5501 printk(KERN_ERR "Unable to start clear_lun thread\n");
5502 return -1;
5503 }
5504 wait_for_completion(&lun->lun_shutdown_comp);
5505
5506 return 0;
5507}
5508
5509/* transport_generic_wait_for_tasks():
5510 *
5511 * Called from frontend or passthrough context to wait for storage engine
5512 * to pause and/or release frontend generated struct se_cmd.
5513 */
5514static void transport_generic_wait_for_tasks(
5515 struct se_cmd *cmd,
5516 int remove_cmd,
5517 int session_reinstatement)
5518{
5519 unsigned long flags;
5520
5521 if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && !(cmd->se_tmr_req))
5522 return;
5523
5524 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5525 /*
5526 * If we are already stopped due to an external event (ie: LUN shutdown)
5527 * sleep until the connection can have the passed struct se_cmd back.
5528 * The T_TASK(cmd)->transport_lun_stopped_sem will be upped by
5529 * transport_clear_lun_from_sessions() once the ConfigFS context caller
5530 * has completed its operation on the struct se_cmd.
5531 */
5532 if (atomic_read(&T_TASK(cmd)->transport_lun_stop)) {
5533
5534 DEBUG_TRANSPORT_S("wait_for_tasks: Stopping"
5535 " wait_for_completion(&T_TASK(cmd)transport_lun_fe"
5536 "_stop_comp); for ITT: 0x%08x\n",
5537 CMD_TFO(cmd)->get_task_tag(cmd));
5538 /*
5539 * There is a special case for WRITES where a FE exception +
5540 * LUN shutdown means ConfigFS context is still sleeping on
5541 * transport_lun_stop_comp in transport_lun_wait_for_tasks().
5542 * We go ahead and up transport_lun_stop_comp just to be sure
5543 * here.
5544 */
5545 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5546 complete(&T_TASK(cmd)->transport_lun_stop_comp);
5547 wait_for_completion(&T_TASK(cmd)->transport_lun_fe_stop_comp);
5548 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5549
5550 transport_all_task_dev_remove_state(cmd);
5551 /*
5552 * At this point, the frontend who was the originator of this
5553 * struct se_cmd, now owns the structure and can be released through
5554 * normal means below.
5555 */
5556 DEBUG_TRANSPORT_S("wait_for_tasks: Stopped"
5557 " wait_for_completion(&T_TASK(cmd)transport_lun_fe_"
5558 "stop_comp); for ITT: 0x%08x\n",
5559 CMD_TFO(cmd)->get_task_tag(cmd));
5560
5561 atomic_set(&T_TASK(cmd)->transport_lun_stop, 0);
5562 }
52208ae3
NB		 5563 if (!atomic_read(&T_TASK(cmd)->t_transport_active) ||
5564 atomic_read(&T_TASK(cmd)->t_transport_aborted))
c66ac9db
NB		 5565 goto remove;
5566
5567 atomic_set(&T_TASK(cmd)->t_transport_stop, 1);
5568
5569 DEBUG_TRANSPORT_S("wait_for_tasks: Stopping %p ITT: 0x%08x"
5570 " i_state: %d, t_state/def_t_state: %d/%d, t_transport_stop"
5571 " = TRUE\n", cmd, CMD_TFO(cmd)->get_task_tag(cmd),
5572 CMD_TFO(cmd)->get_cmd_state(cmd), cmd->t_state,
5573 cmd->deferred_t_state);
5574
5575 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5576
5577 wake_up_interruptible(&SE_DEV(cmd)->dev_queue_obj->thread_wq);
5578
5579 wait_for_completion(&T_TASK(cmd)->t_transport_stop_comp);
5580
5581 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5582 atomic_set(&T_TASK(cmd)->t_transport_active, 0);
5583 atomic_set(&T_TASK(cmd)->t_transport_stop, 0);
5584
5585 DEBUG_TRANSPORT_S("wait_for_tasks: Stopped wait_for_compltion("
5586 "&T_TASK(cmd)->t_transport_stop_comp) for ITT: 0x%08x\n",
5587 CMD_TFO(cmd)->get_task_tag(cmd));
5588remove:
5589 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5590 if (!remove_cmd)
5591 return;
5592
5593 transport_generic_free_cmd(cmd, 0, 0, session_reinstatement);
5594}
5595
5596static int transport_get_sense_codes(
5597 struct se_cmd *cmd,
5598 u8 *asc,
5599 u8 *ascq)
5600{
5601 *asc = cmd->scsi_asc;
5602 *ascq = cmd->scsi_ascq;
5603
5604 return 0;
5605}
5606
5607static int transport_set_sense_codes(
5608 struct se_cmd *cmd,
5609 u8 asc,
5610 u8 ascq)
5611{
5612 cmd->scsi_asc = asc;
5613 cmd->scsi_ascq = ascq;
5614
5615 return 0;
5616}
5617
5618int transport_send_check_condition_and_sense(
5619 struct se_cmd *cmd,
5620 u8 reason,
5621 int from_transport)
5622{
5623 unsigned char *buffer = cmd->sense_buffer;
5624 unsigned long flags;
5625 int offset;
5626 u8 asc = 0, ascq = 0;
5627
5628 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5629 if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
5630 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5631 return 0;
5632 }
5633 cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
5634 spin_unlock_irqrestore(&T_TASK(cmd)->t_state_lock, flags);
5635
5636 if (!reason && from_transport)
5637 goto after_reason;
5638
5639 if (!from_transport)
5640 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
5641 /*
5642 * Data Segment and SenseLength of the fabric response PDU.
5643 *
5644 * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE
5645 * from include/scsi/scsi_cmnd.h
5646 */
5647 offset = CMD_TFO(cmd)->set_fabric_sense_len(cmd,
5648 TRANSPORT_SENSE_BUFFER);
5649 /*
5650 * Actual SENSE DATA, see SPC-3 7.23.2 SPC_SENSE_KEY_OFFSET uses
5651 * SENSE KEY values from include/scsi/scsi.h
5652 */
5653 switch (reason) {
5654 case TCM_NON_EXISTENT_LUN:
5655 case TCM_UNSUPPORTED_SCSI_OPCODE:
5656 case TCM_SECTOR_COUNT_TOO_MANY:
5657 /* CURRENT ERROR */
5658 buffer[offset] = 0x70;
5659 /* ILLEGAL REQUEST */
5660 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5661 /* INVALID COMMAND OPERATION CODE */
5662 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20;
5663 break;
5664 case TCM_UNKNOWN_MODE_PAGE:
5665 /* CURRENT ERROR */
5666 buffer[offset] = 0x70;
5667 /* ILLEGAL REQUEST */
5668 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5669 /* INVALID FIELD IN CDB */
5670 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
5671 break;
5672 case TCM_CHECK_CONDITION_ABORT_CMD:
5673 /* CURRENT ERROR */
5674 buffer[offset] = 0x70;
5675 /* ABORTED COMMAND */
5676 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5677 /* BUS DEVICE RESET FUNCTION OCCURRED */
5678 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29;
5679 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03;
5680 break;
5681 case TCM_INCORRECT_AMOUNT_OF_DATA:
5682 /* CURRENT ERROR */
5683 buffer[offset] = 0x70;
5684 /* ABORTED COMMAND */
5685 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5686 /* WRITE ERROR */
5687 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
5688 /* NOT ENOUGH UNSOLICITED DATA */
5689 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d;
5690 break;
5691 case TCM_INVALID_CDB_FIELD:
5692 /* CURRENT ERROR */
5693 buffer[offset] = 0x70;
5694 /* ABORTED COMMAND */
5695 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5696 /* INVALID FIELD IN CDB */
5697 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
5698 break;
5699 case TCM_INVALID_PARAMETER_LIST:
5700 /* CURRENT ERROR */
5701 buffer[offset] = 0x70;
5702 /* ABORTED COMMAND */
5703 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5704 /* INVALID FIELD IN PARAMETER LIST */
5705 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26;
5706 break;
5707 case TCM_UNEXPECTED_UNSOLICITED_DATA:
5708 /* CURRENT ERROR */
5709 buffer[offset] = 0x70;
5710 /* ABORTED COMMAND */
5711 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5712 /* WRITE ERROR */
5713 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
5714 /* UNEXPECTED_UNSOLICITED_DATA */
5715 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c;
5716 break;
5717 case TCM_SERVICE_CRC_ERROR:
5718 /* CURRENT ERROR */
5719 buffer[offset] = 0x70;
5720 /* ABORTED COMMAND */
5721 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5722 /* PROTOCOL SERVICE CRC ERROR */
5723 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47;
5724 /* N/A */
5725 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05;
5726 break;
5727 case TCM_SNACK_REJECTED:
5728 /* CURRENT ERROR */
5729 buffer[offset] = 0x70;
5730 /* ABORTED COMMAND */
5731 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5732 /* READ ERROR */
5733 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11;
5734 /* FAILED RETRANSMISSION REQUEST */
5735 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13;
5736 break;
5737 case TCM_WRITE_PROTECTED:
5738 /* CURRENT ERROR */
5739 buffer[offset] = 0x70;
5740 /* DATA PROTECT */
5741 buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
5742 /* WRITE PROTECTED */
5743 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27;
5744 break;
5745 case TCM_CHECK_CONDITION_UNIT_ATTENTION:
5746 /* CURRENT ERROR */
5747 buffer[offset] = 0x70;
5748 /* UNIT ATTENTION */
5749 buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
5750 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
5751 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
5752 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
5753 break;
5754 case TCM_CHECK_CONDITION_NOT_READY:
5755 /* CURRENT ERROR */
5756 buffer[offset] = 0x70;
5757 /* Not Ready */
5758 buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY;
5759 transport_get_sense_codes(cmd, &asc, &ascq);
5760 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
5761 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
5762 break;
5763 case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
5764 default:
5765 /* CURRENT ERROR */
5766 buffer[offset] = 0x70;
5767 /* ILLEGAL REQUEST */
5768 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5769 /* LOGICAL UNIT COMMUNICATION FAILURE */
5770 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80;
5771 break;
5772 }
5773 /*
5774 * This code uses linux/include/scsi/scsi.h SAM status codes!
5775 */
5776 cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
5777 /*
5778 * Automatically padded, this value is encoded in the fabric's
5779 * data_length response PDU containing the SCSI defined sense data.
5780 */
5781 cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset;
5782
5783after_reason:
5784 CMD_TFO(cmd)->queue_status(cmd);
5785 return 0;
5786}
5787EXPORT_SYMBOL(transport_send_check_condition_and_sense);
5788
5789int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
5790{
5791 int ret = 0;
5792
5793 if (atomic_read(&T_TASK(cmd)->t_transport_aborted) != 0) {
5794 if (!(send_status) ||
5795 (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
5796 return 1;
5797#if 0
5798 printk(KERN_INFO "Sending delayed SAM_STAT_TASK_ABORTED"
5799 " status for CDB: 0x%02x ITT: 0x%08x\n",
5800 T_TASK(cmd)->t_task_cdb[0],
5801 CMD_TFO(cmd)->get_task_tag(cmd));
5802#endif
5803 cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
5804 CMD_TFO(cmd)->queue_status(cmd);
5805 ret = 1;
5806 }
5807 return ret;
5808}
5809EXPORT_SYMBOL(transport_check_aborted_status);
5810
5811void transport_send_task_abort(struct se_cmd *cmd)
5812{
5813 /*
5814 * If there are still expected incoming fabric WRITEs, we wait
5815 * until until they have completed before sending a TASK_ABORTED
5816 * response. This response with TASK_ABORTED status will be
5817 * queued back to fabric module by transport_check_aborted_status().
5818 */
5819 if (cmd->data_direction == DMA_TO_DEVICE) {
5820 if (CMD_TFO(cmd)->write_pending_status(cmd) != 0) {
5821 atomic_inc(&T_TASK(cmd)->t_transport_aborted);
5822 smp_mb__after_atomic_inc();
5823 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
5824 transport_new_cmd_failure(cmd);
5825 return;
5826 }
5827 }
5828 cmd->scsi_status = SAM_STAT_TASK_ABORTED;
5829#if 0
5830 printk(KERN_INFO "Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
5831 " ITT: 0x%08x\n", T_TASK(cmd)->t_task_cdb[0],
5832 CMD_TFO(cmd)->get_task_tag(cmd));
5833#endif
5834 CMD_TFO(cmd)->queue_status(cmd);
5835}
5836
5837/* transport_generic_do_tmr():
5838 *
5839 *
5840 */
5841int transport_generic_do_tmr(struct se_cmd *cmd)
5842{
5843 struct se_cmd *ref_cmd;
5844 struct se_device *dev = SE_DEV(cmd);
5845 struct se_tmr_req *tmr = cmd->se_tmr_req;
5846 int ret;
5847
5848 switch (tmr->function) {
5c6cd613 5849 case TMR_ABORT_TASK:
c66ac9db
NB		 5850 ref_cmd = tmr->ref_cmd;
5851 tmr->response = TMR_FUNCTION_REJECTED;
5852 break;
5c6cd613
NB		 5853 case TMR_ABORT_TASK_SET:
5854 case TMR_CLEAR_ACA:
5855 case TMR_CLEAR_TASK_SET:
c66ac9db
NB		 5856 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
5857 break;
5c6cd613 5858 case TMR_LUN_RESET:
c66ac9db
NB		 5859 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
5860 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
5861 TMR_FUNCTION_REJECTED;
5862 break;
5c6cd613 5863 case TMR_TARGET_WARM_RESET:
c66ac9db
NB		 5864 tmr->response = TMR_FUNCTION_REJECTED;
5865 break;
5c6cd613 5866 case TMR_TARGET_COLD_RESET:
c66ac9db
NB		 5867 tmr->response = TMR_FUNCTION_REJECTED;
5868 break;
c66ac9db
NB		 5869 default:
5870 printk(KERN_ERR "Uknown TMR function: 0x%02x.\n",
5871 tmr->function);
5872 tmr->response = TMR_FUNCTION_REJECTED;
5873 break;
5874 }
5875
5876 cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
5877 CMD_TFO(cmd)->queue_tm_rsp(cmd);
5878
5879 transport_cmd_check_stop(cmd, 2, 0);
5880 return 0;
5881}
5882
5883/*
5884 * Called with spin_lock_irq(&dev->execute_task_lock); held
5885 *
5886 */
5887static struct se_task *
5888transport_get_task_from_state_list(struct se_device *dev)
5889{
5890 struct se_task *task;
5891
5892 if (list_empty(&dev->state_task_list))
5893 return NULL;
5894
5895 list_for_each_entry(task, &dev->state_task_list, t_state_list)
5896 break;
5897
5898 list_del(&task->t_state_list);
5899 atomic_set(&task->task_state_active, 0);
5900
5901 return task;
5902}
5903
5904static void transport_processing_shutdown(struct se_device *dev)
5905{
5906 struct se_cmd *cmd;
5907 struct se_queue_req *qr;
5908 struct se_task *task;
5909 u8 state;
5910 unsigned long flags;
5911 /*
5912 * Empty the struct se_device's struct se_task state list.
5913 */
5914 spin_lock_irqsave(&dev->execute_task_lock, flags);
5915 while ((task = transport_get_task_from_state_list(dev))) {
5916 if (!(TASK_CMD(task))) {
5917 printk(KERN_ERR "TASK_CMD(task) is NULL!\n");
5918 continue;
5919 }
5920 cmd = TASK_CMD(task);
5921
5922 if (!T_TASK(cmd)) {
5923 printk(KERN_ERR "T_TASK(cmd) is NULL for task: %p cmd:"
5924 " %p ITT: 0x%08x\n", task, cmd,
5925 CMD_TFO(cmd)->get_task_tag(cmd));
5926 continue;
5927 }
5928 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
5929
5930 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5931
5932 DEBUG_DO("PT: cmd: %p task: %p ITT/CmdSN: 0x%08x/0x%08x,"
5933 " i_state/def_i_state: %d/%d, t_state/def_t_state:"
5934 " %d/%d cdb: 0x%02x\n", cmd, task,
5935 CMD_TFO(cmd)->get_task_tag(cmd), cmd->cmd_sn,
5936 CMD_TFO(cmd)->get_cmd_state(cmd), cmd->deferred_i_state,
5937 cmd->t_state, cmd->deferred_t_state,
5938 T_TASK(cmd)->t_task_cdb[0]);
5939 DEBUG_DO("PT: ITT[0x%08x] - t_task_cdbs: %d t_task_cdbs_left:"
5940 " %d t_task_cdbs_sent: %d -- t_transport_active: %d"
5941 " t_transport_stop: %d t_transport_sent: %d\n",
5942 CMD_TFO(cmd)->get_task_tag(cmd),
5943 T_TASK(cmd)->t_task_cdbs,
5944 atomic_read(&T_TASK(cmd)->t_task_cdbs_left),
5945 atomic_read(&T_TASK(cmd)->t_task_cdbs_sent),
5946 atomic_read(&T_TASK(cmd)->t_transport_active),
5947 atomic_read(&T_TASK(cmd)->t_transport_stop),
5948 atomic_read(&T_TASK(cmd)->t_transport_sent));
5949
5950 if (atomic_read(&task->task_active)) {
5951 atomic_set(&task->task_stop, 1);
5952 spin_unlock_irqrestore(
5953 &T_TASK(cmd)->t_state_lock, flags);
5954
5955 DEBUG_DO("Waiting for task: %p to shutdown for dev:"
5956 " %p\n", task, dev);
5957 wait_for_completion(&task->task_stop_comp);
5958 DEBUG_DO("Completed task: %p shutdown for dev: %p\n",
5959 task, dev);
5960
5961 spin_lock_irqsave(&T_TASK(cmd)->t_state_lock, flags);
5962 atomic_dec(&T_TASK(cmd)->t_task_cdbs_left);
5963
5964 atomic_set(&task->task_active, 0);
5965 atomic_set(&task->task_stop, 0);
52208ae3
NB		 5966 } else {
5967 if (atomic_read(&task->task_execute_queue) != 0)
5968 transport_remove_task_from_execute_queue(task, dev);
c66ac9db
NB		 5969 }
5970 __transport_stop_task_timer(task, &flags);
5971
5972 if (!(atomic_dec_and_test(&T_TASK(cmd)->t_task_cdbs_ex_left))) {
5973 spin_unlock_irqrestore(
5974 &T_TASK(cmd)->t_state_lock, flags);
5975
5976 DEBUG_DO("Skipping task: %p, dev: %p for"
5977 " t_task_cdbs_ex_left: %d\n", task, dev,
5978 atomic_read(&T_TASK(cmd)->t_task_cdbs_ex_left));
5979
5980 spin_lock_irqsave(&dev->execute_task_lock, flags);
5981 continue;
5982 }
5983
5984 if (atomic_read(&T_TASK(cmd)->t_transport_active)) {
5985 DEBUG_DO("got t_transport_active = 1 for task: %p, dev:"
5986 " %p\n", task, dev);
5987
5988 if (atomic_read(&T_TASK(cmd)->t_fe_count)) {
5989 spin_unlock_irqrestore(
5990 &T_TASK(cmd)->t_state_lock, flags);
5991 transport_send_check_condition_and_sense(
5992 cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE,
5993 0);
5994 transport_remove_cmd_from_queue(cmd,
5995 SE_DEV(cmd)->dev_queue_obj);
5996
5997 transport_lun_remove_cmd(cmd);
5998 transport_cmd_check_stop(cmd, 1, 0);
5999 } else {
6000 spin_unlock_irqrestore(
6001 &T_TASK(cmd)->t_state_lock, flags);
6002
6003 transport_remove_cmd_from_queue(cmd,
6004 SE_DEV(cmd)->dev_queue_obj);
6005
6006 transport_lun_remove_cmd(cmd);
6007
6008 if (transport_cmd_check_stop(cmd, 1, 0))
6009 transport_generic_remove(cmd, 0, 0);
6010 }
6011
6012 spin_lock_irqsave(&dev->execute_task_lock, flags);
6013 continue;
6014 }
6015 DEBUG_DO("Got t_transport_active = 0 for task: %p, dev: %p\n",
6016 task, dev);
6017
6018 if (atomic_read(&T_TASK(cmd)->t_fe_count)) {
6019 spin_unlock_irqrestore(
6020 &T_TASK(cmd)->t_state_lock, flags);
6021 transport_send_check_condition_and_sense(cmd,
6022 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
6023 transport_remove_cmd_from_queue(cmd,
6024 SE_DEV(cmd)->dev_queue_obj);
6025
6026 transport_lun_remove_cmd(cmd);
6027 transport_cmd_check_stop(cmd, 1, 0);
6028 } else {
6029 spin_unlock_irqrestore(
6030 &T_TASK(cmd)->t_state_lock, flags);
6031
6032 transport_remove_cmd_from_queue(cmd,
6033 SE_DEV(cmd)->dev_queue_obj);
6034 transport_lun_remove_cmd(cmd);
6035
6036 if (transport_cmd_check_stop(cmd, 1, 0))
6037 transport_generic_remove(cmd, 0, 0);
6038 }
6039
6040 spin_lock_irqsave(&dev->execute_task_lock, flags);
6041 }
6042 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
6043 /*
6044 * Empty the struct se_device's struct se_cmd list.
6045 */
6046 spin_lock_irqsave(&dev->dev_queue_obj->cmd_queue_lock, flags);
6047 while ((qr = __transport_get_qr_from_queue(dev->dev_queue_obj))) {
6048 spin_unlock_irqrestore(
6049 &dev->dev_queue_obj->cmd_queue_lock, flags);
6050 cmd = (struct se_cmd *)qr->cmd;
6051 state = qr->state;
6052 kfree(qr);
6053
6054 DEBUG_DO("From Device Queue: cmd: %p t_state: %d\n",
6055 cmd, state);
6056
6057 if (atomic_read(&T_TASK(cmd)->t_fe_count)) {
6058 transport_send_check_condition_and_sense(cmd,
6059 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
6060
6061 transport_lun_remove_cmd(cmd);
6062 transport_cmd_check_stop(cmd, 1, 0);
6063 } else {
6064 transport_lun_remove_cmd(cmd);
6065 if (transport_cmd_check_stop(cmd, 1, 0))
6066 transport_generic_remove(cmd, 0, 0);
6067 }
6068 spin_lock_irqsave(&dev->dev_queue_obj->cmd_queue_lock, flags);
6069 }
6070 spin_unlock_irqrestore(&dev->dev_queue_obj->cmd_queue_lock, flags);
6071}
6072
6073/* transport_processing_thread():
6074 *
6075 *
6076 */
6077static int transport_processing_thread(void *param)
6078{
6079 int ret, t_state;
6080 struct se_cmd *cmd;
6081 struct se_device *dev = (struct se_device *) param;
6082 struct se_queue_req *qr;
6083
6084 set_user_nice(current, -20);
6085
6086 while (!kthread_should_stop()) {
6087 ret = wait_event_interruptible(dev->dev_queue_obj->thread_wq,
6088 atomic_read(&dev->dev_queue_obj->queue_cnt) ||
6089 kthread_should_stop());
6090 if (ret < 0)
6091 goto out;
6092
6093 spin_lock_irq(&dev->dev_status_lock);
6094 if (dev->dev_status & TRANSPORT_DEVICE_SHUTDOWN) {
6095 spin_unlock_irq(&dev->dev_status_lock);
6096 transport_processing_shutdown(dev);
6097 continue;
6098 }
6099 spin_unlock_irq(&dev->dev_status_lock);
6100
6101get_cmd:
6102 __transport_execute_tasks(dev);
6103
6104 qr = transport_get_qr_from_queue(dev->dev_queue_obj);
6105 if (!(qr))
6106 continue;
6107
6108 cmd = (struct se_cmd *)qr->cmd;
6109 t_state = qr->state;
6110 kfree(qr);
6111
6112 switch (t_state) {
6113 case TRANSPORT_NEW_CMD_MAP:
6114 if (!(CMD_TFO(cmd)->new_cmd_map)) {
6115 printk(KERN_ERR "CMD_TFO(cmd)->new_cmd_map is"
6116 " NULL for TRANSPORT_NEW_CMD_MAP\n");
6117 BUG();
6118 }
6119 ret = CMD_TFO(cmd)->new_cmd_map(cmd);
6120 if (ret < 0) {
6121 cmd->transport_error_status = ret;
6122 transport_generic_request_failure(cmd, NULL,
6123 0, (cmd->data_direction !=
6124 DMA_TO_DEVICE));
6125 break;
6126 }
6127 /* Fall through */
6128 case TRANSPORT_NEW_CMD:
6129 ret = transport_generic_new_cmd(cmd);
6130 if (ret < 0) {
6131 cmd->transport_error_status = ret;
6132 transport_generic_request_failure(cmd, NULL,
6133 0, (cmd->data_direction !=
6134 DMA_TO_DEVICE));
6135 }
6136 break;
6137 case TRANSPORT_PROCESS_WRITE:
6138 transport_generic_process_write(cmd);
6139 break;
6140 case TRANSPORT_COMPLETE_OK:
6141 transport_stop_all_task_timers(cmd);
6142 transport_generic_complete_ok(cmd);
6143 break;
6144 case TRANSPORT_REMOVE:
6145 transport_generic_remove(cmd, 1, 0);
6146 break;
f4366772
NB		 6147 case TRANSPORT_FREE_CMD_INTR:
6148 transport_generic_free_cmd(cmd, 0, 1, 0);
6149 break;
c66ac9db
NB		 6150 case TRANSPORT_PROCESS_TMR:
6151 transport_generic_do_tmr(cmd);
6152 break;
6153 case TRANSPORT_COMPLETE_FAILURE:
6154 transport_generic_request_failure(cmd, NULL, 1, 1);
6155 break;
6156 case TRANSPORT_COMPLETE_TIMEOUT:
6157 transport_stop_all_task_timers(cmd);
6158 transport_generic_request_timeout(cmd);
6159 break;
6160 default:
6161 printk(KERN_ERR "Unknown t_state: %d deferred_t_state:"
6162 " %d for ITT: 0x%08x i_state: %d on SE LUN:"
6163 " %u\n", t_state, cmd->deferred_t_state,
6164 CMD_TFO(cmd)->get_task_tag(cmd),
6165 CMD_TFO(cmd)->get_cmd_state(cmd),
6166 SE_LUN(cmd)->unpacked_lun);
6167 BUG();
6168 }
6169
6170 goto get_cmd;
6171 }
6172
6173out:
6174 transport_release_all_cmds(dev);
6175 dev->process_thread = NULL;
6176 return 0;
6177}
Linux 6.x block layer and io_uring tree(s)