SCST core

diff -uprN orig/linux-2.6.33/include/scst/scst_const.h linux-2.6.33/include/scst/scst_const.h
@@ -0,0 +1,330 @@
+/*
+ *  include/scst_const.h
+ *
+ *  Copyright (C) 2004 - 2010 Vladislav Bolkhovitin <[email protected]>
+ *  Copyright (C) 2007 - 2010 ID7 Ltd.
+ *
+ *  Contains common SCST constants.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation, version 2
+ *  of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ */
+
+#ifndef __SCST_CONST_H
+#define __SCST_CONST_H
+
+#include <scsi/scsi.h>
+
+#define SCST_CONST_VERSION "$Revision: 1585 $"
+
+/*** Shared constants between user and kernel spaces ***/
+
+/* Max size of CDB */
+#define SCST_MAX_CDB_SIZE            16
+
+/* Max size of various names */
+#define SCST_MAX_NAME		     50
+
+/* Max size of external names, like initiator name */
+#define SCST_MAX_EXTERNAL_NAME	     256
+
+/*
+ * Size of sense sufficient to carry standard sense data.
+ * Warning! It's allocated on stack!
+ */
+#define SCST_STANDARD_SENSE_LEN      18
+
+/* Max size of sense */
+#define SCST_SENSE_BUFFERSIZE        96
+
+/*************************************************************
+ ** Allowed delivery statuses for cmd's delivery_status
+ *************************************************************/
+
+#define SCST_CMD_DELIVERY_SUCCESS	0
+#define SCST_CMD_DELIVERY_FAILED	-1
+#define SCST_CMD_DELIVERY_ABORTED	-2
+
+/*************************************************************
+ ** Values for task management functions
+ *************************************************************/
+#define SCST_ABORT_TASK              0
+#define SCST_ABORT_TASK_SET          1
+#define SCST_CLEAR_ACA               2
+#define SCST_CLEAR_TASK_SET          3
+#define SCST_LUN_RESET               4
+#define SCST_TARGET_RESET            5
+
+/** SCST extensions **/
+
+/*
+ * Notifies about I_T nexus loss event in the corresponding session.
+ * Aborts all tasks there, resets the reservation, if any, and sets
+ * up the I_T Nexus loss UA.
+ */
+#define SCST_NEXUS_LOSS_SESS         6
+
+/* Aborts all tasks in the corresponding session */
+#define SCST_ABORT_ALL_TASKS_SESS    7
+
+/*
+ * Notifies about I_T nexus loss event. Aborts all tasks in all sessions
+ * of the tgt, resets the reservations, if any,  and sets up the I_T Nexus
+ * loss UA.
+ */
+#define SCST_NEXUS_LOSS              8
+
+/* Aborts all tasks in all sessions of the tgt */
+#define SCST_ABORT_ALL_TASKS         9
+
+/*
+ * Internal TM command issued by SCST in scst_unregister_session(). It is the
+ * same as SCST_NEXUS_LOSS_SESS, except:
+ *  - it doesn't call task_mgmt_affected_cmds_done()
+ *  - it doesn't call task_mgmt_fn_done()
+ *  - it doesn't queue NEXUS LOSS UA.
+ *
+ * Target driver shall NEVER use it!!
+ */
+#define SCST_UNREG_SESS_TM           10
+
+/*************************************************************
+ ** Values for mgmt cmd's status field. Codes taken from iSCSI
+ *************************************************************/
+#define SCST_MGMT_STATUS_SUCCESS		0
+#define SCST_MGMT_STATUS_TASK_NOT_EXIST		-1
+#define SCST_MGMT_STATUS_LUN_NOT_EXIST		-2
+#define SCST_MGMT_STATUS_FN_NOT_SUPPORTED	-5
+#define SCST_MGMT_STATUS_REJECTED		-255
+#define SCST_MGMT_STATUS_FAILED			-129
+
+/*************************************************************
+ ** SCSI task attribute queue types
+ *************************************************************/
+enum scst_cmd_queue_type {
+	SCST_CMD_QUEUE_UNTAGGED = 0,
+	SCST_CMD_QUEUE_SIMPLE,
+	SCST_CMD_QUEUE_ORDERED,
+	SCST_CMD_QUEUE_HEAD_OF_QUEUE,
+	SCST_CMD_QUEUE_ACA
+};
+
+/*************************************************************
+ ** CDB flags
+ *************************************************************/
+enum scst_cdb_flags {
+	SCST_TRANSFER_LEN_TYPE_FIXED =		0x001,
+	SCST_SMALL_TIMEOUT =			0x002,
+	SCST_LONG_TIMEOUT =			0x004,
+	SCST_UNKNOWN_LENGTH =			0x008,
+	SCST_INFO_VALID =			0x010, /* must be single bit */
+	SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED =	0x020,
+	SCST_IMPLICIT_HQ =			0x040,
+	SCST_SKIP_UA =				0x080,
+	SCST_WRITE_MEDIUM =			0x100,
+	SCST_LOCAL_CMD =			0x200,
+	SCST_FULLY_LOCAL_CMD =			0x400,
+	SCST_REG_RESERVE_ALLOWED =		0x800,
+};
+
+/*************************************************************
+ ** Data direction aliases. Changing it don't forget to change
+ ** scst_to_tgt_dma_dir as well!!
+ *************************************************************/
+#define SCST_DATA_UNKNOWN		0
+#define SCST_DATA_WRITE			1
+#define SCST_DATA_READ			2
+#define SCST_DATA_BIDI			(SCST_DATA_WRITE | SCST_DATA_READ)
+#define SCST_DATA_NONE			4
+
+/*************************************************************
+ ** Default suffix for targets with NULL names
+ *************************************************************/
+#define SCST_DEFAULT_TGT_NAME_SUFFIX		"_target_"
+
+/*************************************************************
+ ** Sense manipulation and examination
+ *************************************************************/
+#define SCST_LOAD_SENSE(key_asc_ascq) key_asc_ascq
+
+#define SCST_SENSE_VALID(sense)  ((sense != NULL) && \
+				  ((((const uint8_t *)(sense))[0] & 0x70) == 0x70))
+
+#define SCST_NO_SENSE(sense)     ((sense != NULL) && \
+				  (((const uint8_t *)(sense))[2] == 0))
+
+/*************************************************************
+ ** Sense data for the appropriate errors. Can be used with
+ ** scst_set_cmd_error()
+ *************************************************************/
+#define scst_sense_no_sense			NO_SENSE,        0x00, 0
+#define scst_sense_hardw_error			HARDWARE_ERROR,  0x44, 0
+#define scst_sense_aborted_command		ABORTED_COMMAND, 0x00, 0
+#define scst_sense_invalid_opcode		ILLEGAL_REQUEST, 0x20, 0
+#define scst_sense_invalid_field_in_cdb		ILLEGAL_REQUEST, 0x24, 0
+#define scst_sense_invalid_field_in_parm_list	ILLEGAL_REQUEST, 0x26, 0
+#define scst_sense_parameter_value_invalid	ILLEGAL_REQUEST, 0x26, 2
+#define scst_sense_reset_UA			UNIT_ATTENTION,  0x29, 0
+#define scst_sense_nexus_loss_UA		UNIT_ATTENTION,  0x29, 0x7
+#define scst_sense_saving_params_unsup		ILLEGAL_REQUEST, 0x39, 0
+#define scst_sense_lun_not_supported		ILLEGAL_REQUEST, 0x25, 0
+#define scst_sense_data_protect			DATA_PROTECT,    0x00, 0
+#define scst_sense_miscompare_error		MISCOMPARE,      0x1D, 0
+#define scst_sense_block_out_range_error	ILLEGAL_REQUEST, 0x21, 0
+#define scst_sense_medium_changed_UA		UNIT_ATTENTION,  0x28, 0
+#define scst_sense_read_error			MEDIUM_ERROR,    0x11, 0
+#define scst_sense_write_error			MEDIUM_ERROR,    0x03, 0
+#define scst_sense_not_ready			NOT_READY,       0x04, 0x10
+#define scst_sense_invalid_message		ILLEGAL_REQUEST, 0x49, 0
+#define scst_sense_cleared_by_another_ini_UA	UNIT_ATTENTION,  0x2F, 0
+#define scst_sense_capacity_data_changed	UNIT_ATTENTION,  0x2A, 0x9
+#define scst_sense_reported_luns_data_changed	UNIT_ATTENTION,  0x3F, 0xE
+#define scst_sense_inquery_data_changed		UNIT_ATTENTION,  0x3F, 0x3
+
+/*************************************************************
+ * SCSI opcodes not listed anywhere else
+ *************************************************************/
+#define REPORT_DEVICE_IDENTIFIER    0xA3
+#define INIT_ELEMENT_STATUS         0x07
+#define INIT_ELEMENT_STATUS_RANGE   0x37
+#define PREVENT_ALLOW_MEDIUM        0x1E
+#define READ_ATTRIBUTE              0x8C
+#define REQUEST_VOLUME_ADDRESS      0xB5
+#define WRITE_ATTRIBUTE             0x8D
+#define WRITE_VERIFY_16             0x8E
+#define VERIFY_6                    0x13
+#define VERIFY_12                   0xAF
+
+/*************************************************************
+ **  SCSI Architecture Model (SAM) Status codes. Taken from SAM-3 draft
+ **  T10/1561-D Revision 4 Draft dated 7th November 2002.
+ *************************************************************/
+#define SAM_STAT_GOOD            0x00
+#define SAM_STAT_CHECK_CONDITION 0x02
+#define SAM_STAT_CONDITION_MET   0x04
+#define SAM_STAT_BUSY            0x08
+#define SAM_STAT_INTERMEDIATE    0x10
+#define SAM_STAT_INTERMEDIATE_CONDITION_MET 0x14
+#define SAM_STAT_RESERVATION_CONFLICT 0x18
+#define SAM_STAT_COMMAND_TERMINATED 0x22	/* obsolete in SAM-3 */
+#define SAM_STAT_TASK_SET_FULL   0x28
+#define SAM_STAT_ACA_ACTIVE      0x30
+#define SAM_STAT_TASK_ABORTED    0x40
+
+/*************************************************************
+ ** Control byte field in CDB
+ *************************************************************/
+#define CONTROL_BYTE_LINK_BIT       0x01
+#define CONTROL_BYTE_NACA_BIT       0x04
+
+/*************************************************************
+ ** Byte 1 in INQUIRY CDB
+ *************************************************************/
+#define SCST_INQ_EVPD                0x01
+
+/*************************************************************
+ ** Byte 3 in Standard INQUIRY data
+ *************************************************************/
+#define SCST_INQ_BYTE3               3
+
+#define SCST_INQ_NORMACA_BIT         0x20
+
+/*************************************************************
+ ** Byte 2 in RESERVE_10 CDB
+ *************************************************************/
+#define SCST_RES_3RDPTY              0x10
+#define SCST_RES_LONGID              0x02
+
+/*************************************************************
+ ** Values for the control mode page TST field
+ *************************************************************/
+#define SCST_CONTR_MODE_ONE_TASK_SET  0
+#define SCST_CONTR_MODE_SEP_TASK_SETS 1
+
+/*******************************************************************
+ ** Values for the control mode page QUEUE ALGORITHM MODIFIER field
+ *******************************************************************/
+#define SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER   0
+#define SCST_CONTR_MODE_QUEUE_ALG_UNRESTRICTED_REORDER 1
+
+/*************************************************************
+ ** Values for the control mode page D_SENSE field
+ *************************************************************/
+#define SCST_CONTR_MODE_FIXED_SENSE  0
+#define SCST_CONTR_MODE_DESCR_SENSE 1
+
+/*************************************************************
+ ** Misc SCSI constants
+ *************************************************************/
+#define SCST_SENSE_ASC_UA_RESET      0x29
+#define READ_CAP_LEN		     8
+#define READ_CAP16_LEN		     32
+#define BYTCHK			     0x02
+#define POSITION_LEN_SHORT           20
+#define POSITION_LEN_LONG            32
+
+/*************************************************************
+ ** Various timeouts
+ *************************************************************/
+#define SCST_DEFAULT_TIMEOUT			(60 * HZ)
+
+#define SCST_GENERIC_CHANGER_TIMEOUT		(3 * HZ)
+#define SCST_GENERIC_CHANGER_LONG_TIMEOUT	(14000 * HZ)
+
+#define SCST_GENERIC_PROCESSOR_TIMEOUT		(3 * HZ)
+#define SCST_GENERIC_PROCESSOR_LONG_TIMEOUT	(14000 * HZ)
+
+#define SCST_GENERIC_TAPE_SMALL_TIMEOUT		(3 * HZ)
+#define SCST_GENERIC_TAPE_REG_TIMEOUT		(900 * HZ)
+#define SCST_GENERIC_TAPE_LONG_TIMEOUT		(14000 * HZ)
+
+#define SCST_GENERIC_MODISK_SMALL_TIMEOUT	(3 * HZ)
+#define SCST_GENERIC_MODISK_REG_TIMEOUT		(900 * HZ)
+#define SCST_GENERIC_MODISK_LONG_TIMEOUT	(14000 * HZ)
+
+#define SCST_GENERIC_DISK_SMALL_TIMEOUT		(3 * HZ)
+#define SCST_GENERIC_DISK_REG_TIMEOUT		(60 * HZ)
+#define SCST_GENERIC_DISK_LONG_TIMEOUT		(3600 * HZ)
+
+#define SCST_GENERIC_RAID_TIMEOUT		(3 * HZ)
+#define SCST_GENERIC_RAID_LONG_TIMEOUT		(14000 * HZ)
+
+#define SCST_GENERIC_CDROM_SMALL_TIMEOUT	(3 * HZ)
+#define SCST_GENERIC_CDROM_REG_TIMEOUT		(900 * HZ)
+#define SCST_GENERIC_CDROM_LONG_TIMEOUT		(14000 * HZ)
+
+#define SCST_MAX_OTHER_TIMEOUT			(14000 * HZ)
+
+/*************************************************************
+ ** I/O grouping attribute string values. Must match constants
+ ** w/o '_STR' suffix!
+ *************************************************************/
+#define SCST_IO_GROUPING_AUTO_STR		"auto"
+#define SCST_IO_GROUPING_THIS_GROUP_ONLY_STR	"this_group_only"
+#define SCST_IO_GROUPING_NEVER_STR		"never"
+
+/*************************************************************
+ ** Threads pool type attribute string values.
+ ** Must match scst_dev_type_threads_pool_type!
+ *************************************************************/
+#define SCST_THREADS_POOL_PER_INITIATOR_STR	"per_initiator"
+#define SCST_THREADS_POOL_SHARED_STR		"shared"
+
+/*************************************************************
+ ** Misc constants
+ *************************************************************/
+#define SCST_SYSFS_BLOCK_SIZE			PAGE_SIZE
+
+#define SCST_SYSFS_KEY_MARK			"[key]"
+
+#define SCST_MIN_REL_TGT_ID			1
+#define SCST_MAX_REL_TGT_ID			65535
+
+#endif /* __SCST_CONST_H */

2,670 lines not on this page · raw message

diff -uprN orig/linux-2.6.33/drivers/scst/scst_main.c linux-2.6.33/drivers/scst/scst_main.c
@@ -0,0 +1,2047 @@
+/*
+ *  scst_main.c
+ *
+ *  Copyright (C) 2004 - 2010 Vladislav Bolkhovitin <[email protected]>
+ *  Copyright (C) 2004 - 2005 Leonid Stoljar
+ *  Copyright (C) 2007 - 2010 ID7 Ltd.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation, version 2
+ *  of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/unistd.h>
+#include <linux/string.h>
+#include <linux/kthread.h>
+
+#include "scst.h"
+#include "scst_priv.h"
+#include "scst_mem.h"
+
+#if defined(CONFIG_HIGHMEM4G) || defined(CONFIG_HIGHMEM64G)
+#warning "HIGHMEM kernel configurations are fully supported, but not\
+ recommended for performance reasons. Consider changing VMSPLIT\
+ option or use a 64-bit configuration instead. See README file for\
+ details."
+#endif
+
+/**
+ ** SCST global variables. They are all uninitialized to have their layout in
+ ** memory be exactly as specified. Otherwise compiler puts zero-initialized
+ ** variable separately from nonzero-initialized ones.
+ **/
+
+/*
+ * Main SCST mutex. All targets, devices and dev_types management is done
+ * under this mutex.
+ *
+ * It must NOT be used in any works (schedule_work(), etc.), because
+ * otherwise a deadlock (double lock, actually) is possible, e.g., with
+ * scst_user detach_tgt(), which is called under scst_mutex and calls
+ * flush_scheduled_work().
+ */
+struct mutex scst_mutex;
+EXPORT_SYMBOL_GPL(scst_mutex);
+
+ /* All 3 protected by scst_mutex */
+struct list_head scst_template_list;
+struct list_head scst_dev_list;
+struct list_head scst_dev_type_list;
+
+spinlock_t scst_main_lock;
+
+static struct kmem_cache *scst_mgmt_cachep;
+mempool_t *scst_mgmt_mempool;
+static struct kmem_cache *scst_mgmt_stub_cachep;
+mempool_t *scst_mgmt_stub_mempool;
+static struct kmem_cache *scst_ua_cachep;
+mempool_t *scst_ua_mempool;
+static struct kmem_cache *scst_sense_cachep;
+mempool_t *scst_sense_mempool;
+static struct kmem_cache *scst_aen_cachep;
+mempool_t *scst_aen_mempool;
+struct kmem_cache *scst_tgtd_cachep;
+struct kmem_cache *scst_sess_cachep;
+struct kmem_cache *scst_acgd_cachep;
+
+unsigned int scst_setup_id;
+
+spinlock_t scst_init_lock;
+wait_queue_head_t scst_init_cmd_list_waitQ;
+struct list_head scst_init_cmd_list;
+unsigned int scst_init_poll_cnt;
+
+struct kmem_cache *scst_cmd_cachep;
+
+#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
+unsigned long scst_trace_flag;
+#endif
+
+unsigned long scst_flags;
+atomic_t scst_cmd_count;
+
+struct scst_cmd_threads scst_main_cmd_threads;
+
+struct scst_tasklet scst_tasklets[NR_CPUS];
+
+spinlock_t scst_mcmd_lock;
+struct list_head scst_active_mgmt_cmd_list;
+struct list_head scst_delayed_mgmt_cmd_list;
+wait_queue_head_t scst_mgmt_cmd_list_waitQ;
+
+wait_queue_head_t scst_mgmt_waitQ;
+spinlock_t scst_mgmt_lock;
+struct list_head scst_sess_init_list;
+struct list_head scst_sess_shut_list;
+
+wait_queue_head_t scst_dev_cmd_waitQ;
+
+static struct mutex scst_suspend_mutex;
+/* protected by scst_suspend_mutex */
+static struct list_head scst_cmd_threads_list;
+
+int scst_threads;
+static struct task_struct *scst_init_cmd_thread;
+static struct task_struct *scst_mgmt_thread;
+static struct task_struct *scst_mgmt_cmd_thread;
+
+static int suspend_count;
+
+static int scst_virt_dev_last_id; /* protected by scst_mutex */
+
+static unsigned int scst_max_cmd_mem;
+unsigned int scst_max_dev_cmd_mem;
+
+module_param_named(scst_threads, scst_threads, int, 0);
+MODULE_PARM_DESC(scst_threads, "SCSI target threads count");
+
+module_param_named(scst_max_cmd_mem, scst_max_cmd_mem, int, S_IRUGO);
+MODULE_PARM_DESC(scst_max_cmd_mem, "Maximum memory allowed to be consumed by "
+	"all SCSI commands of all devices at any given time in MB");
+
+module_param_named(scst_max_dev_cmd_mem, scst_max_dev_cmd_mem, int, S_IRUGO);
+MODULE_PARM_DESC(scst_max_dev_cmd_mem, "Maximum memory allowed to be consumed "
+	"by all SCSI commands of a device at any given time in MB");
+
+struct scst_dev_type scst_null_devtype = {
+	.name = "none",
+};
+
+static void __scst_resume_activity(void);
+
+/**
+ * __scst_register_target_template() - register target template.
+ * @vtt:	target template
+ * @version:	SCST_INTERFACE_VERSION version string to ensure that
+ *		SCST core and the target driver use the same version of
+ *		the SCST interface
+ *
+ * Description:
+ *    Registers a target template and returns 0 on success or appropriate
+ *    error code otherwise.
+ *
+ *    Note: *vtt must be static!
+ */
+int __scst_register_target_template(struct scst_tgt_template *vtt,
+	const char *version)
+{
+	int res = 0;
+	struct scst_tgt_template *t;
+	static DEFINE_MUTEX(m);
+
+	INIT_LIST_HEAD(&vtt->tgt_list);
+
+	if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
+		PRINT_ERROR("Incorrect version of target %s", vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	if (!vtt->detect) {
+		PRINT_ERROR("Target driver %s must have "
+			"detect() method.", vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	if (!vtt->release) {
+		PRINT_ERROR("Target driver %s must have "
+			"release() method.", vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	if (!vtt->xmit_response) {
+		PRINT_ERROR("Target driver %s must have "
+			"xmit_response() method.", vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	if (vtt->threads_num < 0) {
+		PRINT_ERROR("Wrong threads_num value %d for "
+			"target \"%s\"", vtt->threads_num,
+			vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	if (!vtt->enable_target || !vtt->is_target_enabled) {
+		PRINT_WARNING("Target driver %s doesn't have enable_target() "
+			"and/or is_target_enabled() method(s). This is unsafe "
+			"and can lead that initiators connected on the "
+			"initialization time can see an unexpected set of "
+			"devices or no devices at all!", vtt->name);
+	}
+
+	if (((vtt->add_target != NULL) && (vtt->del_target == NULL)) ||
+	    ((vtt->add_target == NULL) && (vtt->del_target != NULL))) {
+		PRINT_ERROR("Target driver %s must either define both "
+			"add_target() and del_target(), or none.", vtt->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	res = scst_create_tgtt_sysfs(vtt);
+	if (res)
+		goto out_sysfs_err;
+
+	if (vtt->rdy_to_xfer == NULL)
+		vtt->rdy_to_xfer_atomic = 1;
+
+	if (mutex_lock_interruptible(&m) != 0)
+		goto out_sysfs_err;
+
+	if (mutex_lock_interruptible(&scst_mutex) != 0)
+		goto out_m_err;
+	list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
+		if (strcmp(t->name, vtt->name) == 0) {
+			PRINT_ERROR("Target driver %s already registered",
+				vtt->name);
+			mutex_unlock(&scst_mutex);
+			goto out_m_err;
+		}
+	}
+	mutex_unlock(&scst_mutex);
+
+	TRACE_DBG("%s", "Calling target driver's detect()");
+	res = vtt->detect(vtt);
+	TRACE_DBG("Target driver's detect() returned %d", res);
+	if (res < 0) {
+		PRINT_ERROR("%s", "The detect() routine failed");
+		res = -EINVAL;
+		goto out_m_err;
+	}
+
+	mutex_lock(&scst_mutex);
+	list_add_tail(&vtt->scst_template_list_entry, &scst_template_list);
+	mutex_unlock(&scst_mutex);
+
+	res = 0;
+
+	PRINT_INFO("Target template %s registered successfully", vtt->name);
+
+	mutex_unlock(&m);
+
+out:
+	return res;
+
+out_m_err:
+	mutex_unlock(&m);
+
+out_sysfs_err:
+	scst_tgtt_sysfs_put(vtt);
+
+out_err:
+	PRINT_ERROR("Failed to register target template %s", vtt->name);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(__scst_register_target_template);
+
+static int scst_check_non_gpl_target_template(struct scst_tgt_template *vtt)
+{
+	int res;
+
+	if (vtt->task_mgmt_affected_cmds_done || vtt->threads_num) {
+		PRINT_ERROR("Not allowed functionality in non-GPL version for "
+			"target template %s", vtt->name);
+		res = -EPERM;
+		goto out;
+	}
+
+	res = 0;
+
+out:
+	return res;
+}
+
+/**
+ * __scst_register_target_template_non_gpl() - register target template,
+ *					      non-GPL version
+ * @vtt:	target template
+ * @version:	SCST_INTERFACE_VERSION version string to ensure that
+ *		SCST core and the target driver use the same version of
+ *		the SCST interface
+ *
+ * Description:
+ *    Registers a target template and returns 0 on success or appropriate
+ *    error code otherwise.
+ *
+ *    Note: *vtt must be static!
+ */
+int __scst_register_target_template_non_gpl(struct scst_tgt_template *vtt,
+	const char *version)
+{
+	int res;
+
+	res = scst_check_non_gpl_target_template(vtt);
+	if (res != 0)
+		goto out;
+
+	res = __scst_register_target_template(vtt, version);
+
+out:
+	return res;
+}
+EXPORT_SYMBOL(__scst_register_target_template_non_gpl);
+
+/**
+ * scst_unregister_target_template() - unregister target template
+ */
+void scst_unregister_target_template(struct scst_tgt_template *vtt)
+{
+	struct scst_tgt *tgt;
+	struct scst_tgt_template *t;
+	int found = 0;
+
+	mutex_lock(&scst_mutex);
+
+	list_for_each_entry(t, &scst_template_list, scst_template_list_entry) {
+		if (strcmp(t->name, vtt->name) == 0) {
+			found = 1;
+			break;
+		}
+	}
+	if (!found) {
+		PRINT_ERROR("Target driver %s isn't registered", vtt->name);
+		goto out_err_up;
+	}
+
+restart:
+	list_for_each_entry(tgt, &vtt->tgt_list, tgt_list_entry) {
+		mutex_unlock(&scst_mutex);
+		scst_unregister_target(tgt);
+		mutex_lock(&scst_mutex);
+		goto restart;
+	}
+	list_del(&vtt->scst_template_list_entry);
+
+	mutex_unlock(&scst_mutex);
+
+	scst_tgtt_sysfs_put(vtt);
+
+	PRINT_INFO("Target template %s unregistered successfully", vtt->name);
+
+out:
+	return;
+
+out_err_up:
+	mutex_unlock(&scst_mutex);
+	goto out;
+}
+EXPORT_SYMBOL(scst_unregister_target_template);
+
+/**
+ * scst_register_target() - register target
+ *
+ * Registers a target for template vtt and returns new target structure on
+ * success or NULL otherwise.
+ */
+struct scst_tgt *scst_register_target(struct scst_tgt_template *vtt,
+	const char *target_name)
+{
+	struct scst_tgt *tgt;
+	int rc = 0;
+
+	rc = scst_alloc_tgt(vtt, &tgt);
+	if (rc != 0)
+		goto out_err;
+
+	rc = scst_suspend_activity(true);
+	if (rc != 0)
+		goto out_free_tgt_err;
+
+	if (mutex_lock_interruptible(&scst_mutex) != 0) {
+		rc = -EINTR;
+		goto out_resume_free;
+	}
+
+	if (target_name != NULL) {
+
+		tgt->tgt_name = kmalloc(strlen(target_name) + 1, GFP_KERNEL);
+		if (tgt->tgt_name == NULL) {
+			TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name %s failed",
+				target_name);
+			rc = -ENOMEM;
+			goto out_unlock_resume;
+		}
+		strcpy(tgt->tgt_name, target_name);
+	} else {
+		static int tgt_num; /* protected by scst_mutex */
+		int len = strlen(vtt->name) +
+			strlen(SCST_DEFAULT_TGT_NAME_SUFFIX) + 11 + 1;
+
+		tgt->tgt_name = kmalloc(len, GFP_KERNEL);
+		if (tgt->tgt_name == NULL) {
+			TRACE(TRACE_OUT_OF_MEM, "Allocation of tgt name failed "
+				"(template name %s)", vtt->name);
+			rc = -ENOMEM;
+			goto out_unlock_resume;
+		}
+		sprintf(tgt->tgt_name, "%s%s%d", vtt->name,
+			SCST_DEFAULT_TGT_NAME_SUFFIX, tgt_num++);
+	}
+
+	tgt->default_acg = scst_alloc_add_acg(NULL, tgt->tgt_name);
+	if (tgt->default_acg == NULL)
+		goto out_free_tgt_name;
+
+	INIT_LIST_HEAD(&tgt->tgt_acg_list);
+
+	rc = scst_create_tgt_sysfs(tgt);
+	if (rc < 0)
+		goto out_clear_acg;
+
+	list_add_tail(&tgt->tgt_list_entry, &vtt->tgt_list);
+
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	PRINT_INFO("Target %s for template %s registered successfully",
+		tgt->tgt_name, vtt->name);
+
+	TRACE_DBG("tgt %p", tgt);
+
+out:
+	return tgt;
+
+out_clear_acg:
+	scst_clear_acg(tgt->default_acg);
+
+out_free_tgt_name:
+	kfree(tgt->tgt_name);
+
+out_unlock_resume:
+	mutex_unlock(&scst_mutex);
+
+out_resume_free:
+	scst_resume_activity();
+
+out_free_tgt_err:
+	scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
+	tgt = NULL;
+
+out_err:
+	PRINT_ERROR("Failed to register target %s for template %s (error %d)",
+		(tgt->tgt_name != NULL) ? tgt->tgt_name : target_name,
+		vtt->name, rc);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(scst_register_target);
+
+/**
+ * scst_register_target_non_gpl() - register target, non-GPL version
+ *
+ * Registers a target for template vtt and returns new target structure on
+ * success or NULL otherwise.
+ */
+struct scst_tgt *scst_register_target_non_gpl(struct scst_tgt_template *vtt,
+	const char *target_name)
+{
+	struct scst_tgt *res;
+
+	if (scst_check_non_gpl_target_template(vtt)) {
+		res = NULL;
+		goto out;
+	}
+
+	res = scst_register_target(vtt, target_name);
+
+out:
+	return res;
+}
+EXPORT_SYMBOL(scst_register_target_non_gpl);
+
+static inline int test_sess_list(struct scst_tgt *tgt)
+{
+	int res;
+	mutex_lock(&scst_mutex);
+	res = list_empty(&tgt->sess_list);
+	mutex_unlock(&scst_mutex);
+	return res;
+}
+
+/**
+ * scst_unregister_target() - unregister target
+ */
+void scst_unregister_target(struct scst_tgt *tgt)
+{
+	struct scst_session *sess;
+	struct scst_tgt_template *vtt = tgt->tgtt;
+	struct scst_acg *acg, *acg_tmp;
+
+	scst_tgt_sysfs_prepare_put(tgt);
+
+	TRACE_DBG("%s", "Calling target driver's release()");
+	tgt->tgtt->release(tgt);
+	TRACE_DBG("%s", "Target driver's release() returned");
+
+	mutex_lock(&scst_mutex);
+again:
+	list_for_each_entry(sess, &tgt->sess_list, sess_list_entry) {
+		if (sess->shut_phase == SCST_SESS_SPH_READY) {
+			/*
+			 * Sometimes it's hard for target driver to track all
+			 * its sessions (see scst_local, for example), so let's
+			 * help it.
+			 */
+			mutex_unlock(&scst_mutex);
+			scst_unregister_session(sess, 0, NULL);
+			mutex_lock(&scst_mutex);
+			goto again;
+		}
+	}
+	mutex_unlock(&scst_mutex);
+
+	TRACE_DBG("%s", "Waiting for sessions shutdown");
+	wait_event(tgt->unreg_waitQ, test_sess_list(tgt));
+	TRACE_DBG("%s", "wait_event() returned");
+
+	scst_suspend_activity(false);
+	mutex_lock(&scst_mutex);
+
+	list_del(&tgt->tgt_list_entry);
+
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	scst_clear_acg(tgt->default_acg);
+
+	list_for_each_entry_safe(acg, acg_tmp, &tgt->tgt_acg_list,
+					acg_list_entry) {
+		scst_acg_sysfs_put(acg);
+	}
+
+	del_timer_sync(&tgt->retry_timer);
+
+	PRINT_INFO("Target %s for template %s unregistered successfully",
+		tgt->tgt_name, vtt->name);
+
+	scst_tgt_sysfs_put(tgt); /* must not be called under scst_mutex */
+
+	TRACE_DBG("Unregistering tgt %p finished", tgt);
+	return;
+}
+EXPORT_SYMBOL(scst_unregister_target);
+
+static int scst_susp_wait(bool interruptible)
+{
+	int res = 0;
+
+	if (interruptible) {
+		res = wait_event_interruptible_timeout(scst_dev_cmd_waitQ,
+			(atomic_read(&scst_cmd_count) == 0),
+			SCST_SUSPENDING_TIMEOUT);
+		if (res <= 0) {
+			__scst_resume_activity();
+			if (res == 0)
+				res = -EBUSY;
+		} else
+			res = 0;
+	} else
+		wait_event(scst_dev_cmd_waitQ,
+			   atomic_read(&scst_cmd_count) == 0);
+
+	TRACE_MGMT_DBG("wait_event() returned %d", res);
+	return res;
+}
+
+/**
+ * scst_suspend_activity() - globally suspend any activity
+ *
+ * Description:
+ *    Globally suspends any activity and doesn't return, until there are any
+ *    active commands (state after SCST_CMD_STATE_INIT). If "interruptible"
+ *    is true, it returns after SCST_SUSPENDING_TIMEOUT or if it was interrupted
+ *    by a signal with the corresponding error status < 0. If "interruptible"
+ *    is false, it will wait virtually forever. On success returns 0.
+ *
+ *    New arriving commands stay in the suspended state until
+ *    scst_resume_activity() is called.
+ */
+int scst_suspend_activity(bool interruptible)
+{
+	int res = 0;
+	bool rep = false;
+
+	if (interruptible) {
+		if (mutex_lock_interruptible(&scst_suspend_mutex) != 0) {
+			res = -EINTR;
+			goto out;
+		}
+	} else
+		mutex_lock(&scst_suspend_mutex);
+
+	TRACE_MGMT_DBG("suspend_count %d", suspend_count);
+	suspend_count++;
+	if (suspend_count > 1)
+		goto out_up;
+
+	set_bit(SCST_FLAG_SUSPENDING, &scst_flags);
+	set_bit(SCST_FLAG_SUSPENDED, &scst_flags);
+	/*
+	 * Assignment of SCST_FLAG_SUSPENDING and SCST_FLAG_SUSPENDED must be
+	 * ordered with scst_cmd_count. Otherwise lockless logic in
+	 * scst_translate_lun() and scst_mgmt_translate_lun() won't work.
+	 */
+	smp_mb__after_set_bit();
+
+	/*
+	 * See comment in scst_user.c::dev_user_task_mgmt_fn() for more
+	 * information about scst_user behavior.
+	 *
+	 * ToDo: make the global suspending unneeded (switch to per-device
+	 * reference counting? That would mean to switch off from lockless
+	 * implementation of scst_translate_lun().. )
+	 */
+
+	if (atomic_read(&scst_cmd_count) != 0) {
+		PRINT_INFO("Waiting for %d active commands to complete... This "
+			"might take few minutes for disks or few hours for "
+			"tapes, if you use long executed commands, like "
+			"REWIND or FORMAT. In case, if you have a hung user "
+			"space device (i.e. made using scst_user module) not "
+			"responding to any commands, if might take virtually "
+			"forever until the corresponding user space "
+			"program recovers and starts responding or gets "
+			"killed.", atomic_read(&scst_cmd_count));
+		rep = true;
+	}
+
+	res = scst_susp_wait(interruptible);
+	if (res != 0)
+		goto out_clear;
+
+	clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
+	/* See comment about smp_mb() above */
+	smp_mb__after_clear_bit();
+
+	TRACE_MGMT_DBG("Waiting for %d active commands finally to complete",
+		atomic_read(&scst_cmd_count));
+
+	res = scst_susp_wait(interruptible);
+	if (res != 0)
+		goto out_clear;
+
+	if (rep)
+		PRINT_INFO("%s", "All active commands completed");
+
+out_up:
+	mutex_unlock(&scst_suspend_mutex);
+
+out:
+	return res;
+
+out_clear:
+	clear_bit(SCST_FLAG_SUSPENDING, &scst_flags);
+	/* See comment about smp_mb() above */
+	smp_mb__after_clear_bit();
+	goto out_up;
+}
+EXPORT_SYMBOL_GPL(scst_suspend_activity);
+
+static void __scst_resume_activity(void)
+{
+	struct scst_cmd_threads *l;
+
+	suspend_count--;
+	TRACE_MGMT_DBG("suspend_count %d left", suspend_count);
+	if (suspend_count > 0)
+		goto out;
+
+	clear_bit(SCST_FLAG_SUSPENDED, &scst_flags);
+	/*
+	 * The barrier is needed to make sure all woken up threads see the
+	 * cleared flag. Not sure if it's really needed, but let's be safe.
+	 */
+	smp_mb__after_clear_bit();
+
+	list_for_each_entry(l, &scst_cmd_threads_list, lists_list_entry) {
+		wake_up_all(&l->cmd_list_waitQ);
+	}
+	wake_up_all(&scst_init_cmd_list_waitQ);
+
+	spin_lock_irq(&scst_mcmd_lock);
+	if (!list_empty(&scst_delayed_mgmt_cmd_list)) {
+		struct scst_mgmt_cmd *m;
+		m = list_entry(scst_delayed_mgmt_cmd_list.next, typeof(*m),
+				mgmt_cmd_list_entry);
+		TRACE_MGMT_DBG("Moving delayed mgmt cmd %p to head of active "
+			"mgmt cmd list", m);
+		list_move(&m->mgmt_cmd_list_entry, &scst_active_mgmt_cmd_list);
+	}
+	spin_unlock_irq(&scst_mcmd_lock);
+	wake_up_all(&scst_mgmt_cmd_list_waitQ);
+
+out:
+	return;
+}
+
+/**
+ * scst_resume_activity() - globally resume all activities
+ *
+ * Resumes suspended by scst_suspend_activity() activities.
+ */
+void scst_resume_activity(void)
+{
+
+	mutex_lock(&scst_suspend_mutex);
+	__scst_resume_activity();
+	mutex_unlock(&scst_suspend_mutex);
+	return;
+}
+EXPORT_SYMBOL_GPL(scst_resume_activity);
+
+static int scst_register_device(struct scsi_device *scsidp)
+{
+	int res = 0;
+	struct scst_device *dev, *d;
+	struct scst_dev_type *dt;
+
+	res = scst_suspend_activity(true);
+	if (res != 0)
+		goto out_err;
+
+	if (mutex_lock_interruptible(&scst_mutex) != 0) {
+		res = -EINTR;
+		goto out_resume;
+	}
+
+	res = scst_alloc_device(GFP_KERNEL, &dev);
+	if (res != 0)
+		goto out_up;
+
+	dev->type = scsidp->type;
+
+	dev->virt_name = kmalloc(50, GFP_KERNEL);
+	if (dev->virt_name == NULL) {
+		PRINT_ERROR("%s", "Unable to alloc device name");
+		res = -ENOMEM;
+		goto out_free_dev;
+	}
+	snprintf(dev->virt_name, 50, "%d:%d:%d:%d", scsidp->host->host_no,
+		scsidp->channel, scsidp->id, scsidp->lun);
+
+	list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
+		if (strcmp(d->virt_name, dev->virt_name) == 0) {
+			PRINT_ERROR("Device %s already exists", dev->virt_name);
+			res = -EEXIST;
+			goto out_free_dev;
+		}
+	}
+
+	dev->scsi_dev = scsidp;
+
+	list_add_tail(&dev->dev_list_entry, &scst_dev_list);
+
+	res = scst_create_device_sysfs(dev);
+	if (res != 0)
+		goto out_free;
+
+	list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
+		if (dt->type == scsidp->type) {
+			res = scst_assign_dev_handler(dev, dt);
+			if (res != 0)
+				goto out_free;
+			break;
+		}
+	}
+
+out_up:
+	mutex_unlock(&scst_mutex);
+
+out_resume:
+	scst_resume_activity();
+
+out_err:
+	if (res == 0) {
+		PRINT_INFO("Attached to scsi%d, channel %d, id %d, lun %d, "
+			"type %d", scsidp->host->host_no, scsidp->channel,
+			scsidp->id, scsidp->lun, scsidp->type);
+	} else {
+		PRINT_ERROR("Failed to attach to scsi%d, channel %d, id %d, "
+			"lun %d, type %d", scsidp->host->host_no,
+			scsidp->channel, scsidp->id, scsidp->lun, scsidp->type);
+	}
+	return res;
+
+out_free:
+	list_del(&dev->dev_list_entry);
+
+out_free_dev:
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+	scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
+	goto out_err;
+}
+
+static void scst_unregister_device(struct scsi_device *scsidp)
+{
+	struct scst_device *d, *dev = NULL;
+	struct scst_acg_dev *acg_dev, *aa;
+
+	scst_suspend_activity(false);
+	mutex_lock(&scst_mutex);
+
+	list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
+		if (d->scsi_dev == scsidp) {
+			dev = d;
+			TRACE_DBG("Target device %p found", dev);
+			break;
+		}
+	}
+	if (dev == NULL) {
+		PRINT_ERROR("%s", "Target device not found");
+		goto out_resume;
+	}
+
+	list_del(&dev->dev_list_entry);
+
+	list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
+				 dev_acg_dev_list_entry) {
+		scst_acg_remove_dev(acg_dev->acg, dev, true);
+	}
+
+	scst_assign_dev_handler(dev, &scst_null_devtype);
+
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
+
+	PRINT_INFO("Detached from scsi%d, channel %d, id %d, lun %d, type %d",
+		scsidp->host->host_no, scsidp->channel, scsidp->id,
+		scsidp->lun, scsidp->type);
+
+out:
+	return;
+
+out_resume:
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+	goto out;
+}
+
+static int scst_dev_handler_check(struct scst_dev_type *dev_handler)
+{
+	int res = 0;
+
+	if (dev_handler->parse == NULL) {
+		PRINT_ERROR("scst dev handler %s must have "
+			"parse() method.", dev_handler->name);
+		res = -EINVAL;
+		goto out;
+	}
+
+	if (((dev_handler->add_device != NULL) &&
+	     (dev_handler->del_device == NULL)) ||
+	    ((dev_handler->add_device == NULL) &&
+	     (dev_handler->del_device != NULL))) {
+		PRINT_ERROR("Dev handler %s must either define both "
+			"add_device() and del_device(), or none.",
+			dev_handler->name);
+		res = -EINVAL;
+		goto out;
+	}
+
+	if (dev_handler->exec == NULL) {
+#ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
+		dev_handler->exec_atomic = 1;
+#else
+		dev_handler->exec_atomic = 0;
+#endif
+	}
+
+	if (dev_handler->dev_done == NULL)
+		dev_handler->dev_done_atomic = 1;
+
+out:
+	return res;
+}
+
+/**
+ * scst_register_virtual_device() - register a virtual device.
+ * @dev_handler: the device's device handler
+ * @dev_name:	the new device name, NULL-terminated string. Must be uniq
+ *              among all virtual devices in the system.
+ *
+ * Registers a virtual device and returns assinged to the device ID on
+ * success, or negative value otherwise
+ */
+int scst_register_virtual_device(struct scst_dev_type *dev_handler,
+	const char *dev_name)
+{
+	int res, rc;
+	struct scst_device *dev;
+
+	if (dev_handler == NULL) {
+		PRINT_ERROR("%s: valid device handler must be supplied",
+			    __func__);
+		res = -EINVAL;
+		goto out;
+	}
+
+	if (dev_name == NULL) {
+		PRINT_ERROR("%s: device name must be non-NULL", __func__);
+		res = -EINVAL;
+		goto out;
+	}
+
+	res = scst_dev_handler_check(dev_handler);
+	if (res != 0)
+		goto out;
+
+	list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
+		if (strcmp(dev->virt_name, dev_name) == 0) {
+			PRINT_ERROR("Device %s already exists", dev_name);
+			res = -EEXIST;
+			goto out;
+		}
+	}
+
+	res = scst_suspend_activity(true);
+	if (res != 0)
+		goto out;
+
+	if (mutex_lock_interruptible(&scst_mutex) != 0) {
+		res = -EINTR;
+		goto out_resume;
+	}
+
+	res = scst_alloc_device(GFP_KERNEL, &dev);
+	if (res != 0)
+		goto out_up;
+
+	dev->type = dev_handler->type;
+	dev->scsi_dev = NULL;
+	dev->virt_name = kstrdup(dev_name, GFP_KERNEL);
+	if (dev->virt_name == NULL) {
+		PRINT_ERROR("Unable to allocate virt_name for dev %s",
+			dev_name);
+		res = -ENOMEM;
+		goto out_release;
+	}
+	dev->virt_id = scst_virt_dev_last_id++;
+
+	list_add_tail(&dev->dev_list_entry, &scst_dev_list);
+
+	res = dev->virt_id;
+
+	rc = scst_create_device_sysfs(dev);
+	if (rc != 0) {
+		res = rc;
+		goto out_free_del;
+	}
+
+	rc = scst_assign_dev_handler(dev, dev_handler);
+	if (rc != 0) {
+		res = rc;
+		goto out_free_del;
+	}
+
+out_up:
+	mutex_unlock(&scst_mutex);
+
+out_resume:
+	scst_resume_activity();
+
+out:
+	if (res > 0)
+		PRINT_INFO("Attached to virtual device %s (id %d)",
+			dev_name, dev->virt_id);
+	else
+		PRINT_INFO("Failed to attach to virtual device %s", dev_name);
+	return res;
+
+out_free_del:
+	list_del(&dev->dev_list_entry);
+
+out_release:
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+	scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
+	goto out;
+}
+EXPORT_SYMBOL_GPL(scst_register_virtual_device);
+
+/**
+ * scst_unregister_virtual_device() - unegister a virtual device.
+ * @id:		the device's ID, returned by the registration function
+ */
+void scst_unregister_virtual_device(int id)
+{
+	struct scst_device *d, *dev = NULL;
+	struct scst_acg_dev *acg_dev, *aa;
+
+	scst_suspend_activity(false);
+	mutex_lock(&scst_mutex);
+
+	list_for_each_entry(d, &scst_dev_list, dev_list_entry) {
+		if (d->virt_id == id) {
+			dev = d;
+			TRACE_DBG("Target device %p (id %d) found", dev, id);
+			break;
+		}
+	}
+	if (dev == NULL) {
+		PRINT_ERROR("Target virtual device (id %d) not found", id);
+		goto out_unblock;
+	}
+
+	list_del(&dev->dev_list_entry);
+
+	list_for_each_entry_safe(acg_dev, aa, &dev->dev_acg_dev_list,
+				 dev_acg_dev_list_entry) {
+		scst_acg_remove_dev(acg_dev->acg, dev, true);
+	}
+
+	scst_assign_dev_handler(dev, &scst_null_devtype);
+
+	PRINT_INFO("Detached from virtual device %s (id %d)",
+		dev->virt_name, dev->virt_id);
+
+out_unblock:
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	scst_device_sysfs_put(dev); /* must not be called under scst_mutex */
+	return;
+}
+EXPORT_SYMBOL_GPL(scst_unregister_virtual_device);
+
+/**
+ * __scst_register_dev_driver() - register pass-through dev handler driver
+ * @dev_type:	dev handler template
+ * @version:	SCST_INTERFACE_VERSION version string to ensure that
+ *		SCST core and the dev handler use the same version of
+ *		the SCST interface
+ *
+ * Description:
+ *    Registers a pass-through dev handler driver. Returns 0 on success
+ *    or appropriate error code otherwise.
+ *
+ *    Note: *dev_type must be static!
+ */
+int __scst_register_dev_driver(struct scst_dev_type *dev_type,
+	const char *version)
+{
+	struct scst_dev_type *dt;
+	struct scst_device *dev;
+	int res;
+	int exist;
+
+	if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
+		PRINT_ERROR("Incorrect version of dev handler %s",
+			dev_type->name);
+		res = -EINVAL;
+		goto out_error;
+	}
+
+	res = scst_dev_handler_check(dev_type);
+	if (res != 0)
+		goto out_error;
+
+	res = scst_suspend_activity(true);
+	if (res != 0)
+		goto out_error;
+
+	if (mutex_lock_interruptible(&scst_mutex) != 0) {
+		res = -EINTR;
+		goto out_err_res;
+	}
+
+	exist = 0;
+	list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
+		if (strcmp(dt->name, dev_type->name) == 0) {
+			PRINT_ERROR("Device type handler \"%s\" already "
+				"exist", dt->name);
+			exist = 1;
+			break;
+		}
+	}
+	if (exist)
+		goto out_up;
+
+	res = scst_create_devt_sysfs(dev_type);
+	if (res < 0)
+		goto out_free;
+
+	list_add_tail(&dev_type->dev_type_list_entry, &scst_dev_type_list);
+
+	list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
+		if (dev->scsi_dev == NULL || dev->handler != &scst_null_devtype)
+			continue;
+		if (dev->scsi_dev->type == dev_type->type)
+			scst_assign_dev_handler(dev, dev_type);
+	}
+
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	if (res == 0) {
+		PRINT_INFO("Device handler \"%s\" for type %d registered "
+			"successfully", dev_type->name, dev_type->type);
+	}
+
+out:
+	return res;
+
+out_free:
+	scst_devt_sysfs_put(dev_type);
+
+out_up:
+	mutex_unlock(&scst_mutex);
+
+out_err_res:
+	scst_resume_activity();
+
+out_error:
+	PRINT_ERROR("Failed to register device handler \"%s\" for type %d",
+		dev_type->name, dev_type->type);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(__scst_register_dev_driver);
+
+/**
+ * scst_unregister_dev_driver() - unregister pass-through dev handler driver
+ */
+void scst_unregister_dev_driver(struct scst_dev_type *dev_type)
+{
+	struct scst_device *dev;
+	struct scst_dev_type *dt;
+	int found = 0;
+
+	scst_suspend_activity(false);
+	mutex_lock(&scst_mutex);
+
+	list_for_each_entry(dt, &scst_dev_type_list, dev_type_list_entry) {
+		if (strcmp(dt->name, dev_type->name) == 0) {
+			found = 1;
+			break;
+		}
+	}
+	if (!found) {
+		PRINT_ERROR("Dev handler \"%s\" isn't registered",
+			dev_type->name);
+		goto out_up;
+	}
+
+	list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
+		if (dev->handler == dev_type) {
+			scst_assign_dev_handler(dev, &scst_null_devtype);
+			TRACE_DBG("Dev handler removed from device %p", dev);
+		}
+	}
+
+	list_del(&dev_type->dev_type_list_entry);
+
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+
+	scst_devt_sysfs_put(dev_type);
+
+	PRINT_INFO("Device handler \"%s\" for type %d unloaded",
+		   dev_type->name, dev_type->type);
+
+out:
+	return;
+
+out_up:
+	mutex_unlock(&scst_mutex);
+	scst_resume_activity();
+	goto out;
+}
+EXPORT_SYMBOL_GPL(scst_unregister_dev_driver);
+
+/**
+ * __scst_register_virtual_dev_driver() - register virtual dev handler driver
+ * @dev_type:	dev handler template
+ * @version:	SCST_INTERFACE_VERSION version string to ensure that
+ *		SCST core and the dev handler use the same version of
+ *		the SCST interface
+ *
+ * Description:
+ *    Registers a virtual dev handler driver. Returns 0 on success or
+ *    appropriate error code otherwise.
+ *
+ *    Note: *dev_type must be static!
+ */
+int __scst_register_virtual_dev_driver(struct scst_dev_type *dev_type,
+	const char *version)
+{
+	int res;
+
+	if (strcmp(version, SCST_INTERFACE_VERSION) != 0) {
+		PRINT_ERROR("Incorrect version of virtual dev handler %s",
+			dev_type->name);
+		res = -EINVAL;
+		goto out_err;
+	}
+
+	res = scst_dev_handler_check(dev_type);
+	if (res != 0)
+		goto out_err;
+
+	res = scst_create_devt_sysfs(dev_type);
+	if (res < 0)
+		goto out_free;
+
+	if (dev_type->type != -1) {
+		PRINT_INFO("Virtual device handler %s for type %d "
+			"registered successfully", dev_type->name,
+			dev_type->type);
+	} else {
+		PRINT_INFO("Virtual device handler \"%s\" registered "
+			"successfully", dev_type->name);
+	}
+
+out:
+	return res;
+
+out_free:
+
+	scst_devt_sysfs_put(dev_type);
+
+out_err:
+	PRINT_ERROR("Failed to register virtual device handler \"%s\"",
+		dev_type->name);
+	goto out;
+}
+EXPORT_SYMBOL_GPL(__scst_register_virtual_dev_driver);
+
+/**
+ * scst_unregister_virtual_dev_driver() - unregister virtual dev driver
+ */
+void scst_unregister_virtual_dev_driver(struct scst_dev_type *dev_type)
+{
+
+	scst_devt_sysfs_put(dev_type);
+
+	PRINT_INFO("Device handler \"%s\" unloaded", dev_type->name);
+	return;
+}
+EXPORT_SYMBOL_GPL(scst_unregister_virtual_dev_driver);
+
+/* scst_mutex supposed to be held */
+int scst_add_threads(struct scst_cmd_threads *cmd_threads,
+	struct scst_device *dev, struct scst_tgt_dev *tgt_dev, int num)
+{
+	int res, i;
+	struct scst_cmd_thread_t *thr;
+	int n = 0, tgt_dev_num = 0;
+
+	list_for_each_entry(thr, &cmd_threads->threads_list, thread_list_entry) {
+		n++;
+	}
+
+	if (tgt_dev != NULL) {
+		struct scst_tgt_dev *t;
+		list_for_each_entry(t, &tgt_dev->dev->dev_tgt_dev_list,
+				dev_tgt_dev_list_entry) {
+			if (t == tgt_dev)
+				break;
+			tgt_dev_num++;
+		}
+	}
+
+	for (i = 0; i < num; i++) {
+		struct scst_cmd_thread_t *thr;
+
+		thr = kmalloc(sizeof(*thr), GFP_KERNEL);
+		if (!thr) {
+			res = -ENOMEM;
+			PRINT_ERROR("fail to allocate thr %d", res);
+			goto out_error;
+		}
+
+		if (dev != NULL) {
+			char nm[14]; /* to limit the name's len */
+			strlcpy(nm, dev->virt_name, ARRAY_SIZE(nm));
+			thr->cmd_thread = kthread_create(scst_cmd_thread,
+				cmd_threads, "%s%d", nm, n++);
+		} else if (tgt_dev != NULL) {
+			char nm[11]; /* to limit the name's len */
+			strlcpy(nm, tgt_dev->dev->virt_name, ARRAY_SIZE(nm));
+			thr->cmd_thread = kthread_create(scst_cmd_thread,
+				cmd_threads, "%s%d_%d", nm, tgt_dev_num, n++);
+		} else
+			thr->cmd_thread = kthread_create(scst_cmd_thread,
+				cmd_threads, "scsi_tgt%d", n++);
+
+		if (IS_ERR(thr->cmd_thread)) {
+			res = PTR_ERR(thr->cmd_thread);
+			PRINT_ERROR("kthread_create() failed: %d", res);
+			kfree(thr);
+			goto out_error;
+		}
+
+		list_add(&thr->thread_list_entry, &cmd_threads->threads_list);
+		cmd_threads->nr_threads++;
+
+		wake_up_process(thr->cmd_thread);
+	}
+
+	res = 0;
+
+out:
+	return res;
+
+out_error:
+	scst_del_threads(cmd_threads, i);
+	goto out;
+}
+
+/* scst_mutex supposed to be held */
+void scst_del_threads(struct scst_cmd_threads *cmd_threads, int num)
+{
+	struct scst_cmd_thread_t *ct, *tmp;
+
+	if (num == 0)
+		goto out;
+
+	list_for_each_entry_safe_reverse(ct, tmp, &cmd_threads->threads_list,
+				thread_list_entry) {
+		int rc;
+		struct scst_device *dev;
+
+		rc = kthread_stop(ct->cmd_thread);
+		if (rc < 0)
+			TRACE_MGMT_DBG("kthread_stop() failed: %d", rc);
+
+		list_del(&ct->thread_list_entry);
+
+		list_for_each_entry(dev, &scst_dev_list, dev_list_entry) {
+			struct scst_tgt_dev *tgt_dev;
+			list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
+					dev_tgt_dev_list_entry) {
+				scst_del_thr_data(tgt_dev, ct->cmd_thread);
+			}
+		}
+
+		kfree(ct);
+
+		cmd_threads->nr_threads--;
+
+		--num;
+		if (num == 0)
+			break;
+	}
+
+	EXTRACHECKS_BUG_ON((cmd_threads->nr_threads == 0) &&
+		(cmd_threads->io_context != NULL));
+
+out:
+	return;
+}
+
+/* The activity supposed to be suspended and scst_mutex held */
+void scst_stop_dev_threads(struct scst_device *dev)
+{
+	struct scst_tgt_dev *tgt_dev;
+
+	list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
+				dev_tgt_dev_list_entry) {
+		scst_tgt_dev_stop_threads(tgt_dev);
+	}
+
+	if ((dev->threads_num > 0) &&
+	    (dev->threads_pool_type == SCST_THREADS_POOL_SHARED))
+		scst_del_threads(&dev->dev_cmd_threads, -1);
+	return;
+}
+
+/* The activity supposed to be suspended and scst_mutex held */
+int scst_create_dev_threads(struct scst_device *dev)
+{
+	int res = 0;
+	struct scst_tgt_dev *tgt_dev;
+
+	list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
+			dev_tgt_dev_list_entry) {
+		res = scst_tgt_dev_setup_threads(tgt_dev);
+		if (res != 0)
+			goto out_err;
+	}
+
+	if ((dev->threads_num > 0) &&
+	    (dev->threads_pool_type == SCST_THREADS_POOL_SHARED)) {
+		res = scst_add_threads(&dev->dev_cmd_threads, dev, NULL,
+			dev->threads_num);
+		if (res != 0)
+			goto out_err;
+	}
+
+out:
+	return res;
+
+out_err:
+	scst_stop_dev_threads(dev);
+	goto out;
+}
+
+/* The activity supposed to be suspended and scst_mutex held */
+int scst_assign_dev_handler(struct scst_device *dev,
+	struct scst_dev_type *handler)
+{
+	int res = 0;
+	struct scst_tgt_dev *tgt_dev;
+	LIST_HEAD(attached_tgt_devs);
+
+	BUG_ON(handler == NULL);
+
+	if (dev->handler == handler)
+		goto out;
+
+	if (dev->handler == NULL)
+		goto assign;
+
+	if (dev->handler->detach_tgt) {
+		list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
+				dev_tgt_dev_list_entry) {
+			TRACE_DBG("Calling dev handler's detach_tgt(%p)",
+				tgt_dev);
+			dev->handler->detach_tgt(tgt_dev);
+			TRACE_DBG("%s", "Dev handler's detach_tgt() returned");
+		}
+	}
+
+	if (dev->handler->detach) {
+		TRACE_DBG("%s", "Calling dev handler's detach()");
+		dev->handler->detach(dev);
+		TRACE_DBG("%s", "Old handler's detach() returned");
+	}
+
+	scst_stop_dev_threads(dev);
+
+	scst_devt_dev_sysfs_put(dev);
+
+assign:
+	dev->handler = handler;
+
+	if (handler == NULL)
+		goto out;
+
+	dev->threads_num = handler->threads_num;
+	dev->threads_pool_type = handler->threads_pool_type;
+
+	res = scst_create_devt_dev_sysfs(dev);
+	if (res != 0)
+		goto out_null;
+
+	if (handler->attach) {
+		TRACE_DBG("Calling new dev handler's attach(%p)", dev);
+		res = handler->attach(dev);
+		TRACE_DBG("New dev handler's attach() returned %d", res);
+		if (res != 0) {
+			PRINT_ERROR("New device handler's %s attach() "
+				"failed: %d", handler->name, res);
+			goto out_remove_sysfs;
+		}
+	}
+
+	if (handler->attach_tgt) {
+		list_for_each_entry(tgt_dev, &dev->dev_tgt_dev_list,
+				dev_tgt_dev_list_entry) {
+			TRACE_DBG("Calling dev handler's attach_tgt(%p)",
+				tgt_dev);
+			res = handler->attach_tgt(tgt_dev);
+			TRACE_DBG("%s", "Dev handler's attach_tgt() returned");
+			if (res != 0) {
+				PRINT_ERROR("Device handler's %s attach_tgt() "
+				    "failed: %d", handler->name, res);
+				goto out_err_detach_tgt;
+			}
+			list_add_tail(&tgt_dev->extra_tgt_dev_list_entry,
+				&attached_tgt_devs);
+		}
+	}
+
+	res = scst_create_dev_threads(dev);
+	if (res != 0)
+		goto out_err_detach_tgt;
+
+out:
+	return res;
+
+out_err_detach_tgt:
+	if (handler && handler->detach_tgt) {
+		list_for_each_entry(tgt_dev, &attached_tgt_devs,
+				 extra_tgt_dev_list_entry) {
+			TRACE_DBG("Calling handler's detach_tgt(%p)",
+				tgt_dev);
+			handler->detach_tgt(tgt_dev);
+			TRACE_DBG("%s", "Handler's detach_tgt() returned");
+		}
+	}
+	if (handler && handler->detach) {
+		TRACE_DBG("%s", "Calling handler's detach()");
+		handler->detach(dev);
+		TRACE_DBG("%s", "Handler's detach() returned");
+	}
+
+out_remove_sysfs:
+	scst_devt_dev_sysfs_put(dev);
+
+out_null:
+	dev->handler = &scst_null_devtype;
+	goto out;
+}
+
+/**
+ * scst_init_threads() - initialize SCST processing threads pool
+ *
+ * Initializes scst_cmd_threads structure
+ */
+void scst_init_threads(struct scst_cmd_threads *cmd_threads)
+{
+
+	spin_lock_init(&cmd_threads->cmd_list_lock);
+	INIT_LIST_HEAD(&cmd_threads->active_cmd_list);
+	init_waitqueue_head(&cmd_threads->cmd_list_waitQ);
+	INIT_LIST_HEAD(&cmd_threads->threads_list);
+
+	mutex_lock(&scst_suspend_mutex);
+	list_add_tail(&cmd_threads->lists_list_entry,
+		&scst_cmd_threads_list);
+	mutex_unlock(&scst_suspend_mutex);
+	return;
+}
+EXPORT_SYMBOL_GPL(scst_init_threads);
+
+/**
+ * scst_deinit_threads() - deinitialize SCST processing threads pool
+ *
+ * Deinitializes scst_cmd_threads structure
+ */
+void scst_deinit_threads(struct scst_cmd_threads *cmd_threads)
+{
+
+	mutex_lock(&scst_suspend_mutex);
+	list_del(&cmd_threads->lists_list_entry);
+	mutex_unlock(&scst_suspend_mutex);
+
+	BUG_ON(cmd_threads->io_context);
+	return;
+}
+EXPORT_SYMBOL_GPL(scst_deinit_threads);
+
+static void scst_stop_all_threads(void)
+{
+
+	mutex_lock(&scst_mutex);
+
+	scst_del_threads(&scst_main_cmd_threads, -1);
+
+	if (scst_mgmt_cmd_thread)
+		kthread_stop(scst_mgmt_cmd_thread);
+	if (scst_mgmt_thread)
+		kthread_stop(scst_mgmt_thread);
+	if (scst_init_cmd_thread)
+		kthread_stop(scst_init_cmd_thread);
+
+	mutex_unlock(&scst_mutex);
+	return;
+}
+
+static int scst_start_all_threads(int num)
+{
+	int res;
+
+	mutex_lock(&scst_mutex);
+
+	res = scst_add_threads(&scst_main_cmd_threads, NULL, NULL, num);
+	if (res < 0)
+		goto out_unlock;
+
+	scst_init_cmd_thread = kthread_run(scst_init_thread,
+		NULL, "scsi_tgt_init");
+	if (IS_ERR(scst_init_cmd_thread)) {
+		res = PTR_ERR(scst_init_cmd_thread);
+		PRINT_ERROR("kthread_create() for init cmd failed: %d", res);
+		scst_init_cmd_thread = NULL;
+		goto out_unlock;
+	}
+
+	scst_mgmt_cmd_thread = kthread_run(scst_tm_thread,
+		NULL, "scsi_tm");
+	if (IS_ERR(scst_mgmt_cmd_thread)) {
+		res = PTR_ERR(scst_mgmt_cmd_thread);
+		PRINT_ERROR("kthread_create() for TM failed: %d", res);
+		scst_mgmt_cmd_thread = NULL;
+		goto out_unlock;
+	}
+
+	scst_mgmt_thread = kthread_run(scst_global_mgmt_thread,
+		NULL, "scsi_tgt_mgmt");
+	if (IS_ERR(scst_mgmt_thread)) {
+		res = PTR_ERR(scst_mgmt_thread);
+		PRINT_ERROR("kthread_create() for mgmt failed: %d", res);
+		scst_mgmt_thread = NULL;
+		goto out_unlock;
+	}
+
+out_unlock:
+	mutex_unlock(&scst_mutex);
+	return res;
+}
+
+/**
+ * scst_get() - increase global SCST ref counter
+ *
+ * Increases global SCST ref counter that prevents from entering into suspended
+ * activities stage, so protects from any global management operations.
+ */
+void scst_get(void)
+{
+	__scst_get(0);
+}
+EXPORT_SYMBOL_GPL(scst_get);
+
+/**
+ * scst_put() - decrease global SCST ref counter
+ *
+ * Decreses global SCST ref counter that prevents from entering into suspended
+ * activities stage, so protects from any global management operations. On
+ * zero, if suspending activities is waiting, they will be suspended.
+ */
+void scst_put(void)
+{
+	__scst_put();
+}
+EXPORT_SYMBOL_GPL(scst_put);
+
+/**
+ * scst_get_setup_id() - return SCST setup ID
+ *
+ * Returns SCST setup ID. This ID can be used for multiple
+ * setups with the same configuration.
+ */
+unsigned int scst_get_setup_id(void)
+{
+	return scst_setup_id;
+}
+EXPORT_SYMBOL_GPL(scst_get_setup_id);
+
+static int scst_add(struct device *cdev, struct class_interface *intf)
+{
+	struct scsi_device *scsidp;
+	int res = 0;
+
+	scsidp = to_scsi_device(cdev->parent);
+
+	if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
+		res = scst_register_device(scsidp);
+	return res;
+}
+
+static void scst_remove(struct device *cdev, struct class_interface *intf)
+{
+	struct scsi_device *scsidp;
+
+	scsidp = to_scsi_device(cdev->parent);
+
+	if (strcmp(scsidp->host->hostt->name, SCST_LOCAL_NAME) != 0)
+		scst_unregister_device(scsidp);
+	return;
+}
+
+static struct class_interface scst_interface = {
+	.add_dev = scst_add,
+	.remove_dev = scst_remove,
+};
+
+static void __init scst_print_config(void)
+{
+	char buf[128];
+	int i, j;
+
+	i = snprintf(buf, sizeof(buf), "Enabled features: ");
+	j = i;
+
+#ifdef CONFIG_SCST_STRICT_SERIALIZING
+	i += snprintf(&buf[i], sizeof(buf) - i, "STRICT_SERIALIZING");
+#endif
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sEXTRACHECKS",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_TRACING
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sTRACING",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_DEBUG
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_DEBUG_TM
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_TM",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_DEBUG_RETRY
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_RETRY",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_DEBUG_OOM
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_OOM",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_DEBUG_SN
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sDEBUG_SN",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_USE_EXPECTED_VALUES
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sUSE_EXPECTED_VALUES",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
+	i += snprintf(&buf[i], sizeof(buf) - i,
+		"%sALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ",
+		(j == i) ? "" : ", ");
+#endif
+
+#ifdef CONFIG_SCST_STRICT_SECURITY
+	i += snprintf(&buf[i], sizeof(buf) - i, "%sSCST_STRICT_SECURITY",
+		(j == i) ? "" : ", ");
+#endif
+
+	if (j != i)
+		PRINT_INFO("%s", buf);
+}
+
+static int __init init_scst(void)
+{
+	int res, i;
+	int scst_num_cpus;
+
+	{
+		struct scsi_sense_hdr *shdr;
+		BUILD_BUG_ON(SCST_SENSE_BUFFERSIZE < sizeof(*shdr));
+	}
+	{
+		struct scst_tgt_dev *t;
+		struct scst_cmd *c;
+		BUILD_BUG_ON(sizeof(t->curr_sn) != sizeof(t->expected_sn));
+		BUILD_BUG_ON(sizeof(c->sn) != sizeof(t->expected_sn));
+	}
+
+	mutex_init(&scst_mutex);
+	INIT_LIST_HEAD(&scst_template_list);
+	INIT_LIST_HEAD(&scst_dev_list);
+	INIT_LIST_HEAD(&scst_dev_type_list);
+	spin_lock_init(&scst_main_lock);
+	spin_lock_init(&scst_init_lock);
+	init_waitqueue_head(&scst_init_cmd_list_waitQ);
+	INIT_LIST_HEAD(&scst_init_cmd_list);
+#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
+	scst_trace_flag = SCST_DEFAULT_LOG_FLAGS;
+#endif
+	atomic_set(&scst_cmd_count, 0);
+	spin_lock_init(&scst_mcmd_lock);
+	INIT_LIST_HEAD(&scst_active_mgmt_cmd_list);
+	INIT_LIST_HEAD(&scst_delayed_mgmt_cmd_list);
+	init_waitqueue_head(&scst_mgmt_cmd_list_waitQ);
+	init_waitqueue_head(&scst_mgmt_waitQ);
+	spin_lock_init(&scst_mgmt_lock);
+	INIT_LIST_HEAD(&scst_sess_init_list);
+	INIT_LIST_HEAD(&scst_sess_shut_list);
+	init_waitqueue_head(&scst_dev_cmd_waitQ);
+	mutex_init(&scst_suspend_mutex);
+	INIT_LIST_HEAD(&scst_cmd_threads_list);
+	scst_virt_dev_last_id = 1;
+
+	scst_init_threads(&scst_main_cmd_threads);
+
+	res = scst_lib_init();
+	if (res != 0)
+		goto out;
+
+	scst_num_cpus = num_online_cpus();
+
+	/* ToDo: register_cpu_notifier() */
+
+	if (scst_threads == 0)
+		scst_threads = scst_num_cpus;
+
+	if (scst_threads < 1) {
+		PRINT_ERROR("%s", "scst_threads can not be less than 1");
+		scst_threads = scst_num_cpus;
+	}
+
+#define INIT_CACHEP(p, s, o) do {					\
+		p = KMEM_CACHE(s, SCST_SLAB_FLAGS);			\
+		TRACE_MEM("Slab create: %s at %p size %zd", #s, p,	\
+			  sizeof(struct s));				\
+		if (p == NULL) {					\
+			res = -ENOMEM;					\
+			goto o;						\
+		}							\
+	} while (0)
+
+	INIT_CACHEP(scst_mgmt_cachep, scst_mgmt_cmd, out_lib_exit);
+	INIT_CACHEP(scst_mgmt_stub_cachep, scst_mgmt_cmd_stub,
+			out_destroy_mgmt_cache);
+	INIT_CACHEP(scst_ua_cachep, scst_tgt_dev_UA,
+			out_destroy_mgmt_stub_cache);
+	{
+		struct scst_sense { uint8_t s[SCST_SENSE_BUFFERSIZE]; };
+		INIT_CACHEP(scst_sense_cachep, scst_sense,
+			    out_destroy_ua_cache);
+	}
+	INIT_CACHEP(scst_aen_cachep, scst_aen, out_destroy_sense_cache);
+	INIT_CACHEP(scst_cmd_cachep, scst_cmd, out_destroy_aen_cache);
+	INIT_CACHEP(scst_sess_cachep, scst_session, out_destroy_cmd_cache);
+	INIT_CACHEP(scst_tgtd_cachep, scst_tgt_dev, out_destroy_sess_cache);
+	INIT_CACHEP(scst_acgd_cachep, scst_acg_dev, out_destroy_tgt_cache);
+
+	scst_mgmt_mempool = mempool_create(64, mempool_alloc_slab,
+		mempool_free_slab, scst_mgmt_cachep);
+	if (scst_mgmt_mempool == NULL) {
+		res = -ENOMEM;
+		goto out_destroy_acg_cache;
+	}
+
+	/*
+	 * All mgmt stubs, UAs and sense buffers are bursty and loosing them
+	 * may have fatal consequences, so let's have big pools for them.
+	 */
+
+	scst_mgmt_stub_mempool = mempool_create(1024, mempool_alloc_slab,
+		mempool_free_slab, scst_mgmt_stub_cachep);
+	if (scst_mgmt_stub_mempool == NULL) {
+		res = -ENOMEM;
+		goto out_destroy_mgmt_mempool;
+	}
+
+	scst_ua_mempool = mempool_create(512, mempool_alloc_slab,
+		mempool_free_slab, scst_ua_cachep);
+	if (scst_ua_mempool == NULL) {
+		res = -ENOMEM;
+		goto out_destroy_mgmt_stub_mempool;
+	}
+
+	scst_sense_mempool = mempool_create(1024, mempool_alloc_slab,
+		mempool_free_slab, scst_sense_cachep);
+	if (scst_sense_mempool == NULL) {
+		res = -ENOMEM;
+		goto out_destroy_ua_mempool;
+	}
+
+	scst_aen_mempool = mempool_create(100, mempool_alloc_slab,
+		mempool_free_slab, scst_aen_cachep);
+	if (scst_aen_mempool == NULL) {
+		res = -ENOMEM;
+		goto out_destroy_sense_mempool;
+	}
+
+	res = scst_sysfs_init();
+	if (res != 0)
+		goto out_destroy_aen_mempool;
+
+	if (scst_max_cmd_mem == 0) {
+		struct sysinfo si;
+		si_meminfo(&si);
+#if BITS_PER_LONG == 32
+		scst_max_cmd_mem = min(
+			(((uint64_t)(si.totalram - si.totalhigh) << PAGE_SHIFT)
+				>> 20) >> 2, (uint64_t)1 << 30);
+#else
+		scst_max_cmd_mem = (((si.totalram - si.totalhigh) << PAGE_SHIFT)
+					>> 20) >> 2;
+#endif
+	}
+
+	if (scst_max_dev_cmd_mem != 0) {
+		if (scst_max_dev_cmd_mem > scst_max_cmd_mem) {
+			PRINT_ERROR("scst_max_dev_cmd_mem (%d) > "
+				"scst_max_cmd_mem (%d)",
+				scst_max_dev_cmd_mem,
+				scst_max_cmd_mem);
+			scst_max_dev_cmd_mem = scst_max_cmd_mem;
+		}
+	} else
+		scst_max_dev_cmd_mem = scst_max_cmd_mem * 2 / 5;
+
+	res = scst_sgv_pools_init(
+		((uint64_t)scst_max_cmd_mem << 10) >> (PAGE_SHIFT - 10), 0);
+	if (res != 0)
+		goto out_sysfs_cleanup;
+
+	res = scsi_register_interface(&scst_interface);
+	if (res != 0)
+		goto out_destroy_sgv_pool;
+
+	for (i = 0; i < (int)ARRAY_SIZE(scst_tasklets); i++) {
+		spin_lock_init(&scst_tasklets[i].tasklet_lock);
+		INIT_LIST_HEAD(&scst_tasklets[i].tasklet_cmd_list);
+		tasklet_init(&scst_tasklets[i].tasklet,
+			     (void *)scst_cmd_tasklet,
+			     (unsigned long)&scst_tasklets[i]);
+	}
+
+	TRACE_DBG("%d CPUs found, starting %d threads", scst_num_cpus,
+		scst_threads);
+
+	res = scst_start_all_threads(scst_threads);
+	if (res < 0)
+		goto out_thread_free;
+
+	PRINT_INFO("SCST version %s loaded successfully (max mem for "
+		"commands %dMB, per device %dMB)", SCST_VERSION_STRING,
+		scst_max_cmd_mem, scst_max_dev_cmd_mem);
+
+	scst_print_config();
+
+out:
+	return res;
+
+out_thread_free:
+	scst_stop_all_threads();
+
+	scsi_unregister_interface(&scst_interface);
+
+out_destroy_sgv_pool:
+	scst_sgv_pools_deinit();
+
+out_sysfs_cleanup:
+	scst_sysfs_cleanup();
+
+out_destroy_aen_mempool:
+	mempool_destroy(scst_aen_mempool);
+
+out_destroy_sense_mempool:
+	mempool_destroy(scst_sense_mempool);
+
+out_destroy_ua_mempool:
+	mempool_destroy(scst_ua_mempool);
+
+out_destroy_mgmt_stub_mempool:
+	mempool_destroy(scst_mgmt_stub_mempool);
+
+out_destroy_mgmt_mempool:
+	mempool_destroy(scst_mgmt_mempool);
+
+out_destroy_acg_cache:
+	kmem_cache_destroy(scst_acgd_cachep);
+
+out_destroy_tgt_cache:
+	kmem_cache_destroy(scst_tgtd_cachep);
+
+out_destroy_sess_cache:
+	kmem_cache_destroy(scst_sess_cachep);
+
+out_destroy_cmd_cache:
+	kmem_cache_destroy(scst_cmd_cachep);
+
+out_destroy_aen_cache:
+	kmem_cache_destroy(scst_aen_cachep);
+
+out_destroy_sense_cache:
+	kmem_cache_destroy(scst_sense_cachep);
+
+out_destroy_ua_cache:
+	kmem_cache_destroy(scst_ua_cachep);
+
+out_destroy_mgmt_stub_cache:
+	kmem_cache_destroy(scst_mgmt_stub_cachep);
+
+out_destroy_mgmt_cache:
+	kmem_cache_destroy(scst_mgmt_cachep);
+
+out_lib_exit:
+	scst_lib_exit();
+	goto out;
+}
+
+static void __exit exit_scst(void)
+{
+
+	/* ToDo: unregister_cpu_notifier() */
+
+	scst_sysfs_cleanup();
+
+	scst_stop_all_threads();
+
+	scst_deinit_threads(&scst_main_cmd_threads);
+
+	scsi_unregister_interface(&scst_interface);
+
+	scst_sgv_pools_deinit();
+
+#define DEINIT_CACHEP(p) do {		\
+		kmem_cache_destroy(p);	\
+		p = NULL;		\
+	} while (0)
+
+	mempool_destroy(scst_mgmt_mempool);
+	mempool_destroy(scst_mgmt_stub_mempool);
+	mempool_destroy(scst_ua_mempool);
+	mempool_destroy(scst_sense_mempool);
+	mempool_destroy(scst_aen_mempool);
+
+	DEINIT_CACHEP(scst_mgmt_cachep);
+	DEINIT_CACHEP(scst_mgmt_stub_cachep);
+	DEINIT_CACHEP(scst_ua_cachep);
+	DEINIT_CACHEP(scst_sense_cachep);
+	DEINIT_CACHEP(scst_aen_cachep);
+	DEINIT_CACHEP(scst_cmd_cachep);
+	DEINIT_CACHEP(scst_sess_cachep);
+	DEINIT_CACHEP(scst_tgtd_cachep);
+	DEINIT_CACHEP(scst_acgd_cachep);
+
+	scst_lib_exit();
+
+	PRINT_INFO("%s", "SCST unloaded");
+	return;
+}
+
+module_init(init_scst);
+module_exit(exit_scst);
+
+MODULE_AUTHOR("Vladislav Bolkhovitin");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SCSI target core");
+MODULE_VERSION(SCST_VERSION_STRING);

diff -uprN orig/linux-2.6.33/drivers/scst/scst_targ.c linux-2.6.33/drivers/scst/scst_targ.c
@@ -0,0 +1,5712 @@
+/*
+ *  scst_targ.c
+ *
+ *  Copyright (C) 2004 - 2010 Vladislav Bolkhovitin <[email protected]>
+ *  Copyright (C) 2004 - 2005 Leonid Stoljar
+ *  Copyright (C) 2007 - 2010 ID7 Ltd.
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation, version 2
+ *  of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/smp_lock.h>
+#include <linux/unistd.h>
+#include <linux/string.h>
+#include <linux/kthread.h>
+#include <linux/delay.h>
+#include <linux/ktime.h>
+
+#include "scst.h"
+#include "scst_priv.h"
+
+#if 0 /* Temporary left for future performance investigations */
+/* Deleting it don't forget to delete write_cmd_count */
+#define CONFIG_SCST_ORDERED_READS
+#endif
+
+#if 0 /* Let's disable it for now to see if users will complain about it */
+/* Deleting it don't forget to delete write_cmd_count */
+#define CONFIG_SCST_PER_DEVICE_CMD_COUNT_LIMIT
+#endif
+
+static void scst_cmd_set_sn(struct scst_cmd *cmd);
+static int __scst_init_cmd(struct scst_cmd *cmd);
+static void scst_finish_cmd_mgmt(struct scst_cmd *cmd);
+static struct scst_cmd *__scst_find_cmd_by_tag(struct scst_session *sess,
+	uint64_t tag, bool to_abort);
+static void scst_process_redirect_cmd(struct scst_cmd *cmd,
+	enum scst_exec_context context, int check_retries);
+
+static inline void scst_schedule_tasklet(struct scst_cmd *cmd)
+{
+	struct scst_tasklet *t = &scst_tasklets[smp_processor_id()];
+	unsigned long flags;
+
+	spin_lock_irqsave(&t->tasklet_lock, flags);
+	TRACE_DBG("Adding cmd %p to tasklet %d cmd list", cmd,
+		smp_processor_id());
+	list_add_tail(&cmd->cmd_list_entry, &t->tasklet_cmd_list);
+	spin_unlock_irqrestore(&t->tasklet_lock, flags);
+
+	tasklet_schedule(&t->tasklet);
+}
+
+/**
+ * scst_rx_cmd() - create new command
+ * @sess:	SCST session
+ * @lun:	LUN for the command
+ * @lun_len:	length of the LUN in bytes
+ * @cdb:	CDB of the command
+ * @cdb_len:	length of the CDB in bytes
+ * @atomic:	true, if current context is atomic
+ *
+ * Description:
+ *    Creates new SCST command. Returns new command on success or
+ *    NULL otherwise.
+ *
+ *    Must not be called in parallel with scst_unregister_session() for the
+ *    same session.
+ */
+struct scst_cmd *scst_rx_cmd(struct scst_session *sess,
+			     const uint8_t *lun, int lun_len,
+			     const uint8_t *cdb, int cdb_len, int atomic)
+{
+	struct scst_cmd *cmd;
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if (unlikely(sess->shut_phase != SCST_SESS_SPH_READY)) {
+		PRINT_CRIT_ERROR("%s",
+			"New cmd while shutting down the session");
+		BUG();
+	}
+#endif
+
+	cmd = scst_alloc_cmd(atomic ? GFP_ATOMIC : GFP_KERNEL);
+	if (cmd == NULL)
+		goto out;
+
+	cmd->sess = sess;
+	cmd->tgt = sess->tgt;
+	cmd->tgtt = sess->tgt->tgtt;
+
+	cmd->lun = scst_unpack_lun(lun, lun_len);
+	if (unlikely(cmd->lun == NO_SUCH_LUN)) {
+		PRINT_ERROR("Wrong LUN %d, finishing cmd", -1);
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_lun_not_supported));
+	}
+
+	/*
+	 * For cdb_len 0 defer the error reporting until scst_cmd_init_done(),
+	 * scst_set_cmd_error() supports nested calls.
+	 */
+	if (unlikely(cdb_len > SCST_MAX_CDB_SIZE)) {
+		PRINT_ERROR("Too big CDB len %d, finishing cmd", cdb_len);
+		cdb_len = SCST_MAX_CDB_SIZE;
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_message));
+	}
+
+	memcpy(cmd->cdb, cdb, cdb_len);
+	cmd->cdb_len = cdb_len;
+
+	TRACE_DBG("cmd %p, sess %p", cmd, sess);
+	scst_sess_get(sess);
+
+out:
+	return cmd;
+}
+EXPORT_SYMBOL(scst_rx_cmd);
+
+/*
+ * No locks, but might be on IRQ. Returns 0 on success, <0 if processing of
+ * this command should be stopped.
+ */
+static int scst_init_cmd(struct scst_cmd *cmd, enum scst_exec_context *context)
+{
+	int rc, res = 0;
+
+	/* See the comment in scst_do_job_init() */
+	if (unlikely(!list_empty(&scst_init_cmd_list))) {
+		TRACE_MGMT_DBG("%s", "init cmd list busy");
+		goto out_redirect;
+	}
+	/*
+	 * Memory barrier isn't necessary here, because CPU appears to
+	 * be self-consistent and we don't care about the race, described
+	 * in comment in scst_do_job_init().
+	 */
+
+	rc = __scst_init_cmd(cmd);
+	if (unlikely(rc > 0))
+		goto out_redirect;
+	else if (unlikely(rc != 0)) {
+		res = 1;
+		goto out;
+	}
+
+	EXTRACHECKS_BUG_ON(*context == SCST_CONTEXT_SAME);
+
+	/* Small context optimization */
+	if (((*context == SCST_CONTEXT_TASKLET) ||
+	     (*context == SCST_CONTEXT_DIRECT_ATOMIC)) &&
+	      scst_cmd_is_expected_set(cmd)) {
+		if (cmd->expected_data_direction & SCST_DATA_WRITE) {
+			if (!test_bit(SCST_TGT_DEV_AFTER_INIT_WR_ATOMIC,
+					&cmd->tgt_dev->tgt_dev_flags))
+				*context = SCST_CONTEXT_THREAD;
+		} else {
+			if (!test_bit(SCST_TGT_DEV_AFTER_INIT_OTH_ATOMIC,
+					&cmd->tgt_dev->tgt_dev_flags))
+				*context = SCST_CONTEXT_THREAD;
+		}
+	}
+
+out:
+	return res;
+
+out_redirect:
+	if (cmd->preprocessing_only) {
+		/*
+		 * Poor man solution for single threaded targets, where
+		 * blocking receiver at least sometimes means blocking all.
+		 * For instance, iSCSI target won't be able to receive
+		 * Data-Out PDUs.
+		 */
+		BUG_ON(*context != SCST_CONTEXT_DIRECT);
+		scst_set_busy(cmd);
+		scst_set_cmd_abnormal_done_state(cmd);
+		res = 1;
+		/* Keep initiator away from too many BUSY commands */
+		msleep(50);
+	} else {
+		unsigned long flags;
+		spin_lock_irqsave(&scst_init_lock, flags);
+		TRACE_MGMT_DBG("Adding cmd %p to init cmd list (scst_cmd_count "
+			"%d)", cmd, atomic_read(&scst_cmd_count));
+		list_add_tail(&cmd->cmd_list_entry, &scst_init_cmd_list);
+		if (test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))
+			scst_init_poll_cnt++;
+		spin_unlock_irqrestore(&scst_init_lock, flags);
+		wake_up(&scst_init_cmd_list_waitQ);
+		res = -1;
+	}
+	goto out;
+}
+
+/**
+ * scst_cmd_init_done() - the command's initialization done
+ * @cmd:	SCST command
+ * @pref_context: preferred command execution context
+ *
+ * Description:
+ *    Notifies SCST that the driver finished its part of the command
+ *    initialization, and the command is ready for execution.
+ *    The second argument sets preferred command execition context.
+ *    See SCST_CONTEXT_* constants for details.
+ *
+ *    !!IMPORTANT!!
+ *
+ *    If cmd->set_sn_on_restart_cmd not set, this function, as well as
+ *    scst_cmd_init_stage1_done() and scst_restart_cmd(), must not be
+ *    called simultaneously for the same session (more precisely,
+ *    for the same session/LUN, i.e. tgt_dev), i.e. they must be
+ *    somehow externally serialized. This is needed to have lock free fast
+ *    path in scst_cmd_set_sn(). For majority of targets those functions are
+ *    naturally serialized by the single source of commands. Only iSCSI
+ *    immediate commands with multiple connections per session seems to be an
+ *    exception. For it, some mutex/lock shall be used for the serialization.
+ */
+void scst_cmd_init_done(struct scst_cmd *cmd,
+	enum scst_exec_context pref_context)
+{
+	unsigned long flags;
+	struct scst_session *sess = cmd->sess;
+	int rc;
+
+	scst_set_start_time(cmd);
+
+	TRACE_DBG("Preferred context: %d (cmd %p)", pref_context, cmd);
+	TRACE(TRACE_SCSI, "tag=%llu, lun=%lld, CDB len=%d, queue_type=%x "
+		"(cmd %p)", (long long unsigned int)cmd->tag,
+		(long long unsigned int)cmd->lun, cmd->cdb_len,
+		cmd->queue_type, cmd);
+	PRINT_BUFF_FLAG(TRACE_SCSI|TRACE_RCV_BOT, "Recieving CDB",
+		cmd->cdb, cmd->cdb_len);
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if (unlikely((in_irq() || irqs_disabled())) &&
+	    ((pref_context == SCST_CONTEXT_DIRECT) ||
+	     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
+		PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
+			"SCST_CONTEXT_THREAD instead", pref_context,
+			cmd->tgtt->name);
+		pref_context = SCST_CONTEXT_THREAD;
+	}
+#endif
+
+	atomic_inc(&sess->sess_cmd_count);
+
+	spin_lock_irqsave(&sess->sess_list_lock, flags);
+
+	if (unlikely(sess->init_phase != SCST_SESS_IPH_READY)) {
+		/*
+		 * We must always keep commands in the sess list from the
+		 * very beginning, because otherwise they can be missed during
+		 * TM processing. This check is needed because there might be
+		 * old, i.e. deferred, commands and new, i.e. just coming, ones.
+		 */
+		if (cmd->sess_cmd_list_entry.next == NULL)
+			list_add_tail(&cmd->sess_cmd_list_entry,
+				&sess->sess_cmd_list);
+		switch (sess->init_phase) {
+		case SCST_SESS_IPH_SUCCESS:
+			break;
+		case SCST_SESS_IPH_INITING:
+			TRACE_DBG("Adding cmd %p to init deferred cmd list",
+				  cmd);
+			list_add_tail(&cmd->cmd_list_entry,
+				&sess->init_deferred_cmd_list);
+			spin_unlock_irqrestore(&sess->sess_list_lock, flags);
+			goto out;
+		case SCST_SESS_IPH_FAILED:
+			spin_unlock_irqrestore(&sess->sess_list_lock, flags);
+			scst_set_busy(cmd);
+			scst_set_cmd_abnormal_done_state(cmd);
+			goto active;
+		default:
+			BUG();
+		}
+	} else
+		list_add_tail(&cmd->sess_cmd_list_entry,
+			      &sess->sess_cmd_list);
+
+	spin_unlock_irqrestore(&sess->sess_list_lock, flags);
+
+	if (unlikely(cmd->cdb_len == 0)) {
+		PRINT_ERROR("%s", "Wrong CDB len 0, finishing cmd");
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_invalid_opcode));
+		scst_set_cmd_abnormal_done_state(cmd);
+		goto active;
+	}
+
+	if (unlikely(cmd->queue_type >= SCST_CMD_QUEUE_ACA)) {
+		PRINT_ERROR("Unsupported queue type %d", cmd->queue_type);
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_message));
+		goto active;
+	}
+
+	/*
+	 * Cmd must be inited here to preserve the order. In case if cmd
+	 * already preliminary completed by target driver we need to init
+	 * cmd anyway to find out in which format we should return sense.
+	 */
+	cmd->state = SCST_CMD_STATE_INIT;
+	rc = scst_init_cmd(cmd, &pref_context);
+	if (unlikely(rc < 0))
+		goto out;
+
+active:
+	/* Here cmd must not be in any cmd list, no locks */
+	switch (pref_context) {
+	case SCST_CONTEXT_TASKLET:
+		scst_schedule_tasklet(cmd);
+		break;
+
+	case SCST_CONTEXT_DIRECT:
+		scst_process_active_cmd(cmd, false);
+		break;
+
+	case SCST_CONTEXT_DIRECT_ATOMIC:
+		scst_process_active_cmd(cmd, true);
+		break;
+
+	default:
+		PRINT_ERROR("Context %x is undefined, using the thread one",
+			pref_context);
+		/* go through */
+	case SCST_CONTEXT_THREAD:
+		spin_lock_irqsave(&cmd->cmd_threads->cmd_list_lock, flags);
+		TRACE_DBG("Adding cmd %p to active cmd list", cmd);
+		if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
+			list_add(&cmd->cmd_list_entry,
+				&cmd->cmd_threads->active_cmd_list);
+		else
+			list_add_tail(&cmd->cmd_list_entry,
+				&cmd->cmd_threads->active_cmd_list);
+		wake_up(&cmd->cmd_threads->cmd_list_waitQ);
+		spin_unlock_irqrestore(&cmd->cmd_threads->cmd_list_lock, flags);
+		break;
+	}
+
+out:
+	return;
+}
+EXPORT_SYMBOL(scst_cmd_init_done);
+
+static int scst_pre_parse(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME;
+	struct scst_device *dev = cmd->dev;
+	int rc;
+
+#ifdef CONFIG_SCST_STRICT_SERIALIZING
+	cmd->inc_expected_sn_on_done = 1;
+#else
+	cmd->inc_expected_sn_on_done = dev->handler->exec_sync ||
+	     (!dev->has_own_order_mgmt &&
+	      (dev->queue_alg == SCST_CONTR_MODE_QUEUE_ALG_RESTRICTED_REORDER ||
+	       cmd->queue_type == SCST_CMD_QUEUE_ORDERED));
+#endif
+
+	/*
+	 * Expected transfer data supplied by the SCSI transport via the
+	 * target driver are untrusted, so we prefer to fetch them from CDB.
+	 * Additionally, not all transports support supplying the expected
+	 * transfer data.
+	 */
+
+	rc = scst_get_cdb_info(cmd);
+	if (unlikely(rc != 0)) {
+		if (rc > 0) {
+			PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
+			goto out_err;
+		}
+
+		EXTRACHECKS_BUG_ON(cmd->op_flags & SCST_INFO_VALID);
+
+		cmd->cdb_len = scst_get_cdb_len(cmd->cdb);
+
+		TRACE(TRACE_MINOR, "Unknown opcode 0x%02x for %s. "
+			"Should you update scst_scsi_op_table?",
+			cmd->cdb[0], dev->handler->name);
+		PRINT_BUFF_FLAG(TRACE_MINOR, "Failed CDB", cmd->cdb,
+			cmd->cdb_len);
+	} else {
+		EXTRACHECKS_BUG_ON(!(cmd->op_flags & SCST_INFO_VALID));
+	}
+
+	cmd->state = SCST_CMD_STATE_DEV_PARSE;
+
+	TRACE_DBG("op_name <%s> (cmd %p), direction=%d "
+		"(expected %d, set %s), transfer_len=%d (expected "
+		"len %d), flags=%d", cmd->op_name, cmd,
+		cmd->data_direction, cmd->expected_data_direction,
+		scst_cmd_is_expected_set(cmd) ? "yes" : "no",
+		cmd->bufflen, cmd->expected_transfer_len,
+		cmd->op_flags);
+
+out:
+	return res;
+
+out_err:
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
+	scst_set_cmd_abnormal_done_state(cmd);
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+}
+
+#ifndef CONFIG_SCST_USE_EXPECTED_VALUES
+static bool scst_is_allowed_to_mismatch_cmd(struct scst_cmd *cmd)
+{
+	bool res = false;
+
+	/* VERIFY commands with BYTCHK unset shouldn't fail here */
+	if ((cmd->op_flags & SCST_VERIFY_BYTCHK_MISMATCH_ALLOWED) &&
+	    (cmd->cdb[1] & BYTCHK) == 0) {
+		res = true;
+		goto out;
+	}
+
+	switch (cmd->cdb[0]) {
+	case TEST_UNIT_READY:
+		/* Crazy VMware people sometimes do TUR with READ direction */
+		res = true;
+		break;
+	}
+
+out:
+	return res;
+}
+#endif
+
+static int scst_parse_cmd(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME;
+	int state;
+	struct scst_device *dev = cmd->dev;
+	int orig_bufflen = cmd->bufflen;
+
+	if (likely(!scst_is_cmd_fully_local(cmd))) {
+		if (unlikely(!dev->handler->parse_atomic &&
+			     scst_cmd_atomic(cmd))) {
+			/*
+			 * It shouldn't be because of the SCST_TGT_DEV_AFTER_*
+			 * optimization.
+			 */
+			TRACE_DBG("Dev handler %s parse() needs thread "
+				"context, rescheduling", dev->handler->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			goto out;
+		}
+
+		TRACE_DBG("Calling dev handler %s parse(%p)",
+		      dev->handler->name, cmd);
+		TRACE_BUFF_FLAG(TRACE_SND_BOT, "Parsing: ",
+				cmd->cdb, cmd->cdb_len);
+		scst_set_cur_start(cmd);
+		state = dev->handler->parse(cmd);
+		/* Caution: cmd can be already dead here */
+		TRACE_DBG("Dev handler %s parse() returned %d",
+			dev->handler->name, state);
+
+		switch (state) {
+		case SCST_CMD_STATE_NEED_THREAD_CTX:
+			scst_set_parse_time(cmd);
+			TRACE_DBG("Dev handler %s parse() requested thread "
+			      "context, rescheduling", dev->handler->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			goto out;
+
+		case SCST_CMD_STATE_STOP:
+			TRACE_DBG("Dev handler %s parse() requested stop "
+				"processing", dev->handler->name);
+			res = SCST_CMD_STATE_RES_CONT_NEXT;
+			goto out;
+		}
+
+		scst_set_parse_time(cmd);
+
+		if (state == SCST_CMD_STATE_DEFAULT)
+			state = SCST_CMD_STATE_PREPARE_SPACE;
+	} else
+		state = SCST_CMD_STATE_PREPARE_SPACE;
+
+	if (unlikely(state == SCST_CMD_STATE_PRE_XMIT_RESP))
+		goto set_res;
+
+	if (unlikely(!(cmd->op_flags & SCST_INFO_VALID))) {
+#ifdef CONFIG_SCST_USE_EXPECTED_VALUES
+		if (scst_cmd_is_expected_set(cmd)) {
+			TRACE(TRACE_MINOR, "Using initiator supplied values: "
+				"direction %d, transfer_len %d",
+				cmd->expected_data_direction,
+				cmd->expected_transfer_len);
+			cmd->data_direction = cmd->expected_data_direction;
+			cmd->bufflen = cmd->expected_transfer_len;
+		} else {
+			PRINT_ERROR("Unknown opcode 0x%02x for %s and "
+			     "target %s not supplied expected values",
+			     cmd->cdb[0], dev->handler->name, cmd->tgtt->name);
+			scst_set_cmd_error(cmd,
+				   SCST_LOAD_SENSE(scst_sense_invalid_opcode));
+			goto out_done;
+		}
+#else
+		PRINT_ERROR("Unknown opcode %x", cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_invalid_opcode));
+		goto out_done;
+#endif
+	}
+
+	if (unlikely(cmd->cdb_len == -1)) {
+		PRINT_ERROR("Unable to get CDB length for "
+			"opcode 0x%02x. Returning INVALID "
+			"OPCODE", cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_opcode));
+		goto out_done;
+	}
+
+	EXTRACHECKS_BUG_ON(cmd->cdb_len == 0);
+
+	TRACE(TRACE_SCSI, "op_name <%s> (cmd %p), direction=%d "
+		"(expected %d, set %s), transfer_len=%d (expected "
+		"len %d), flags=%d", cmd->op_name, cmd,
+		cmd->data_direction, cmd->expected_data_direction,
+		scst_cmd_is_expected_set(cmd) ? "yes" : "no",
+		cmd->bufflen, cmd->expected_transfer_len,
+		cmd->op_flags);
+
+	if (unlikely((cmd->op_flags & SCST_UNKNOWN_LENGTH) != 0)) {
+		if (scst_cmd_is_expected_set(cmd)) {
+			/*
+			 * Command data length can't be easily
+			 * determined from the CDB. ToDo, all such
+			 * commands processing should be fixed. Until
+			 * it's done, get the length from the supplied
+			 * expected value, but limit it to some
+			 * reasonable value (15MB).
+			 */
+			cmd->bufflen = min(cmd->expected_transfer_len,
+						15*1024*1024);
+			cmd->op_flags &= ~SCST_UNKNOWN_LENGTH;
+		} else {
+			PRINT_ERROR("Unknown data transfer length for opcode "
+				"0x%x (handler %s, target %s)", cmd->cdb[0],
+				dev->handler->name, cmd->tgtt->name);
+			PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
+			scst_set_cmd_error(cmd,
+				SCST_LOAD_SENSE(scst_sense_invalid_message));
+			goto out_done;
+		}
+	}
+
+	if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_NACA_BIT)) {
+		PRINT_ERROR("NACA bit in control byte CDB is not supported "
+			    "(opcode 0x%02x)", cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
+		goto out_done;
+	}
+
+	if (unlikely(cmd->cdb[cmd->cdb_len - 1] & CONTROL_BYTE_LINK_BIT)) {
+		PRINT_ERROR("Linked commands are not supported "
+			    "(opcode 0x%02x)", cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
+		goto out_done;
+	}
+
+	if (cmd->dh_data_buf_alloced &&
+	    unlikely((orig_bufflen > cmd->bufflen))) {
+		PRINT_ERROR("Dev handler supplied data buffer (size %d), "
+			"is less, than required (size %d)", cmd->bufflen,
+			orig_bufflen);
+		PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
+		goto out_hw_error;
+	}
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if ((cmd->bufflen != 0) &&
+	    ((cmd->data_direction == SCST_DATA_NONE) ||
+	     ((cmd->sg == NULL) && (state > SCST_CMD_STATE_PREPARE_SPACE)))) {
+		PRINT_ERROR("Dev handler %s parse() returned "
+			"invalid cmd data_direction %d, bufflen %d, state %d "
+			"or sg %p (opcode 0x%x)", dev->handler->name,
+			cmd->data_direction, cmd->bufflen, state, cmd->sg,
+			cmd->cdb[0]);
+		PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
+		goto out_hw_error;
+	}
+#endif
+
+	if (scst_cmd_is_expected_set(cmd)) {
+#ifdef CONFIG_SCST_USE_EXPECTED_VALUES
+#	ifdef CONFIG_SCST_EXTRACHECKS
+		if (unlikely((cmd->data_direction != cmd->expected_data_direction) ||
+			     (cmd->bufflen != cmd->expected_transfer_len))) {
+			TRACE(TRACE_MINOR, "Expected values don't match "
+				"decoded ones: data_direction %d, "
+				"expected_data_direction %d, "
+				"bufflen %d, expected_transfer_len %d",
+				cmd->data_direction,
+				cmd->expected_data_direction,
+				cmd->bufflen, cmd->expected_transfer_len);
+			PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
+				cmd->cdb, cmd->cdb_len);
+		}
+#	endif
+		cmd->data_direction = cmd->expected_data_direction;
+		cmd->bufflen = cmd->expected_transfer_len;
+#else
+		if (unlikely(cmd->data_direction !=
+				cmd->expected_data_direction)) {
+			if (((cmd->expected_data_direction != SCST_DATA_NONE) ||
+			     (cmd->bufflen != 0)) &&
+			    !scst_is_allowed_to_mismatch_cmd(cmd)) {
+				PRINT_ERROR("Expected data direction %d for "
+					"opcode 0x%02x (handler %s, target %s) "
+					"doesn't match decoded value %d",
+					cmd->expected_data_direction,
+					cmd->cdb[0], dev->handler->name,
+					cmd->tgtt->name, cmd->data_direction);
+				PRINT_BUFFER("Failed CDB", cmd->cdb,
+					cmd->cdb_len);
+				scst_set_cmd_error(cmd,
+				   SCST_LOAD_SENSE(scst_sense_invalid_message));
+				goto out_done;
+			}
+		}
+		if (unlikely(cmd->bufflen != cmd->expected_transfer_len)) {
+			TRACE(TRACE_MINOR, "Warning: expected "
+				"transfer length %d for opcode 0x%02x "
+				"(handler %s, target %s) doesn't match "
+				"decoded value %d. Faulty initiator "
+				"(e.g. VMware is known to be such) or "
+				"scst_scsi_op_table should be updated?",
+				cmd->expected_transfer_len, cmd->cdb[0],
+				dev->handler->name, cmd->tgtt->name,
+				cmd->bufflen);
+			PRINT_BUFF_FLAG(TRACE_MINOR, "Suspicious CDB",
+				cmd->cdb, cmd->cdb_len);
+			/* Needed, e.g., to get immediate iSCSI data */
+			cmd->bufflen = max(cmd->bufflen,
+					   cmd->expected_transfer_len);
+		}
+#endif
+	}
+
+	if (unlikely(cmd->data_direction == SCST_DATA_UNKNOWN)) {
+		PRINT_ERROR("Unknown data direction. Opcode 0x%x, handler %s, "
+			"target %s", cmd->cdb[0], dev->handler->name,
+			cmd->tgtt->name);
+		PRINT_BUFFER("Failed CDB", cmd->cdb, cmd->cdb_len);
+		goto out_hw_error;
+	}
+
+set_res:
+	if (cmd->data_len == -1)
+		cmd->data_len = cmd->bufflen;
+
+	if (cmd->bufflen == 0) {
+		/*
+		 * According to SPC bufflen 0 for data transfer commands isn't
+		 * an error, so we need to fix the transfer direction.
+		 */
+		cmd->data_direction = SCST_DATA_NONE;
+	}
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	switch (state) {
+	case SCST_CMD_STATE_PREPARE_SPACE:
+	case SCST_CMD_STATE_PRE_PARSE:
+	case SCST_CMD_STATE_DEV_PARSE:
+	case SCST_CMD_STATE_RDY_TO_XFER:
+	case SCST_CMD_STATE_TGT_PRE_EXEC:
+	case SCST_CMD_STATE_SEND_FOR_EXEC:
+	case SCST_CMD_STATE_LOCAL_EXEC:
+	case SCST_CMD_STATE_REAL_EXEC:
+	case SCST_CMD_STATE_PRE_DEV_DONE:
+	case SCST_CMD_STATE_DEV_DONE:
+	case SCST_CMD_STATE_PRE_XMIT_RESP:
+	case SCST_CMD_STATE_XMIT_RESP:
+	case SCST_CMD_STATE_FINISHED:
+	case SCST_CMD_STATE_FINISHED_INTERNAL:
+#endif
+		cmd->state = state;
+		res = SCST_CMD_STATE_RES_CONT_SAME;
+#ifdef CONFIG_SCST_EXTRACHECKS
+		break;
+
+	default:
+		if (state >= 0) {
+			PRINT_ERROR("Dev handler %s parse() returned "
+			     "invalid cmd state %d (opcode %d)",
+			     dev->handler->name, state, cmd->cdb[0]);
+		} else {
+			PRINT_ERROR("Dev handler %s parse() returned "
+				"error %d (opcode %d)", dev->handler->name,
+				state, cmd->cdb[0]);
+		}
+		goto out_hw_error;
+	}
+#endif
+
+	if (cmd->resp_data_len == -1) {
+		if (cmd->data_direction & SCST_DATA_READ)
+			cmd->resp_data_len = cmd->bufflen;
+		else
+			 cmd->resp_data_len = 0;
+	}
+
+	/* We already completed (with an error) */
+	if (unlikely(cmd->completed))
+		goto out_done;
+
+out:
+	return res;
+
+out_hw_error:
+	/* dev_done() will be called as part of the regular cmd's finish */
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+
+out_done:
+	scst_set_cmd_abnormal_done_state(cmd);
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+}
+
+static int scst_prepare_space(struct scst_cmd *cmd)
+{
+	int r = 0, res = SCST_CMD_STATE_RES_CONT_SAME;
+
+	if (cmd->data_direction == SCST_DATA_NONE)
+		goto done;
+
+	if (cmd->tgt_need_alloc_data_buf) {
+		int orig_bufflen = cmd->bufflen;
+
+		TRACE_MEM("Custom tgt data buf allocation requested (cmd %p)",
+			cmd);
+
+		scst_set_cur_start(cmd);
+		r = cmd->tgtt->alloc_data_buf(cmd);
+		scst_set_alloc_buf_time(cmd);
+
+		if (r > 0)
+			goto alloc;
+		else if (r == 0) {
+			if (unlikely(cmd->bufflen == 0)) {
+				/* See comment in scst_alloc_space() */
+				if (cmd->sg == NULL)
+					goto alloc;
+			}
+
+			cmd->tgt_data_buf_alloced = 1;
+
+			if (unlikely(orig_bufflen < cmd->bufflen)) {
+				PRINT_ERROR("Target driver allocated data "
+					"buffer (size %d), is less, than "
+					"required (size %d)", orig_bufflen,
+					cmd->bufflen);
+				goto out_error;
+			}
+			TRACE_MEM("tgt_data_buf_alloced (cmd %p)", cmd);
+		} else
+			goto check;
+	}
+
+alloc:
+	if (!cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
+		r = scst_alloc_space(cmd);
+	} else if (cmd->dh_data_buf_alloced && !cmd->tgt_data_buf_alloced) {
+		TRACE_MEM("dh_data_buf_alloced set (cmd %p)", cmd);
+		r = 0;
+	} else if (cmd->tgt_data_buf_alloced && !cmd->dh_data_buf_alloced) {
+		TRACE_MEM("tgt_data_buf_alloced set (cmd %p)", cmd);
+		cmd->sg = cmd->tgt_sg;
+		cmd->sg_cnt = cmd->tgt_sg_cnt;
+		cmd->in_sg = cmd->tgt_in_sg;
+		cmd->in_sg_cnt = cmd->tgt_in_sg_cnt;
+		r = 0;
+	} else {
+		TRACE_MEM("Both *_data_buf_alloced set (cmd %p, sg %p, "
+			"sg_cnt %d, tgt_sg %p, tgt_sg_cnt %d)", cmd, cmd->sg,
+			cmd->sg_cnt, cmd->tgt_sg, cmd->tgt_sg_cnt);
+		r = 0;
+	}
+
+check:
+	if (r != 0) {
+		if (scst_cmd_atomic(cmd)) {
+			TRACE_MEM("%s", "Atomic memory allocation failed, "
+			      "rescheduling to the thread");
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			goto out;
+		} else
+			goto out_no_space;
+	}
+
+done:
+	if (cmd->preprocessing_only)
+		cmd->state = SCST_CMD_STATE_PREPROCESSING_DONE;
+	else if (cmd->data_direction & SCST_DATA_WRITE)
+		cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
+	else
+		cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
+
+out:
+	return res;
+
+out_no_space:
+	TRACE(TRACE_OUT_OF_MEM, "Unable to allocate or build requested buffer "
+		"(size %d), sending BUSY or QUEUE FULL status", cmd->bufflen);
+	scst_set_busy(cmd);
+	scst_set_cmd_abnormal_done_state(cmd);
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+
+out_error:
+	scst_set_cmd_error(cmd,	SCST_LOAD_SENSE(scst_sense_hardw_error));
+	scst_set_cmd_abnormal_done_state(cmd);
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+}
+
+static int scst_preprocessing_done(struct scst_cmd *cmd)
+{
+	int res;
+
+	EXTRACHECKS_BUG_ON(!cmd->preprocessing_only);
+
+	cmd->preprocessing_only = 0;
+
+	res = SCST_CMD_STATE_RES_CONT_NEXT;
+	cmd->state = SCST_CMD_STATE_PREPROCESSING_DONE_CALLED;
+
+	TRACE_DBG("Calling preprocessing_done(cmd %p)", cmd);
+	scst_set_cur_start(cmd);
+	cmd->tgtt->preprocessing_done(cmd);
+	TRACE_DBG("%s", "preprocessing_done() returned");
+	return res;
+}
+
+/**
+ * scst_restart_cmd() - restart execution of the command
+ * @cmd:	SCST commands
+ * @status:	completion status
+ * @pref_context: preferred command execition context
+ *
+ * Description:
+ *    Notifies SCST that the driver finished its part of the command's
+ *    preprocessing and it is ready for further processing.
+ *
+ *    The second argument sets completion status
+ *    (see SCST_PREPROCESS_STATUS_* constants for details)
+ *
+ *    See also comment for scst_cmd_init_done() for the serialization
+ *    requirements.
+ */
+void scst_restart_cmd(struct scst_cmd *cmd, int status,
+	enum scst_exec_context pref_context)
+{
+
+	scst_set_restart_waiting_time(cmd);
+
+	TRACE_DBG("Preferred context: %d", pref_context);
+	TRACE_DBG("tag=%llu, status=%#x",
+		  (long long unsigned int)scst_cmd_get_tag(cmd),
+		  status);
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if ((in_irq() || irqs_disabled()) &&
+	    ((pref_context == SCST_CONTEXT_DIRECT) ||
+	     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
+		PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
+			"SCST_CONTEXT_THREAD instead", pref_context,
+			cmd->tgtt->name);
+		pref_context = SCST_CONTEXT_THREAD;
+	}
+#endif
+
+	switch (status) {
+	case SCST_PREPROCESS_STATUS_SUCCESS:
+		if (cmd->data_direction & SCST_DATA_WRITE)
+			cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
+		else
+			cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
+		if (cmd->set_sn_on_restart_cmd)
+			scst_cmd_set_sn(cmd);
+		/* Small context optimization */
+		if ((pref_context == SCST_CONTEXT_TASKLET) ||
+		    (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
+		    ((pref_context == SCST_CONTEXT_SAME) &&
+		     scst_cmd_atomic(cmd))) {
+			if (cmd->data_direction & SCST_DATA_WRITE) {
+				if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_WR_ATOMIC,
+						&cmd->tgt_dev->tgt_dev_flags))
+					pref_context = SCST_CONTEXT_THREAD;
+			} else {
+				if (!test_bit(SCST_TGT_DEV_AFTER_RESTART_OTH_ATOMIC,
+						&cmd->tgt_dev->tgt_dev_flags))
+					pref_context = SCST_CONTEXT_THREAD;
+			}
+		}
+		break;
+
+	case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+
+	case SCST_PREPROCESS_STATUS_ERROR_FATAL:
+		set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
+		/* go through */
+	case SCST_PREPROCESS_STATUS_ERROR:
+		if (cmd->sense != NULL)
+			scst_set_cmd_error(cmd,
+				SCST_LOAD_SENSE(scst_sense_hardw_error));
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+
+	default:
+		PRINT_ERROR("%s() received unknown status %x", __func__,
+			status);
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+	}
+
+	scst_process_redirect_cmd(cmd, pref_context, 1);
+	return;
+}
+EXPORT_SYMBOL(scst_restart_cmd);
+
+static int scst_rdy_to_xfer(struct scst_cmd *cmd)
+{
+	int res, rc;
+	struct scst_tgt_template *tgtt = cmd->tgtt;
+
+	if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
+		TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
+		goto out_dev_done;
+	}
+
+	if ((tgtt->rdy_to_xfer == NULL) || unlikely(cmd->internal)) {
+		cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
+		res = SCST_CMD_STATE_RES_CONT_SAME;
+		goto out;
+	}
+
+	if (unlikely(!tgtt->rdy_to_xfer_atomic && scst_cmd_atomic(cmd))) {
+		/*
+		 * It shouldn't be because of the SCST_TGT_DEV_AFTER_*
+		 * optimization.
+		 */
+		TRACE_DBG("Target driver %s rdy_to_xfer() needs thread "
+			      "context, rescheduling", tgtt->name);
+		res = SCST_CMD_STATE_RES_NEED_THREAD;
+		goto out;
+	}
+
+	while (1) {
+		int finished_cmds = atomic_read(&cmd->tgt->finished_cmds);
+
+		res = SCST_CMD_STATE_RES_CONT_NEXT;
+		cmd->state = SCST_CMD_STATE_DATA_WAIT;
+
+		if (tgtt->on_hw_pending_cmd_timeout != NULL) {
+			struct scst_session *sess = cmd->sess;
+			cmd->hw_pending_start = jiffies;
+			cmd->cmd_hw_pending = 1;
+			if (!test_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED, &sess->sess_aflags)) {
+				TRACE_DBG("Sched HW pending work for sess %p "
+					"(max time %d)", sess,
+					tgtt->max_hw_pending_time);
+				set_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED,
+					&sess->sess_aflags);
+				schedule_delayed_work(&sess->hw_pending_work,
+					tgtt->max_hw_pending_time * HZ);
+			}
+		}
+
+		scst_set_cur_start(cmd);
+
+		TRACE_DBG("Calling rdy_to_xfer(%p)", cmd);
+#ifdef CONFIG_SCST_DEBUG_RETRY
+		if (((scst_random() % 100) == 75))
+			rc = SCST_TGT_RES_QUEUE_FULL;
+		else
+#endif
+			rc = tgtt->rdy_to_xfer(cmd);
+		TRACE_DBG("rdy_to_xfer() returned %d", rc);
+
+		if (likely(rc == SCST_TGT_RES_SUCCESS))
+			goto out;
+
+		scst_set_rdy_to_xfer_time(cmd);
+
+		cmd->cmd_hw_pending = 0;
+
+		/* Restore the previous state */
+		cmd->state = SCST_CMD_STATE_RDY_TO_XFER;
+
+		switch (rc) {
+		case SCST_TGT_RES_QUEUE_FULL:
+			if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
+				break;
+			else
+				continue;
+
+		case SCST_TGT_RES_NEED_THREAD_CTX:
+			TRACE_DBG("Target driver %s "
+			      "rdy_to_xfer() requested thread "
+			      "context, rescheduling", tgtt->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			break;
+
+		default:
+			goto out_error_rc;
+		}
+		break;
+	}
+
+out:
+	return res;
+
+out_error_rc:
+	if (rc == SCST_TGT_RES_FATAL_ERROR) {
+		PRINT_ERROR("Target driver %s rdy_to_xfer() returned "
+		     "fatal error", tgtt->name);
+	} else {
+		PRINT_ERROR("Target driver %s rdy_to_xfer() returned invalid "
+			    "value %d", tgtt->name, rc);
+	}
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+
+out_dev_done:
+	scst_set_cmd_abnormal_done_state(cmd);
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+}
+
+/* No locks, but might be in IRQ */
+static void scst_process_redirect_cmd(struct scst_cmd *cmd,
+	enum scst_exec_context context, int check_retries)
+{
+	struct scst_tgt *tgt = cmd->tgt;
+	unsigned long flags;
+
+	TRACE_DBG("Context: %x", context);
+
+	if (context == SCST_CONTEXT_SAME)
+		context = scst_cmd_atomic(cmd) ? SCST_CONTEXT_DIRECT_ATOMIC :
+						 SCST_CONTEXT_DIRECT;
+
+	switch (context) {
+	case SCST_CONTEXT_DIRECT_ATOMIC:
+		scst_process_active_cmd(cmd, true);
+		break;
+
+	case SCST_CONTEXT_DIRECT:
+		if (check_retries)
+			scst_check_retries(tgt);
+		scst_process_active_cmd(cmd, false);
+		break;
+
+	default:
+		PRINT_ERROR("Context %x is unknown, using the thread one",
+			    context);
+		/* go through */
+	case SCST_CONTEXT_THREAD:
+		if (check_retries)
+			scst_check_retries(tgt);
+		spin_lock_irqsave(&cmd->cmd_threads->cmd_list_lock, flags);
+		TRACE_DBG("Adding cmd %p to active cmd list", cmd);
+		if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
+			list_add(&cmd->cmd_list_entry,
+				&cmd->cmd_threads->active_cmd_list);
+		else
+			list_add_tail(&cmd->cmd_list_entry,
+				&cmd->cmd_threads->active_cmd_list);
+		wake_up(&cmd->cmd_threads->cmd_list_waitQ);
+		spin_unlock_irqrestore(&cmd->cmd_threads->cmd_list_lock, flags);
+		break;
+
+	case SCST_CONTEXT_TASKLET:
+		if (check_retries)
+			scst_check_retries(tgt);
+		scst_schedule_tasklet(cmd);
+		break;
+	}
+	return;
+}
+
+/**
+ * scst_rx_data() - the command's data received
+ * @cmd:	SCST commands
+ * @status:	data receiving completion status
+ * @pref_context: preferred command execition context
+ *
+ * Description:
+ *    Notifies SCST that the driver received all the necessary data
+ *    and the command is ready for further processing.
+ *
+ *    The second argument sets data receiving completion status
+ *    (see SCST_RX_STATUS_* constants for details)
+ */
+void scst_rx_data(struct scst_cmd *cmd, int status,
+	enum scst_exec_context pref_context)
+{
+
+	scst_set_rdy_to_xfer_time(cmd);
+
+	TRACE_DBG("Preferred context: %d", pref_context);
+	TRACE(TRACE_SCSI, "cmd %p, status %#x", cmd, status);
+
+	cmd->cmd_hw_pending = 0;
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if ((in_irq() || irqs_disabled()) &&
+	    ((pref_context == SCST_CONTEXT_DIRECT) ||
+	     (pref_context == SCST_CONTEXT_DIRECT_ATOMIC))) {
+		PRINT_ERROR("Wrong context %d in IRQ from target %s, use "
+			"SCST_CONTEXT_THREAD instead", pref_context,
+			cmd->tgtt->name);
+		pref_context = SCST_CONTEXT_THREAD;
+	}
+#endif
+
+	switch (status) {
+	case SCST_RX_STATUS_SUCCESS:
+#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
+		if (trace_flag & TRACE_RCV_BOT) {
+			int i;
+			struct scatterlist *sg;
+			if (cmd->in_sg != NULL)
+				sg = cmd->in_sg;
+			else if (cmd->tgt_in_sg != NULL)
+				sg = cmd->tgt_in_sg;
+			else if (cmd->tgt_sg != NULL)
+				sg = cmd->tgt_sg;
+			else
+				sg = cmd->sg;
+			if (sg != NULL) {
+				TRACE_RECV_BOT("RX data for cmd %p "
+					"(sg_cnt %d, sg %p, sg[0].page %p)",
+					cmd, cmd->tgt_sg_cnt, sg,
+					(void *)sg_page(&sg[0]));
+				for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
+					PRINT_BUFF_FLAG(TRACE_RCV_BOT, "RX sg",
+						sg_virt(&sg[i]), sg[i].length);
+				}
+			}
+		}
+#endif
+		cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
+
+		/* Small context optimization */
+		if ((pref_context == SCST_CONTEXT_TASKLET) ||
+		    (pref_context == SCST_CONTEXT_DIRECT_ATOMIC) ||
+		    ((pref_context == SCST_CONTEXT_SAME) &&
+		     scst_cmd_atomic(cmd))) {
+			if (!test_bit(SCST_TGT_DEV_AFTER_RX_DATA_ATOMIC,
+					&cmd->tgt_dev->tgt_dev_flags))
+				pref_context = SCST_CONTEXT_THREAD;
+		}
+		break;
+
+	case SCST_RX_STATUS_ERROR_SENSE_SET:
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+
+	case SCST_RX_STATUS_ERROR_FATAL:
+		set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
+		/* go through */
+	case SCST_RX_STATUS_ERROR:
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_hardw_error));
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+
+	default:
+		PRINT_ERROR("scst_rx_data() received unknown status %x",
+			status);
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+	}
+
+	scst_process_redirect_cmd(cmd, pref_context, 1);
+	return;
+}
+EXPORT_SYMBOL(scst_rx_data);
+
+static int scst_tgt_pre_exec(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
+
+	cmd->state = SCST_CMD_STATE_SEND_FOR_EXEC;
+
+	if ((cmd->tgtt->pre_exec == NULL) || unlikely(cmd->internal))
+		goto out;
+
+	TRACE_DBG("Calling pre_exec(%p)", cmd);
+	scst_set_cur_start(cmd);
+	rc = cmd->tgtt->pre_exec(cmd);
+	scst_set_pre_exec_time(cmd);
+	TRACE_DBG("pre_exec() returned %d", rc);
+
+	if (unlikely(rc != SCST_PREPROCESS_STATUS_SUCCESS)) {
+		switch (rc) {
+		case SCST_PREPROCESS_STATUS_ERROR_SENSE_SET:
+			scst_set_cmd_abnormal_done_state(cmd);
+			break;
+		case SCST_PREPROCESS_STATUS_ERROR_FATAL:
+			set_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags);
+			/* go through */
+		case SCST_PREPROCESS_STATUS_ERROR:
+			scst_set_cmd_error(cmd,
+				   SCST_LOAD_SENSE(scst_sense_hardw_error));
+			scst_set_cmd_abnormal_done_state(cmd);
+			break;
+		case SCST_PREPROCESS_STATUS_NEED_THREAD:
+			TRACE_DBG("Target driver's %s pre_exec() requested "
+				"thread context, rescheduling",
+				cmd->tgtt->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			cmd->state = SCST_CMD_STATE_TGT_PRE_EXEC;
+			break;
+		default:
+			BUG();
+			break;
+		}
+	}
+
+out:
+	return res;
+}
+
+static void scst_do_cmd_done(struct scst_cmd *cmd, int result,
+	const uint8_t *rq_sense, int rq_sense_len, int resid)
+{
+
+	scst_set_exec_time(cmd);
+
+	cmd->status = result & 0xff;
+	cmd->msg_status = msg_byte(result);
+	cmd->host_status = host_byte(result);
+	cmd->driver_status = driver_byte(result);
+	if (unlikely(resid != 0)) {
+#ifdef CONFIG_SCST_EXTRACHECKS
+		if ((resid < 0) || (resid > cmd->resp_data_len)) {
+			PRINT_ERROR("Wrong resid %d (cmd->resp_data_len=%d, "
+				"op %x)", resid, cmd->resp_data_len,
+				cmd->cdb[0]);
+		} else
+#endif
+			scst_set_resp_data_len(cmd, cmd->resp_data_len - resid);
+	}
+
+	if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION)) {
+		/* We might have double reset UA here */
+		cmd->dbl_ua_orig_resp_data_len = cmd->resp_data_len;
+		cmd->dbl_ua_orig_data_direction = cmd->data_direction;
+
+		scst_alloc_set_sense(cmd, 1, rq_sense, rq_sense_len);
+	}
+
+	TRACE(TRACE_SCSI, "cmd %p, result=%x, cmd->status=%x, resid=%d, "
+	      "cmd->msg_status=%x, cmd->host_status=%x, "
+	      "cmd->driver_status=%x (cmd %p)", cmd, result, cmd->status, resid,
+	      cmd->msg_status, cmd->host_status, cmd->driver_status, cmd);
+
+	cmd->completed = 1;
+	return;
+}
+
+/* For small context optimization */
+static inline enum scst_exec_context scst_optimize_post_exec_context(
+	struct scst_cmd *cmd, enum scst_exec_context context)
+{
+	if (((context == SCST_CONTEXT_SAME) && scst_cmd_atomic(cmd)) ||
+	    (context == SCST_CONTEXT_TASKLET) ||
+	    (context == SCST_CONTEXT_DIRECT_ATOMIC)) {
+		if (!test_bit(SCST_TGT_DEV_AFTER_EXEC_ATOMIC,
+				&cmd->tgt_dev->tgt_dev_flags))
+			context = SCST_CONTEXT_THREAD;
+	}
+	return context;
+}
+
+static void scst_cmd_done(void *data, char *sense, int result, int resid)
+{
+	struct scst_cmd *cmd;
+
+	cmd = (struct scst_cmd *)data;
+	if (cmd == NULL)
+		goto out;
+
+	scst_do_cmd_done(cmd, result, sense, SCSI_SENSE_BUFFERSIZE, resid);
+
+	cmd->state = SCST_CMD_STATE_PRE_DEV_DONE;
+
+	scst_process_redirect_cmd(cmd,
+	    scst_optimize_post_exec_context(cmd, scst_estimate_context()), 0);
+
+out:
+	return;
+}
+
+static void scst_cmd_done_local(struct scst_cmd *cmd, int next_state,
+	enum scst_exec_context pref_context)
+{
+
+	scst_set_exec_time(cmd);
+
+	if (next_state == SCST_CMD_STATE_DEFAULT)
+		next_state = SCST_CMD_STATE_PRE_DEV_DONE;
+
+#if defined(CONFIG_SCST_DEBUG)
+	if (next_state == SCST_CMD_STATE_PRE_DEV_DONE) {
+		if ((trace_flag & TRACE_RCV_TOP) && (cmd->sg != NULL)) {
+			int i;
+			struct scatterlist *sg = cmd->sg;
+			TRACE_RECV_TOP("Exec'd %d S/G(s) at %p sg[0].page at "
+				"%p", cmd->sg_cnt, sg, (void *)sg_page(&sg[0]));
+			for (i = 0; i < cmd->sg_cnt; ++i) {
+				TRACE_BUFF_FLAG(TRACE_RCV_TOP,
+					"Exec'd sg", sg_virt(&sg[i]),
+					sg[i].length);
+			}
+		}
+	}
+#endif
+
+	cmd->state = next_state;
+
+#ifdef CONFIG_SCST_EXTRACHECKS
+	if ((next_state != SCST_CMD_STATE_PRE_DEV_DONE) &&
+	    (next_state != SCST_CMD_STATE_PRE_XMIT_RESP) &&
+	    (next_state != SCST_CMD_STATE_FINISHED) &&
+	    (next_state != SCST_CMD_STATE_FINISHED_INTERNAL)) {
+		PRINT_ERROR("%s() received invalid cmd state %d (opcode %d)",
+			__func__, next_state, cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+				   SCST_LOAD_SENSE(scst_sense_hardw_error));
+		scst_set_cmd_abnormal_done_state(cmd);
+	}
+#endif
+	pref_context = scst_optimize_post_exec_context(cmd, pref_context);
+	scst_process_redirect_cmd(cmd, pref_context, 0);
+	return;
+}
+
+static int scst_report_luns_local(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_COMPLETED, rc;
+	int dev_cnt = 0;
+	int buffer_size;
+	int i;
+	struct scst_tgt_dev *tgt_dev = NULL;
+	uint8_t *buffer;
+	int offs, overflow = 0;
+
+	if (scst_cmd_atomic(cmd)) {
+		res = SCST_EXEC_NEED_THREAD;
+		goto out;
+	}
+
+	rc = scst_check_local_events(cmd);
+	if (unlikely(rc != 0))
+		goto out_done;
+
+	cmd->status = 0;
+	cmd->msg_status = 0;
+	cmd->host_status = DID_OK;
+	cmd->driver_status = 0;
+
+	if ((cmd->cdb[2] != 0) && (cmd->cdb[2] != 2)) {
+		PRINT_ERROR("Unsupported SELECT REPORT value %x in REPORT "
+			"LUNS command", cmd->cdb[2]);
+		goto out_err;
+	}
+
+	buffer_size = scst_get_buf_first(cmd, &buffer);
+	if (unlikely(buffer_size == 0))
+		goto out_compl;
+	else if (unlikely(buffer_size < 0))
+		goto out_hw_err;
+
+	if (buffer_size < 16)
+		goto out_put_err;
+
+	memset(buffer, 0, buffer_size);
+	offs = 8;
+
+	/*
+	 * cmd won't allow to suspend activities, so we can access
+	 * sess->sess_tgt_dev_list_hash without any additional protection.
+	 */
+	for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
+		struct list_head *sess_tgt_dev_list_head =
+			&cmd->sess->sess_tgt_dev_list_hash[i];
+		list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
+				sess_tgt_dev_list_entry) {
+			if (!overflow) {
+				if (offs >= buffer_size) {
+					scst_put_buf(cmd, buffer);
+					buffer_size = scst_get_buf_next(cmd,
+								       &buffer);
+					if (buffer_size > 0) {
+						memset(buffer, 0, buffer_size);
+						offs = 0;
+					} else {
+						overflow = 1;
+						goto inc_dev_cnt;
+					}
+				}
+				if ((buffer_size - offs) < 8) {
+					PRINT_ERROR("Buffer allocated for "
+						"REPORT LUNS command doesn't "
+						"allow to fit 8 byte entry "
+						"(buffer_size=%d)",
+						buffer_size);
+					goto out_put_hw_err;
+				}
+				if ((cmd->sess->acg->addr_method == SCST_LUN_ADDR_METHOD_FLAT) &&
+				    (tgt_dev->lun != 0)) {
+					buffer[offs] = (tgt_dev->lun >> 8) & 0x3f;
+					buffer[offs] = buffer[offs] | 0x40;
+					buffer[offs+1] = tgt_dev->lun & 0xff;
+				} else {
+					buffer[offs] = (tgt_dev->lun >> 8) & 0xff;
+					buffer[offs+1] = tgt_dev->lun & 0xff;
+				}
+				offs += 8;
+			}
+inc_dev_cnt:
+			dev_cnt++;
+		}
+	}
+	if (!overflow)
+		scst_put_buf(cmd, buffer);
+
+	/* Set the response header */
+	buffer_size = scst_get_buf_first(cmd, &buffer);
+	if (unlikely(buffer_size == 0))
+		goto out_compl;
+	else if (unlikely(buffer_size < 0))
+		goto out_hw_err;
+
+	dev_cnt *= 8;
+	buffer[0] = (dev_cnt >> 24) & 0xff;
+	buffer[1] = (dev_cnt >> 16) & 0xff;
+	buffer[2] = (dev_cnt >> 8) & 0xff;
+	buffer[3] = dev_cnt & 0xff;
+
+	scst_put_buf(cmd, buffer);
+
+	dev_cnt += 8;
+	if (dev_cnt < cmd->resp_data_len)
+		scst_set_resp_data_len(cmd, dev_cnt);
+
+out_compl:
+	cmd->completed = 1;
+
+	/* Clear left sense_reported_luns_data_changed UA, if any. */
+
+	/*
+	 * cmd won't allow to suspend activities, so we can access
+	 * sess->sess_tgt_dev_list_hash without any additional protection.
+	 */
+	for (i = 0; i < TGT_DEV_HASH_SIZE; i++) {
+		struct list_head *sess_tgt_dev_list_head =
+			&cmd->sess->sess_tgt_dev_list_hash[i];
+
+		list_for_each_entry(tgt_dev, sess_tgt_dev_list_head,
+				sess_tgt_dev_list_entry) {
+			struct scst_tgt_dev_UA *ua;
+
+			spin_lock_bh(&tgt_dev->tgt_dev_lock);
+			list_for_each_entry(ua, &tgt_dev->UA_list,
+						UA_list_entry) {
+				if (scst_analyze_sense(ua->UA_sense_buffer,
+						ua->UA_valid_sense_len,
+						SCST_SENSE_ALL_VALID,
+						SCST_LOAD_SENSE(scst_sense_reported_luns_data_changed))) {
+					TRACE_MGMT_DBG("Freeing not needed "
+						"REPORTED LUNS DATA CHANGED UA "
+						"%p", ua);
+					list_del(&ua->UA_list_entry);
+					mempool_free(ua, scst_ua_mempool);
+					break;
+				}
+			}
+			spin_unlock_bh(&tgt_dev->tgt_dev_lock);
+		}
+	}
+
+out_done:
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+
+out:
+	return res;
+
+out_put_err:
+	scst_put_buf(cmd, buffer);
+
+out_err:
+	scst_set_cmd_error(cmd,
+		   SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
+	goto out_compl;
+
+out_put_hw_err:
+	scst_put_buf(cmd, buffer);
+
+out_hw_err:
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+	goto out_compl;
+}
+
+static int scst_request_sense_local(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_COMPLETED, rc;
+	struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+	uint8_t *buffer;
+	int buffer_size = 0, sl = 0;
+
+	rc = scst_check_local_events(cmd);
+	if (unlikely(rc != 0))
+		goto out_done;
+
+	cmd->status = 0;
+	cmd->msg_status = 0;
+	cmd->host_status = DID_OK;
+	cmd->driver_status = 0;
+
+	spin_lock_bh(&tgt_dev->tgt_dev_lock);
+
+	if (tgt_dev->tgt_dev_valid_sense_len == 0)
+		goto out_not_completed;
+
+	TRACE(TRACE_SCSI, "%s: Returning stored sense", cmd->op_name);
+
+	buffer_size = scst_get_buf_first(cmd, &buffer);
+	if (unlikely(buffer_size == 0))
+		goto out_compl;
+	else if (unlikely(buffer_size < 0))
+		goto out_hw_err;
+
+	memset(buffer, 0, buffer_size);
+
+	if (((tgt_dev->tgt_dev_sense[0] == 0x70) ||
+	     (tgt_dev->tgt_dev_sense[0] == 0x71)) && (cmd->cdb[1] & 1)) {
+		PRINT_WARNING("%s: Fixed format of the saved sense, but "
+			"descriptor format requested. Convertion will "
+			"truncated data", cmd->op_name);
+		PRINT_BUFFER("Original sense", tgt_dev->tgt_dev_sense,
+			tgt_dev->tgt_dev_valid_sense_len);
+
+		buffer_size = min(SCST_STANDARD_SENSE_LEN, buffer_size);
+		sl = scst_set_sense(buffer, buffer_size, true,
+			tgt_dev->tgt_dev_sense[2], tgt_dev->tgt_dev_sense[12],
+			tgt_dev->tgt_dev_sense[13]);
+	} else if (((tgt_dev->tgt_dev_sense[0] == 0x72) ||
+		    (tgt_dev->tgt_dev_sense[0] == 0x73)) && !(cmd->cdb[1] & 1)) {
+		PRINT_WARNING("%s: Descriptor format of the "
+			"saved sense, but fixed format requested. Convertion "
+			"will truncated data", cmd->op_name);
+		PRINT_BUFFER("Original sense", tgt_dev->tgt_dev_sense,
+			tgt_dev->tgt_dev_valid_sense_len);
+
+		buffer_size = min(SCST_STANDARD_SENSE_LEN, buffer_size);
+		sl = scst_set_sense(buffer, buffer_size, false,
+			tgt_dev->tgt_dev_sense[1], tgt_dev->tgt_dev_sense[2],
+			tgt_dev->tgt_dev_sense[3]);
+	} else {
+		if (buffer_size >= tgt_dev->tgt_dev_valid_sense_len)
+			sl = tgt_dev->tgt_dev_valid_sense_len;
+		else {
+			sl = buffer_size;
+			PRINT_WARNING("%s: Being returned sense truncated to "
+				"size %d (needed %d)", cmd->op_name,
+				buffer_size, tgt_dev->tgt_dev_valid_sense_len);
+		}
+		memcpy(buffer, tgt_dev->tgt_dev_sense, sl);
+	}
+
+	scst_put_buf(cmd, buffer);
+
+	tgt_dev->tgt_dev_valid_sense_len = 0;
+
+	spin_unlock_bh(&tgt_dev->tgt_dev_lock);
+
+	scst_set_resp_data_len(cmd, sl);
+
+out_compl:
+	cmd->completed = 1;
+
+out_done:
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+
+out:
+	return res;
+
+out_hw_err:
+	spin_unlock_bh(&tgt_dev->tgt_dev_lock);
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+	goto out_compl;
+
+out_not_completed:
+	spin_unlock_bh(&tgt_dev->tgt_dev_lock);
+	res = SCST_EXEC_NOT_COMPLETED;
+	goto out;
+}
+
+static int scst_pre_select(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_NOT_COMPLETED;
+
+	if (scst_cmd_atomic(cmd)) {
+		res = SCST_EXEC_NEED_THREAD;
+		goto out;
+	}
+
+	scst_block_dev_cmd(cmd, 1);
+
+	/* Check for local events will be done when cmd will be executed */
+
+out:
+	return res;
+}
+
+static int scst_reserve_local(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_NOT_COMPLETED, rc;
+	struct scst_device *dev;
+	struct scst_tgt_dev *tgt_dev_tmp;
+
+	if (scst_cmd_atomic(cmd)) {
+		res = SCST_EXEC_NEED_THREAD;
+		goto out;
+	}
+
+	if ((cmd->cdb[0] == RESERVE_10) && (cmd->cdb[2] & SCST_RES_3RDPTY)) {
+		PRINT_ERROR("RESERVE_10: 3rdPty RESERVE not implemented "
+		     "(lun=%lld)", (long long unsigned int)cmd->lun);
+		scst_set_cmd_error(cmd,
+			SCST_LOAD_SENSE(scst_sense_invalid_field_in_cdb));
+		goto out_done;
+	}
+
+	dev = cmd->dev;
+
+	if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
+		scst_block_dev_cmd(cmd, 1);
+
+	rc = scst_check_local_events(cmd);
+	if (unlikely(rc != 0))
+		goto out_done;
+
+	spin_lock_bh(&dev->dev_lock);
+
+	if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
+		spin_unlock_bh(&dev->dev_lock);
+		scst_set_cmd_error_status(cmd, SAM_STAT_RESERVATION_CONFLICT);
+		goto out_done;
+	}
+
+	list_for_each_entry(tgt_dev_tmp, &dev->dev_tgt_dev_list,
+			    dev_tgt_dev_list_entry) {
+		if (cmd->tgt_dev != tgt_dev_tmp)
+			set_bit(SCST_TGT_DEV_RESERVED,
+				&tgt_dev_tmp->tgt_dev_flags);
+	}
+	dev->dev_reserved = 1;
+
+	spin_unlock_bh(&dev->dev_lock);
+
+out:
+	return res;
+
+out_done:
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+	res = SCST_EXEC_COMPLETED;
+	goto out;
+}
+
+static int scst_release_local(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_NOT_COMPLETED, rc;
+	struct scst_tgt_dev *tgt_dev_tmp;
+	struct scst_device *dev;
+
+	if (scst_cmd_atomic(cmd)) {
+		res = SCST_EXEC_NEED_THREAD;
+		goto out;
+	}
+
+	dev = cmd->dev;
+
+	if (dev->tst == SCST_CONTR_MODE_ONE_TASK_SET)
+		scst_block_dev_cmd(cmd, 1);
+
+	rc = scst_check_local_events(cmd);
+	if (unlikely(rc != 0))
+		goto out_done;
+
+	spin_lock_bh(&dev->dev_lock);
+
+	/*
+	 * The device could be RELEASED behind us, if RESERVING session
+	 * is closed (see scst_free_tgt_dev()), but this actually doesn't
+	 * matter, so use lock and no retest for DEV_RESERVED bits again
+	 */
+	if (test_bit(SCST_TGT_DEV_RESERVED, &cmd->tgt_dev->tgt_dev_flags)) {
+		res = SCST_EXEC_COMPLETED;
+		cmd->status = 0;
+		cmd->msg_status = 0;
+		cmd->host_status = DID_OK;
+		cmd->driver_status = 0;
+		cmd->completed = 1;
+	} else {
+		list_for_each_entry(tgt_dev_tmp,
+				    &dev->dev_tgt_dev_list,
+				    dev_tgt_dev_list_entry) {
+			clear_bit(SCST_TGT_DEV_RESERVED,
+				&tgt_dev_tmp->tgt_dev_flags);
+		}
+		dev->dev_reserved = 0;
+	}
+
+	spin_unlock_bh(&dev->dev_lock);
+
+	if (res == SCST_EXEC_COMPLETED)
+		goto out_done;
+
+out:
+	return res;
+
+out_done:
+	res = SCST_EXEC_COMPLETED;
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+	goto out;
+}
+
+/**
+ * scst_check_local_events() - check if there are any local SCSI events
+ *
+ * Description:
+ *    Checks if the command can be executed or there are local events,
+ *    like reservatons, pending UAs, etc. Returns < 0 if command must be
+ *    aborted, > 0 if there is an event and command should be immediately
+ *    completed, or 0 otherwise.
+ *
+ * !! Dev handlers implementing exec() callback must call this function there
+ * !! just before the actual command's execution!
+ *
+ *    On call no locks, no IRQ or IRQ-disabled context allowed.
+ */
+int scst_check_local_events(struct scst_cmd *cmd)
+{
+	int res, rc;
+	struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+	struct scst_device *dev = cmd->dev;
+
+	/*
+	 * There's no race here, because we need to trace commands sent
+	 * *after* dev_double_ua_possible flag was set.
+	 */
+	if (unlikely(dev->dev_double_ua_possible))
+		cmd->double_ua_possible = 1;
+
+	if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags))) {
+		TRACE_MGMT_DBG("ABORTED set, aborting cmd %p", cmd);
+		goto out_uncomplete;
+	}
+
+	/* Reserve check before Unit Attention */
+	if (unlikely(test_bit(SCST_TGT_DEV_RESERVED,
+			      &tgt_dev->tgt_dev_flags))) {
+		if ((cmd->op_flags & SCST_REG_RESERVE_ALLOWED) == 0) {
+			scst_set_cmd_error_status(cmd,
+				SAM_STAT_RESERVATION_CONFLICT);
+			goto out_complete;
+		}
+	}
+
+	/* If we had internal bus reset, set the command error unit attention */
+	if ((dev->scsi_dev != NULL) &&
+	    unlikely(dev->scsi_dev->was_reset)) {
+		if (scst_is_ua_command(cmd)) {
+			int done = 0;
+			/*
+			 * Prevent more than 1 cmd to be triggered by
+			 * was_reset.
+			 */
+			spin_lock_bh(&dev->dev_lock);
+			if (dev->scsi_dev->was_reset) {
+				TRACE(TRACE_MGMT, "was_reset is %d", 1);
+				scst_set_cmd_error(cmd,
+					  SCST_LOAD_SENSE(scst_sense_reset_UA));
+				/*
+				 * It looks like it is safe to clear was_reset
+				 * here.
+				 */
+				dev->scsi_dev->was_reset = 0;
+				done = 1;
+			}
+			spin_unlock_bh(&dev->dev_lock);
+
+			if (done)
+				goto out_complete;
+		}
+	}
+
+	if (unlikely(test_bit(SCST_TGT_DEV_UA_PENDING,
+			&cmd->tgt_dev->tgt_dev_flags))) {
+		if (scst_is_ua_command(cmd)) {
+			rc = scst_set_pending_UA(cmd);
+			if (rc == 0)
+				goto out_complete;
+		}
+	}
+
+	res = 0;
+
+out:
+	return res;
+
+out_complete:
+	res = 1;
+	BUG_ON(!cmd->completed);
+	goto out;
+
+out_uncomplete:
+	res = -1;
+	goto out;
+}
+EXPORT_SYMBOL_GPL(scst_check_local_events);
+
+/* No locks */
+void scst_inc_expected_sn(struct scst_tgt_dev *tgt_dev, atomic_t *slot)
+{
+	if (slot == NULL)
+		goto inc;
+
+	/* Optimized for lockless fast path */
+
+	TRACE_SN("Slot %zd, *cur_sn_slot %d", slot - tgt_dev->sn_slots,
+		atomic_read(slot));
+
+	if (!atomic_dec_and_test(slot))
+		goto out;
+
+	TRACE_SN("Slot is 0 (num_free_sn_slots=%d)",
+		tgt_dev->num_free_sn_slots);
+	if (tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1) {
+		spin_lock_irq(&tgt_dev->sn_lock);
+		if (likely(tgt_dev->num_free_sn_slots < (int)ARRAY_SIZE(tgt_dev->sn_slots)-1)) {
+			if (tgt_dev->num_free_sn_slots < 0)
+				tgt_dev->cur_sn_slot = slot;
+			/*
+			 * To be in-sync with SIMPLE case in scst_cmd_set_sn()
+			 */
+			smp_mb();
+			tgt_dev->num_free_sn_slots++;
+			TRACE_SN("Incremented num_free_sn_slots (%d)",
+				tgt_dev->num_free_sn_slots);
+
+		}
+		spin_unlock_irq(&tgt_dev->sn_lock);
+	}
+
+inc:
+	/*
+	 * No protection of expected_sn is needed, because only one thread
+	 * at time can be here (serialized by sn). Also it is supposed that
+	 * there could not be half-incremented halves.
+	 */
+	tgt_dev->expected_sn++;
+	/*
+	 * Write must be before def_cmd_count read to be in sync. with
+	 * scst_post_exec_sn(). See comment in scst_send_for_exec().
+	 */
+	smp_mb();
+	TRACE_SN("Next expected_sn: %d", tgt_dev->expected_sn);
+
+out:
+	return;
+}
+
+/* No locks */
+static struct scst_cmd *scst_post_exec_sn(struct scst_cmd *cmd,
+	bool make_active)
+{
+	/* For HQ commands SN is not set */
+	bool inc_expected_sn = !cmd->inc_expected_sn_on_done &&
+			       cmd->sn_set && !cmd->retry;
+	struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+	struct scst_cmd *res;
+
+	if (inc_expected_sn)
+		scst_inc_expected_sn(tgt_dev, cmd->sn_slot);
+
+	if (make_active) {
+		scst_make_deferred_commands_active(tgt_dev);
+		res = NULL;
+	} else
+		res = scst_check_deferred_commands(tgt_dev);
+	return res;
+}
+
+/* cmd must be additionally referenced to not die inside */
+static int scst_do_real_exec(struct scst_cmd *cmd)
+{
+	int res = SCST_EXEC_NOT_COMPLETED;
+	int rc;
+	bool atomic = scst_cmd_atomic(cmd);
+	struct scst_device *dev = cmd->dev;
+	struct scst_dev_type *handler = dev->handler;
+	struct io_context *old_ctx = NULL;
+	bool ctx_changed = false;
+
+	if (!atomic)
+		ctx_changed = scst_set_io_context(cmd, &old_ctx);
+
+	cmd->state = SCST_CMD_STATE_REAL_EXECUTING;
+
+	if (handler->exec) {
+		if (unlikely(!dev->handler->exec_atomic && atomic)) {
+			/*
+			 * It shouldn't be because of the SCST_TGT_DEV_AFTER_*
+			 * optimization.
+			 */
+			TRACE_DBG("Dev handler %s exec() needs thread "
+				"context, rescheduling", dev->handler->name);
+			res = SCST_EXEC_NEED_THREAD;
+			goto out_restore;
+		}
+
+		TRACE_DBG("Calling dev handler %s exec(%p)",
+		      handler->name, cmd);
+		TRACE_BUFF_FLAG(TRACE_SND_TOP, "Execing: ", cmd->cdb,
+			cmd->cdb_len);
+		scst_set_cur_start(cmd);
+		res = handler->exec(cmd);
+		TRACE_DBG("Dev handler %s exec() returned %d",
+		      handler->name, res);
+
+		if (res == SCST_EXEC_COMPLETED)
+			goto out_complete;
+		else if (res == SCST_EXEC_NEED_THREAD)
+			goto out_restore;
+
+		scst_set_exec_time(cmd);
+
+		BUG_ON(res != SCST_EXEC_NOT_COMPLETED);
+	}
+
+	TRACE_DBG("Sending cmd %p to SCSI mid-level", cmd);
+
+	if (unlikely(dev->scsi_dev == NULL)) {
+		PRINT_ERROR("Command for virtual device must be "
+			"processed by device handler (LUN %lld)!",
+			(long long unsigned int)cmd->lun);
+		goto out_error;
+	}
+
+	res = scst_check_local_events(cmd);
+	if (unlikely(res != 0))
+		goto out_done;
+
+#ifndef CONFIG_SCST_ALLOW_PASSTHROUGH_IO_SUBMIT_IN_SIRQ
+	if (unlikely(atomic)) {
+		TRACE_DBG("Pass-through exec() can not be called in atomic "
+			"context, rescheduling to the thread (handler %s)",
+			handler->name);
+		res = SCST_EXEC_NEED_THREAD;
+		goto out_restore;
+	}
+#endif
+
+	scst_set_cur_start(cmd);
+
+	rc = scst_scsi_exec_async(cmd, scst_cmd_done);
+	if (unlikely(rc != 0)) {
+		if (atomic) {
+			res = SCST_EXEC_NEED_THREAD;
+			goto out_restore;
+		} else {
+			PRINT_ERROR("scst pass-through exec failed: %x", rc);
+			goto out_error;
+		}
+	}
+
+out_complete:
+	res = SCST_EXEC_COMPLETED;
+
+out_reset_ctx:
+	if (ctx_changed)
+		scst_reset_io_context(cmd->tgt_dev, old_ctx);
+	return res;
+
+out_restore:
+	scst_set_exec_time(cmd);
+	/* Restore the state */
+	cmd->state = SCST_CMD_STATE_REAL_EXEC;
+	goto out_reset_ctx;
+
+out_error:
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+	goto out_done;
+
+out_done:
+	res = SCST_EXEC_COMPLETED;
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+	goto out_complete;
+}
+
+static inline int scst_real_exec(struct scst_cmd *cmd)
+{
+	int res;
+
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
+
+	__scst_cmd_get(cmd);
+
+	res = scst_do_real_exec(cmd);
+
+	if (likely(res == SCST_EXEC_COMPLETED)) {
+		scst_post_exec_sn(cmd, true);
+		if (cmd->dev->scsi_dev != NULL)
+			generic_unplug_device(
+				cmd->dev->scsi_dev->request_queue);
+	} else
+		BUG_ON(res != SCST_EXEC_NEED_THREAD);
+
+	__scst_cmd_put(cmd);
+
+	/* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
+	return res;
+}
+
+static int scst_do_local_exec(struct scst_cmd *cmd)
+{
+	int res;
+	struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+
+	/* Check READ_ONLY device status */
+	if ((cmd->op_flags & SCST_WRITE_MEDIUM) &&
+	    (tgt_dev->acg_dev->rd_only || cmd->dev->swp ||
+	     cmd->dev->rd_only)) {
+		PRINT_WARNING("Attempt of write access to read-only device: "
+			"initiator %s, LUN %lld, op %x",
+			cmd->sess->initiator_name, cmd->lun, cmd->cdb[0]);
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_data_protect));
+		goto out_done;
+	}
+
+	if (!scst_is_cmd_local(cmd)) {
+		res = SCST_EXEC_NOT_COMPLETED;
+		goto out;
+	}
+
+	switch (cmd->cdb[0]) {
+	case MODE_SELECT:
+	case MODE_SELECT_10:
+	case LOG_SELECT:
+		res = scst_pre_select(cmd);
+		break;
+	case RESERVE:
+	case RESERVE_10:
+		res = scst_reserve_local(cmd);
+		break;
+	case RELEASE:
+	case RELEASE_10:
+		res = scst_release_local(cmd);
+		break;
+	case REPORT_LUNS:
+		res = scst_report_luns_local(cmd);
+		break;
+	case REQUEST_SENSE:
+		res = scst_request_sense_local(cmd);
+		break;
+	default:
+		res = SCST_EXEC_NOT_COMPLETED;
+		break;
+	}
+
+out:
+	return res;
+
+out_done:
+	/* Report the result */
+	cmd->scst_cmd_done(cmd, SCST_CMD_STATE_DEFAULT, SCST_CONTEXT_SAME);
+	res = SCST_EXEC_COMPLETED;
+	goto out;
+}
+
+static int scst_local_exec(struct scst_cmd *cmd)
+{
+	int res;
+
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_SAME != SCST_EXEC_NOT_COMPLETED);
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_CONT_NEXT != SCST_EXEC_COMPLETED);
+	BUILD_BUG_ON(SCST_CMD_STATE_RES_NEED_THREAD != SCST_EXEC_NEED_THREAD);
+
+	__scst_cmd_get(cmd);
+
+	res = scst_do_local_exec(cmd);
+	if (likely(res == SCST_EXEC_NOT_COMPLETED))
+		cmd->state = SCST_CMD_STATE_REAL_EXEC;
+	else if (res == SCST_EXEC_COMPLETED)
+		scst_post_exec_sn(cmd, true);
+	else
+		BUG_ON(res != SCST_EXEC_NEED_THREAD);
+
+	__scst_cmd_put(cmd);
+
+	/* SCST_EXEC_* match SCST_CMD_STATE_RES_* */
+	return res;
+}
+
+static int scst_exec(struct scst_cmd **active_cmd)
+{
+	struct scst_cmd *cmd = *active_cmd;
+	struct scst_cmd *ref_cmd;
+	struct scst_device *dev = cmd->dev;
+	int res = SCST_CMD_STATE_RES_CONT_NEXT, count;
+
+	if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
+		goto out;
+
+	/* To protect tgt_dev */
+	ref_cmd = cmd;
+	__scst_cmd_get(ref_cmd);
+
+	count = 0;
+	while (1) {
+		int rc;
+
+		cmd->sent_for_exec = 1;
+		/*
+		 * To sync with scst_abort_cmd(). The above assignment must
+		 * be before SCST_CMD_ABORTED test, done later in
+		 * scst_check_local_events(). It's far from here, so the order
+		 * is virtually guaranteed, but let's have it just in case.
+		 */
+		smp_mb();
+
+		cmd->scst_cmd_done = scst_cmd_done_local;
+		cmd->state = SCST_CMD_STATE_LOCAL_EXEC;
+
+		rc = scst_do_local_exec(cmd);
+		if (likely(rc == SCST_EXEC_NOT_COMPLETED))
+			/* Nothing to do */;
+		else if (rc == SCST_EXEC_NEED_THREAD) {
+			TRACE_DBG("%s", "scst_do_local_exec() requested "
+				"thread context, rescheduling");
+			scst_dec_on_dev_cmd(cmd);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			break;
+		} else {
+			BUG_ON(rc != SCST_EXEC_COMPLETED);
+			goto done;
+		}
+
+		cmd->state = SCST_CMD_STATE_REAL_EXEC;
+
+		rc = scst_do_real_exec(cmd);
+		if (likely(rc == SCST_EXEC_COMPLETED))
+			/* Nothing to do */;
+		else if (rc == SCST_EXEC_NEED_THREAD) {
+			TRACE_DBG("scst_real_exec() requested thread "
+				"context, rescheduling (cmd %p)", cmd);
+			scst_dec_on_dev_cmd(cmd);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			break;
+		} else
+			BUG();
+
+done:
+		count++;
+
+		cmd = scst_post_exec_sn(cmd, false);
+		if (cmd == NULL)
+			break;
+
+		if (unlikely(scst_inc_on_dev_cmd(cmd) != 0))
+			break;
+
+		__scst_cmd_put(ref_cmd);
+		ref_cmd = cmd;
+		__scst_cmd_get(ref_cmd);
+	}
+
+	*active_cmd = cmd;
+
+	if (count == 0)
+		goto out_put;
+
+	if (dev->scsi_dev != NULL)
+		generic_unplug_device(dev->scsi_dev->request_queue);
+
+out_put:
+	__scst_cmd_put(ref_cmd);
+	/* !! At this point sess, dev and tgt_dev can be already freed !! */
+
+out:
+	return res;
+}
+
+static int scst_send_for_exec(struct scst_cmd **active_cmd)
+{
+	int res;
+	struct scst_cmd *cmd = *active_cmd;
+	struct scst_tgt_dev *tgt_dev = cmd->tgt_dev;
+	typeof(tgt_dev->expected_sn) expected_sn;
+
+	if (unlikely(cmd->internal))
+		goto exec;
+
+	if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
+		goto exec;
+
+	BUG_ON(!cmd->sn_set);
+
+	expected_sn = tgt_dev->expected_sn;
+	/* Optimized for lockless fast path */
+	if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
+		spin_lock_irq(&tgt_dev->sn_lock);
+
+		tgt_dev->def_cmd_count++;
+		/*
+		 * Memory barrier is needed here to implement lockless fast
+		 * path. We need the exact order of read and write between
+		 * def_cmd_count and expected_sn. Otherwise, we can miss case,
+		 * when expected_sn was changed to be equal to cmd->sn while
+		 * we are queuing cmd the deferred list after the expected_sn
+		 * below. It will lead to a forever stuck command. But with
+		 * the barrier in such case __scst_check_deferred_commands()
+		 * will be called and it will take sn_lock, so we will be
+		 * synchronized.
+		 */
+		smp_mb();
+
+		expected_sn = tgt_dev->expected_sn;
+		if ((cmd->sn != expected_sn) || (tgt_dev->hq_cmd_count > 0)) {
+			if (unlikely(test_bit(SCST_CMD_ABORTED,
+					      &cmd->cmd_flags))) {
+				/* Necessary to allow aborting out of sn cmds */
+				TRACE_MGMT_DBG("Aborting out of sn cmd %p "
+					"(tag %llu, sn %u)", cmd,
+					(long long unsigned)cmd->tag, cmd->sn);
+				tgt_dev->def_cmd_count--;
+				scst_set_cmd_abnormal_done_state(cmd);
+				res = SCST_CMD_STATE_RES_CONT_SAME;
+			} else {
+				TRACE_SN("Deferring cmd %p (sn=%d, set %d, "
+					"expected_sn=%d)", cmd, cmd->sn,
+					cmd->sn_set, expected_sn);
+				list_add_tail(&cmd->sn_cmd_list_entry,
+					      &tgt_dev->deferred_cmd_list);
+				res = SCST_CMD_STATE_RES_CONT_NEXT;
+			}
+			spin_unlock_irq(&tgt_dev->sn_lock);
+			goto out;
+		} else {
+			TRACE_SN("Somebody incremented expected_sn %d, "
+				"continuing", expected_sn);
+			tgt_dev->def_cmd_count--;
+			spin_unlock_irq(&tgt_dev->sn_lock);
+		}
+	}
+
+exec:
+	res = scst_exec(active_cmd);
+
+out:
+	return res;
+}
+
+/* No locks supposed to be held */
+static int scst_check_sense(struct scst_cmd *cmd)
+{
+	int res = 0;
+	struct scst_device *dev = cmd->dev;
+
+	if (unlikely(cmd->ua_ignore))
+		goto out;
+
+	/* If we had internal bus reset behind us, set the command error UA */
+	if ((dev->scsi_dev != NULL) &&
+	    unlikely(cmd->host_status == DID_RESET) &&
+	    scst_is_ua_command(cmd)) {
+		TRACE(TRACE_MGMT, "DID_RESET: was_reset=%d host_status=%x",
+		      dev->scsi_dev->was_reset, cmd->host_status);
+		scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_reset_UA));
+		/* It looks like it is safe to clear was_reset here */
+		dev->scsi_dev->was_reset = 0;
+	}
+
+	if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
+	    SCST_SENSE_VALID(cmd->sense)) {
+		PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
+			cmd->sense_valid_len);
+
+		/* Check Unit Attention Sense Key */
+		if (scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) {
+			if (scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASC_VALID,
+					0, SCST_SENSE_ASC_UA_RESET, 0)) {
+				if (cmd->double_ua_possible) {
+					TRACE_MGMT_DBG("Double UA "
+						"detected for device %p", dev);
+					TRACE_MGMT_DBG("Retrying cmd"
+						" %p (tag %llu)", cmd,
+						(long long unsigned)cmd->tag);
+
+					cmd->status = 0;
+					cmd->msg_status = 0;
+					cmd->host_status = DID_OK;
+					cmd->driver_status = 0;
+					cmd->completed = 0;
+
+					mempool_free(cmd->sense,
+						     scst_sense_mempool);
+					cmd->sense = NULL;
+
+					scst_check_restore_sg_buff(cmd);
+
+					BUG_ON(cmd->dbl_ua_orig_resp_data_len < 0);
+					cmd->data_direction =
+						cmd->dbl_ua_orig_data_direction;
+					cmd->resp_data_len =
+						cmd->dbl_ua_orig_resp_data_len;
+
+					cmd->state = SCST_CMD_STATE_REAL_EXEC;
+					cmd->retry = 1;
+					res = 1;
+					goto out;
+				}
+			}
+			scst_dev_check_set_UA(dev, cmd,	cmd->sense,
+				cmd->sense_valid_len);
+		}
+	}
+
+	if (unlikely(cmd->double_ua_possible)) {
+		if (scst_is_ua_command(cmd)) {
+			TRACE_MGMT_DBG("Clearing dbl_ua_possible flag (dev %p, "
+				"cmd %p)", dev, cmd);
+			/*
+			 * Lock used to protect other flags in the bitfield
+			 * (just in case, actually). Those flags can't be
+			 * changed in parallel, because the device is
+			 * serialized.
+			 */
+			spin_lock_bh(&dev->dev_lock);
+			dev->dev_double_ua_possible = 0;
+			spin_unlock_bh(&dev->dev_lock);
+		}
+	}
+
+out:
+	return res;
+}
+
+static int scst_check_auto_sense(struct scst_cmd *cmd)
+{
+	int res = 0;
+
+	if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION) &&
+	    (!SCST_SENSE_VALID(cmd->sense) ||
+	     SCST_NO_SENSE(cmd->sense))) {
+		TRACE(TRACE_SCSI|TRACE_MINOR_AND_MGMT_DBG, "CHECK_CONDITION, "
+			"but no sense: cmd->status=%x, cmd->msg_status=%x, "
+		      "cmd->host_status=%x, cmd->driver_status=%x (cmd %p)",
+		      cmd->status, cmd->msg_status, cmd->host_status,
+		      cmd->driver_status, cmd);
+		res = 1;
+	} else if (unlikely(cmd->host_status)) {
+		if ((cmd->host_status == DID_REQUEUE) ||
+		    (cmd->host_status == DID_IMM_RETRY) ||
+		    (cmd->host_status == DID_SOFT_ERROR) ||
+		    (cmd->host_status == DID_ABORT)) {
+			scst_set_busy(cmd);
+		} else {
+			TRACE(TRACE_SCSI|TRACE_MINOR_AND_MGMT_DBG, "Host "
+				"status %x received, returning HARDWARE ERROR "
+				"instead (cmd %p)", cmd->host_status, cmd);
+			scst_set_cmd_error(cmd,
+				SCST_LOAD_SENSE(scst_sense_hardw_error));
+		}
+	}
+	return res;
+}
+
+static int scst_pre_dev_done(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME, rc;
+
+	if (unlikely(scst_check_auto_sense(cmd))) {
+		PRINT_INFO("Command finished with CHECK CONDITION, but "
+			    "without sense data (opcode 0x%x), issuing "
+			    "REQUEST SENSE", cmd->cdb[0]);
+		rc = scst_prepare_request_sense(cmd);
+		if (rc == 0)
+			res = SCST_CMD_STATE_RES_CONT_NEXT;
+		else {
+			PRINT_ERROR("%s", "Unable to issue REQUEST SENSE, "
+				    "returning HARDWARE ERROR");
+			scst_set_cmd_error(cmd,
+				SCST_LOAD_SENSE(scst_sense_hardw_error));
+		}
+		goto out;
+	} else if (unlikely(scst_check_sense(cmd)))
+		goto out;
+
+	if (likely(scsi_status_is_good(cmd->status))) {
+		unsigned char type = cmd->dev->type;
+		if (unlikely((cmd->cdb[0] == MODE_SENSE ||
+			      cmd->cdb[0] == MODE_SENSE_10)) &&
+		    (cmd->tgt_dev->acg_dev->rd_only || cmd->dev->swp ||
+		     cmd->dev->rd_only) &&
+		    (type == TYPE_DISK ||
+		     type == TYPE_WORM ||
+		     type == TYPE_MOD ||
+		     type == TYPE_TAPE)) {
+			int32_t length;
+			uint8_t *address;
+			bool err = false;
+
+			length = scst_get_buf_first(cmd, &address);
+			if (length < 0) {
+				PRINT_ERROR("%s", "Unable to get "
+					"MODE_SENSE buffer");
+				scst_set_cmd_error(cmd,
+					SCST_LOAD_SENSE(
+						scst_sense_hardw_error));
+				err = true;
+			} else if (length > 2 && cmd->cdb[0] == MODE_SENSE)
+				address[2] |= 0x80;   /* Write Protect*/
+			else if (length > 3 && cmd->cdb[0] == MODE_SENSE_10)
+				address[3] |= 0x80;   /* Write Protect*/
+			scst_put_buf(cmd, address);
+
+			if (err)
+				goto out;
+		}
+
+		/*
+		 * Check and clear NormACA option for the device, if necessary,
+		 * since we don't support ACA
+		 */
+		if (unlikely((cmd->cdb[0] == INQUIRY)) &&
+		    /* Std INQUIRY data (no EVPD) */
+		    !(cmd->cdb[1] & SCST_INQ_EVPD) &&
+		    (cmd->resp_data_len > SCST_INQ_BYTE3)) {
+			uint8_t *buffer;
+			int buflen;
+			bool err = false;
+
+			/* ToDo: all pages ?? */
+			buflen = scst_get_buf_first(cmd, &buffer);
+			if (buflen > SCST_INQ_BYTE3) {
+#ifdef CONFIG_SCST_EXTRACHECKS
+				if (buffer[SCST_INQ_BYTE3] & SCST_INQ_NORMACA_BIT) {
+					PRINT_INFO("NormACA set for device: "
+					    "lun=%lld, type 0x%02x. Clear it, "
+					    "since it's unsupported.",
+					    (long long unsigned int)cmd->lun,
+					    buffer[0]);
+				}
+#endif
+				buffer[SCST_INQ_BYTE3] &= ~SCST_INQ_NORMACA_BIT;
+			} else if (buflen != 0) {
+				PRINT_ERROR("%s", "Unable to get INQUIRY "
+				    "buffer");
+				scst_set_cmd_error(cmd,
+				       SCST_LOAD_SENSE(scst_sense_hardw_error));
+				err = true;
+			}
+			if (buflen > 0)
+				scst_put_buf(cmd, buffer);
+
+			if (err)
+				goto out;
+		}
+
+		if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
+		    (cmd->cdb[0] == MODE_SELECT_10) ||
+		    (cmd->cdb[0] == LOG_SELECT))) {
+			TRACE(TRACE_SCSI,
+				"MODE/LOG SELECT succeeded (LUN %lld)",
+				(long long unsigned int)cmd->lun);
+			cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
+			goto out;
+		}
+	} else {
+		TRACE(TRACE_SCSI, "cmd %p not succeeded with status %x",
+			cmd, cmd->status);
+
+		if ((cmd->cdb[0] == RESERVE) || (cmd->cdb[0] == RESERVE_10)) {
+			if (!test_bit(SCST_TGT_DEV_RESERVED,
+					&cmd->tgt_dev->tgt_dev_flags)) {
+				struct scst_tgt_dev *tgt_dev_tmp;
+				struct scst_device *dev = cmd->dev;
+
+				TRACE(TRACE_SCSI, "RESERVE failed lun=%lld, "
+					"status=%x",
+					(long long unsigned int)cmd->lun,
+					cmd->status);
+				PRINT_BUFF_FLAG(TRACE_SCSI, "Sense", cmd->sense,
+					cmd->sense_valid_len);
+
+				/* Clearing the reservation */
+				spin_lock_bh(&dev->dev_lock);
+				list_for_each_entry(tgt_dev_tmp,
+						    &dev->dev_tgt_dev_list,
+						    dev_tgt_dev_list_entry) {
+					clear_bit(SCST_TGT_DEV_RESERVED,
+						&tgt_dev_tmp->tgt_dev_flags);
+				}
+				dev->dev_reserved = 0;
+				spin_unlock_bh(&dev->dev_lock);
+			}
+		}
+
+		/* Check for MODE PARAMETERS CHANGED UA */
+		if ((cmd->dev->scsi_dev != NULL) &&
+		    (cmd->status == SAM_STAT_CHECK_CONDITION) &&
+		    scst_is_ua_sense(cmd->sense, cmd->sense_valid_len) &&
+		    scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASCx_VALID,
+					0, 0x2a, 0x01)) {
+			TRACE(TRACE_SCSI, "MODE PARAMETERS CHANGED UA (lun "
+				"%lld)", (long long unsigned int)cmd->lun);
+			cmd->state = SCST_CMD_STATE_MODE_SELECT_CHECKS;
+			goto out;
+		}
+	}
+
+	cmd->state = SCST_CMD_STATE_DEV_DONE;
+
+out:
+	return res;
+}
+
+static int scst_mode_select_checks(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME;
+	int atomic = scst_cmd_atomic(cmd);
+
+	if (likely(scsi_status_is_good(cmd->status))) {
+		if (unlikely((cmd->cdb[0] == MODE_SELECT) ||
+		    (cmd->cdb[0] == MODE_SELECT_10) ||
+		    (cmd->cdb[0] == LOG_SELECT))) {
+			struct scst_device *dev = cmd->dev;
+			int sl;
+			uint8_t sense_buffer[SCST_STANDARD_SENSE_LEN];
+
+			if (atomic && (dev->scsi_dev != NULL)) {
+				TRACE_DBG("%s", "MODE/LOG SELECT: thread "
+					"context required");
+				res = SCST_CMD_STATE_RES_NEED_THREAD;
+				goto out;
+			}
+
+			TRACE(TRACE_SCSI, "MODE/LOG SELECT succeeded, "
+				"setting the SELECT UA (lun=%lld)",
+				(long long unsigned int)cmd->lun);
+
+			spin_lock_bh(&dev->dev_lock);
+			if (cmd->cdb[0] == LOG_SELECT) {
+				sl = scst_set_sense(sense_buffer,
+					sizeof(sense_buffer),
+					dev->d_sense,
+					UNIT_ATTENTION, 0x2a, 0x02);
+			} else {
+				sl = scst_set_sense(sense_buffer,
+					sizeof(sense_buffer),
+					dev->d_sense,
+					UNIT_ATTENTION, 0x2a, 0x01);
+			}
+			scst_dev_check_set_local_UA(dev, cmd, sense_buffer, sl);
+			spin_unlock_bh(&dev->dev_lock);
+
+			if (dev->scsi_dev != NULL)
+				scst_obtain_device_parameters(dev);
+		}
+	} else if ((cmd->status == SAM_STAT_CHECK_CONDITION) &&
+		    scst_is_ua_sense(cmd->sense, cmd->sense_valid_len) &&
+		     /* mode parameters changed */
+		    (scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASCx_VALID,
+					0, 0x2a, 0x01) ||
+		     scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASC_VALID,
+					0, 0x29, 0) /* reset */ ||
+		     scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASC_VALID,
+					0, 0x28, 0) /* medium changed */ ||
+		     /* cleared by another ini (just in case) */
+		     scst_analyze_sense(cmd->sense, cmd->sense_valid_len,
+					SCST_SENSE_ASC_VALID,
+					0, 0x2F, 0))) {
+		if (atomic) {
+			TRACE_DBG("Possible parameters changed UA %x: "
+				"thread context required", cmd->sense[12]);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			goto out;
+		}
+
+		TRACE(TRACE_SCSI, "Possible parameters changed UA %x "
+			"(LUN %lld): getting new parameters", cmd->sense[12],
+			(long long unsigned int)cmd->lun);
+
+		scst_obtain_device_parameters(cmd->dev);
+	} else
+		BUG();
+
+	cmd->state = SCST_CMD_STATE_DEV_DONE;
+
+out:
+	return res;
+}
+
+static void scst_inc_check_expected_sn(struct scst_cmd *cmd)
+{
+	if (likely(cmd->sn_set))
+		scst_inc_expected_sn(cmd->tgt_dev, cmd->sn_slot);
+
+	scst_make_deferred_commands_active(cmd->tgt_dev);
+}
+
+static int scst_dev_done(struct scst_cmd *cmd)
+{
+	int res = SCST_CMD_STATE_RES_CONT_SAME;
+	int state;
+	struct scst_device *dev = cmd->dev;
+
+	state = SCST_CMD_STATE_PRE_XMIT_RESP;
+
+	if (likely(!scst_is_cmd_fully_local(cmd)) &&
+	    likely(dev->handler->dev_done != NULL)) {
+		int rc;
+
+		if (unlikely(!dev->handler->dev_done_atomic &&
+			     scst_cmd_atomic(cmd))) {
+			/*
+			 * It shouldn't be because of the SCST_TGT_DEV_AFTER_*
+			 * optimization.
+			 */
+			TRACE_DBG("Dev handler %s dev_done() needs thread "
+			      "context, rescheduling", dev->handler->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			goto out;
+		}
+
+		TRACE_DBG("Calling dev handler %s dev_done(%p)",
+			dev->handler->name, cmd);
+		scst_set_cur_start(cmd);
+		rc = dev->handler->dev_done(cmd);
+		scst_set_dev_done_time(cmd);
+		TRACE_DBG("Dev handler %s dev_done() returned %d",
+		      dev->handler->name, rc);
+		if (rc != SCST_CMD_STATE_DEFAULT)
+			state = rc;
+	}
+
+	switch (state) {
+#ifdef CONFIG_SCST_EXTRACHECKS
+	case SCST_CMD_STATE_PRE_XMIT_RESP:
+	case SCST_CMD_STATE_DEV_PARSE:
+	case SCST_CMD_STATE_PRE_PARSE:
+	case SCST_CMD_STATE_PREPARE_SPACE:
+	case SCST_CMD_STATE_RDY_TO_XFER:
+	case SCST_CMD_STATE_TGT_PRE_EXEC:
+	case SCST_CMD_STATE_SEND_FOR_EXEC:
+	case SCST_CMD_STATE_LOCAL_EXEC:
+	case SCST_CMD_STATE_REAL_EXEC:
+	case SCST_CMD_STATE_PRE_DEV_DONE:
+	case SCST_CMD_STATE_MODE_SELECT_CHECKS:
+	case SCST_CMD_STATE_DEV_DONE:
+	case SCST_CMD_STATE_XMIT_RESP:
+	case SCST_CMD_STATE_FINISHED:
+	case SCST_CMD_STATE_FINISHED_INTERNAL:
+#else
+	default:
+#endif
+		cmd->state = state;
+		break;
+	case SCST_CMD_STATE_NEED_THREAD_CTX:
+		TRACE_DBG("Dev handler %s dev_done() requested "
+		      "thread context, rescheduling",
+		      dev->handler->name);
+		res = SCST_CMD_STATE_RES_NEED_THREAD;
+		break;
+#ifdef CONFIG_SCST_EXTRACHECKS
+	default:
+		if (state >= 0) {
+			PRINT_ERROR("Dev handler %s dev_done() returned "
+				"invalid cmd state %d",
+				dev->handler->name, state);
+		} else {
+			PRINT_ERROR("Dev handler %s dev_done() returned "
+				"error %d", dev->handler->name,
+				state);
+		}
+		scst_set_cmd_error(cmd,
+			   SCST_LOAD_SENSE(scst_sense_hardw_error));
+		scst_set_cmd_abnormal_done_state(cmd);
+		break;
+#endif
+	}
+
+	if (cmd->needs_unblocking)
+		scst_unblock_dev_cmd(cmd);
+
+	if (likely(cmd->dec_on_dev_needed))
+		scst_dec_on_dev_cmd(cmd);
+
+	if (cmd->inc_expected_sn_on_done && cmd->sent_for_exec)
+		scst_inc_check_expected_sn(cmd);
+
+	if (unlikely(cmd->internal))
+		cmd->state = SCST_CMD_STATE_FINISHED_INTERNAL;
+
+out:
+	return res;
+}
+
+static int scst_pre_xmit_response(struct scst_cmd *cmd)
+{
+	int res;
+
+	EXTRACHECKS_BUG_ON(cmd->internal);
+
+#ifdef CONFIG_SCST_DEBUG_TM
+	if (cmd->tm_dbg_delayed &&
+			!test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)) {
+		if (scst_cmd_atomic(cmd)) {
+			TRACE_MGMT_DBG("%s",
+				"DEBUG_TM delayed cmd needs a thread");
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			return res;
+		}
+		TRACE_MGMT_DBG("Delaying cmd %p (tag %llu) for 1 second",
+			cmd, cmd->tag);
+		schedule_timeout_uninterruptible(HZ);
+	}
+#endif
+
+	if (likely(cmd->tgt_dev != NULL)) {
+		/*
+		 * Those counters protect from not getting too long processing
+		 * latency, so we should decrement them after cmd completed.
+		 */
+		atomic_dec(&cmd->tgt_dev->tgt_dev_cmd_count);
+#ifdef CONFIG_SCST_PER_DEVICE_CMD_COUNT_LIMIT
+		atomic_dec(&cmd->dev->dev_cmd_count);
+#endif
+#ifdef CONFIG_SCST_ORDERED_READS
+		/* If expected values not set, expected direction is UNKNOWN */
+		if (cmd->expected_data_direction & SCST_DATA_WRITE)
+			atomic_dec(&cmd->dev->write_cmd_count);
+#endif
+		if (unlikely(cmd->queue_type == SCST_CMD_QUEUE_HEAD_OF_QUEUE))
+			scst_on_hq_cmd_response(cmd);
+
+		if (unlikely(!cmd->sent_for_exec)) {
+			TRACE_SN("cmd %p was not sent to mid-lev"
+				" (sn %d, set %d)",
+				cmd, cmd->sn, cmd->sn_set);
+			scst_unblock_deferred(cmd->tgt_dev, cmd);
+			cmd->sent_for_exec = 1;
+		}
+	}
+
+	cmd->done = 1;
+	smp_mb(); /* to sync with scst_abort_cmd() */
+
+	if (unlikely(test_bit(SCST_CMD_ABORTED, &cmd->cmd_flags)))
+		scst_xmit_process_aborted_cmd(cmd);
+	else if (unlikely(cmd->status == SAM_STAT_CHECK_CONDITION))
+		scst_store_sense(cmd);
+
+	if (unlikely(test_bit(SCST_CMD_NO_RESP, &cmd->cmd_flags))) {
+		TRACE_MGMT_DBG("Flag NO_RESP set for cmd %p (tag %llu),"
+				" skipping",
+				cmd, (long long unsigned int)cmd->tag);
+		cmd->state = SCST_CMD_STATE_FINISHED;
+		res = SCST_CMD_STATE_RES_CONT_SAME;
+		goto out;
+	}
+
+	cmd->state = SCST_CMD_STATE_XMIT_RESP;
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+
+out:
+	return res;
+}
+
+static int scst_xmit_response(struct scst_cmd *cmd)
+{
+	struct scst_tgt_template *tgtt = cmd->tgtt;
+	int res, rc;
+
+	EXTRACHECKS_BUG_ON(cmd->internal);
+
+	if (unlikely(!tgtt->xmit_response_atomic &&
+		     scst_cmd_atomic(cmd))) {
+		/*
+		 * It shouldn't be because of the SCST_TGT_DEV_AFTER_*
+		 * optimization.
+		 */
+		TRACE_DBG("Target driver %s xmit_response() needs thread "
+			      "context, rescheduling", tgtt->name);
+		res = SCST_CMD_STATE_RES_NEED_THREAD;
+		goto out;
+	}
+
+	while (1) {
+		int finished_cmds = atomic_read(&cmd->tgt->finished_cmds);
+
+		res = SCST_CMD_STATE_RES_CONT_NEXT;
+		cmd->state = SCST_CMD_STATE_XMIT_WAIT;
+
+		TRACE_DBG("Calling xmit_response(%p)", cmd);
+
+#if defined(CONFIG_SCST_DEBUG) || defined(CONFIG_SCST_TRACING)
+		if (trace_flag & TRACE_SND_BOT) {
+			int i;
+			struct scatterlist *sg;
+			if (cmd->tgt_sg != NULL)
+				sg = cmd->tgt_sg;
+			else
+				sg = cmd->sg;
+			if (sg != NULL) {
+				TRACE(TRACE_SND_BOT, "Xmitting data for cmd %p "
+					"(sg_cnt %d, sg %p, sg[0].page %p)",
+					cmd, cmd->tgt_sg_cnt, sg,
+					(void *)sg_page(&sg[0]));
+				for (i = 0; i < cmd->tgt_sg_cnt; ++i) {
+					PRINT_BUFF_FLAG(TRACE_SND_BOT,
+						"Xmitting sg", sg_virt(&sg[i]),
+						sg[i].length);
+				}
+			}
+		}
+#endif
+
+		if (tgtt->on_hw_pending_cmd_timeout != NULL) {
+			struct scst_session *sess = cmd->sess;
+			cmd->hw_pending_start = jiffies;
+			cmd->cmd_hw_pending = 1;
+			if (!test_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED, &sess->sess_aflags)) {
+				TRACE_DBG("Sched HW pending work for sess %p "
+					"(max time %d)", sess,
+					tgtt->max_hw_pending_time);
+				set_bit(SCST_SESS_HW_PENDING_WORK_SCHEDULED,
+					&sess->sess_aflags);
+				schedule_delayed_work(&sess->hw_pending_work,
+					tgtt->max_hw_pending_time * HZ);
+			}
+		}
+
+		scst_set_cur_start(cmd);
+
+#ifdef CONFIG_SCST_DEBUG_RETRY
+		if (((scst_random() % 100) == 77))
+			rc = SCST_TGT_RES_QUEUE_FULL;
+		else
+#endif
+			rc = tgtt->xmit_response(cmd);
+		TRACE_DBG("xmit_response() returned %d", rc);
+
+		if (likely(rc == SCST_TGT_RES_SUCCESS))
+			goto out;
+
+		scst_set_xmit_time(cmd);
+
+		cmd->cmd_hw_pending = 0;
+
+		/* Restore the previous state */
+		cmd->state = SCST_CMD_STATE_XMIT_RESP;
+
+		switch (rc) {
+		case SCST_TGT_RES_QUEUE_FULL:
+			if (scst_queue_retry_cmd(cmd, finished_cmds) == 0)
+				break;
+			else
+				continue;
+
+		case SCST_TGT_RES_NEED_THREAD_CTX:
+			TRACE_DBG("Target driver %s xmit_response() "
+			      "requested thread context, rescheduling",
+			      tgtt->name);
+			res = SCST_CMD_STATE_RES_NEED_THREAD;
+			break;
+
+		default:
+			goto out_error;
+		}
+		break;
+	}
+
+out:
+	/* Caution: cmd can be already dead here */
+	return res;
+
+out_error:
+	if (rc == SCST_TGT_RES_FATAL_ERROR) {
+		PRINT_ERROR("Target driver %s xmit_response() returned "
+			"fatal error", tgtt->name);
+	} else {
+		PRINT_ERROR("Target driver %s xmit_response() returned "
+			"invalid value %d", tgtt->name, rc);
+	}
+	scst_set_cmd_error(cmd, SCST_LOAD_SENSE(scst_sense_hardw_error));
+	cmd->state = SCST_CMD_STATE_FINISHED;
+	res = SCST_CMD_STATE_RES_CONT_SAME;
+	goto out;
+}
+
+/**
+ * scst_tgt_cmd_done() - the command's processing done
+ * @cmd:	SCST command
+ * @pref_context: preferred command execution context
+ *
+ * Description:
+ *    Notifies SCST that the driver sent the response and the command
+ *    can be freed now. Don't forget to set the delivery status, if it
+ *    isn't success, using scst_set_delivery_status() before calling
+ *    this function. The third argument sets preferred command execition
+ *    context (see SCST_CONTEXT_* constants for details)
+ */
+void scst_tgt_cmd_done(struct scst_cmd *cmd,
+	enum scst_exec_context pref_context)
+{
+
+	BUG_ON(cmd->state != SCST_CMD_STATE_XMIT_WAIT);
+
+	scst_set_xmit_time(cmd);
+
+	cmd->cmd_hw_pending = 0;
+
+	cmd->state = SCST_CMD_STATE_FINISHED;
+	scst_process_redirect_cmd(cmd, pref_context, 1);
+	return;
+}
+EXPORT_SYMBOL(scst_tgt_cmd_done);
+
+static int scst_finish_cmd(struct scst_cmd *cmd)
+{
+	int res;
+	struct scst_session *sess = cmd->sess;
+
+	scst_update_lat_stats(cmd);
+
+	if (unlikely(cmd->delivery_status != SCST_CMD_DELIVERY_SUCCESS)) {
+		if ((cmd->tgt_dev != NULL) &&
+		    scst_is_ua_sense(cmd->sense, cmd->sense_valid_len)) {
+			/* This UA delivery failed, so we need to requeue it */
+			if (scst_cmd_atomic(cmd) &&
+			    scst_is_ua_global(cmd->sense, cmd->sense_valid_len)) {
+				TRACE_MGMT_DBG("Requeuing of global UA for "
+					"failed cmd %p needs a thread", cmd);
+				res = SCST_CMD_STATE_RES_NEED_THREAD;
+				goto out;
+			}
+			scst_requeue_ua(cmd);
+		}
+	}
+
[… more lines in the raw message]