Index: iscsi_globals.h
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_globals.h,v
retrieving revision 1.5
diff -u -r1.5 iscsi_globals.h
--- iscsi_globals.h	12 Aug 2012 13:26:18 -0000	1.5
+++ iscsi_globals.h	11 Sep 2012 19:22:21 -0000
@@ -84,7 +84,7 @@
 effectively says "don't bother testing these values", and is used right
 now only in iscsi_send.c.
  */
-#define ISCSI_TROTTLING_ENABLED	1
+#define ISCSI_THROTTLING_ENABLED	1
 #define ISCSI_SERVER_TRUSTED	1
 
 /*
@@ -131,14 +131,15 @@
 
 /* CCB Flags */
 
-#define CCBF_COMPLETE   0x01	/* received status */
-#define CCBF_RESENT     0x02	/* ccb was resent */
-#define CCBF_SENDTARGET 0x04	/* SendTargets text request, not negotiation */
-#define CCBF_WAITING    0x08	/* CCB is waiting for MaxCmdSN, wake it up */
-#define CCBF_GOT_RSP    0x10	/* Got at least one response to this request */
-#define CCBF_REASSIGN	0x20	/* Command can be reassigned */
-#define CCBF_OTHERCONN	0x40	/* a logout for a different connection */
-
+#define CCBF_COMPLETE   0x0001	/* received status */
+#define CCBF_RESENT     0x0002	/* ccb was resent */
+#define CCBF_SENDTARGET 0x0004	/* SendTargets text request, not negotiation */
+#define CCBF_WAITING    0x0008	/* CCB is waiting for MaxCmdSN, wake it up */
+#define CCBF_GOT_RSP    0x0010	/* Got at least one response to this request */
+#define CCBF_REASSIGN	0x0020	/* Command can be reassigned */
+#define CCBF_OTHERCONN	0x0040	/* a logout for a different connection */
+#define CCBF_WAITQUEUE  0x0080	/* CCB is on waiting queue */
+#define CCBF_THROTTLING 0x0100	/* CCB is on throttling queue */
 
 /* ---------------------------  Global Types  ------------------------------- */
 
@@ -602,9 +603,6 @@
 
 /* Critical section macros */
 
-#define CS_BEGIN     { int s = splbio ();
-#define CS_END       splx (s); }
-
 /* misc stuff */
 #define min(a, b) ((a) < (b)) ? (a) : (b)
 #define max(a, b) ((a) < (b)) ? (b) : (a)
@@ -801,12 +799,12 @@
 void create_ccbs(session_t *);
 ccb_t *get_ccb(connection_t *, bool);
 void free_ccb(ccb_t *);
-void wake_ccb(ccb_t *);
-void complete_ccb(ccb_t *);
+void suspend_ccb(ccb_t *, bool);
+void throttle_ccb(ccb_t *, bool);
+void wake_ccb(ccb_t *, uint32_t);
 
 void create_pdus(connection_t *);
-pdu_t *get_pdu(connection_t *);
-pdu_t *get_pdu_c(connection_t *, bool);
+pdu_t *get_pdu(connection_t *, bool);
 void free_pdu(pdu_t *);
 
 void init_sernum(sernum_buffer_t *);
Index: iscsi_ioctl.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_ioctl.c,v
retrieving revision 1.5
diff -u -r1.5 iscsi_ioctl.c
--- iscsi_ioctl.c	12 Aug 2012 13:26:18 -0000	1.5
+++ iscsi_ioctl.c	11 Sep 2012 19:22:21 -0000
@@ -90,6 +90,7 @@
 {
 	event_handler_t *handler;
 	int was_empty;
+	int s;
 
 	handler = malloc(sizeof(event_handler_t), M_DEVBUF, M_WAITOK | M_ZERO);
 	if (handler == NULL) {
@@ -101,7 +102,7 @@
 	TAILQ_INIT(&handler->events);
 
 	/* create a unique ID */
-	CS_BEGIN;
+	s = splbio();
 	do {
 		++handler_id;
 	} while (!handler_id || find_handler(handler_id) != NULL);
@@ -114,7 +115,7 @@
 	if (was_empty) {
 		wakeup(&iscsi_cleanup_list);
 	}
-	CS_END;
+	splx(s);
 
 	par->status = ISCSI_STATUS_SUCCESS;
 	DEB(5, ("Register Event OK, ID %d\n", par->event_id));
@@ -134,6 +135,7 @@
 {
 	event_handler_t *handler;
 	event_t *evt;
+	int s;
 
 	handler = find_handler(par->event_id);
 	if (handler == NULL) {
@@ -141,9 +143,11 @@
 		par->status = ISCSI_STATUS_INVALID_EVENT_ID;
 		return;
 	}
-	CS_BEGIN;
+
+	s = splbio();
 	TAILQ_REMOVE(&event_handlers, handler, link);
-	CS_END;
+	splx(s);
+
 	if (handler->waiter != NULL) {
 		handler->waiter->status = ISCSI_STATUS_EVENT_DEREGISTERED;
 		wakeup(handler->waiter);
@@ -238,10 +242,12 @@
 {
 	event_handler_t *curr;
 	event_t *evt;
+	int s;
 
 	DEB(9, ("Add_event kind %d, sid %d, cid %d, reason %d\n",
 		kind, sid, cid, reason));
 
+	s = splbio();
 	TAILQ_FOREACH(curr, &event_handlers, link) {
 		evt = malloc(sizeof(*evt), M_TEMP, M_WAITOK);
 		if (evt == NULL) {
@@ -251,14 +257,14 @@
 		evt->session_id = sid;
 		evt->connection_id = cid;
 		evt->reason = reason;
-		CS_BEGIN;
+
 		TAILQ_INSERT_TAIL(&curr->events, evt, link);
 		if (curr->waiter != NULL) {
 			wakeup(curr->waiter);
 			curr->waiter = NULL;
 		}
-		CS_END;
 	}
+	splx(s);
 }
 
 
@@ -273,6 +279,8 @@
  *    list has changed at all. If not, the event is deregistered.
  *    Note that this will not detect dead handlers if no events are pending,
  *    but we don't care as long as events don't accumulate in the list.
+ *
+ *    this function must be called at splbio
  */
 
 STATIC void
@@ -367,11 +375,14 @@
 find_session(uint32_t id)
 {
 	session_t *curr;
+	int s;
 
+	s = splbio();
 	TAILQ_FOREACH(curr, &iscsi_sessions, sessions)
 		if (curr->id == id) {
 			break;
 		}
+	splx(s);
 	return curr;
 }
 
@@ -389,11 +400,14 @@
 find_connection(session_t *session, uint32_t id)
 {
 	connection_t *curr;
+	int s;
 
+	s = splbio();
 	TAILQ_FOREACH(curr, &session->conn_list, connections)
 		if (curr->id == id) {
 			break;
 		}
+	splx(s);
 	return curr;
 }
 
@@ -413,6 +427,7 @@
 kill_connection(connection_t *conn, uint32_t status, int logout, bool recover)
 {
 	session_t *sess = conn->session;
+	int s;
 
 	DEBC(conn, 1, ("Kill_connection: terminating=%d, status=%d, logout=%d, "
 			   "state=%d\n",
@@ -427,13 +442,15 @@
 	}
 
 	if (!recover || conn->destroy) {
-		CS_BEGIN;
+
+		s = splbio();
 		if (conn->in_session) {
 			conn->in_session = FALSE;
 			TAILQ_REMOVE(&sess->conn_list, conn, connections);
 			sess->mru_connection = TAILQ_FIRST(&sess->conn_list);
 		}
-		CS_END;
+		splx(s);
+
 		if (!conn->destroy) {
 			DEBC(conn, 1, ("Kill_connection setting destroy flag\n"));
 			conn->destroy = TRUE;
@@ -512,6 +529,7 @@
 {
 	connection_t *curr;
 	ccb_t *ccb;
+	int s;
 
 	DEB(1, ("ISCSI: kill_session %d, status %d, logout %d, recover %d\n",
 			session->id, status, logout, recover));
@@ -543,15 +561,16 @@
 	}
 
 	/* remove from session list */
+	s = splbio();
 	TAILQ_REMOVE(&iscsi_sessions, session, sessions);
+	splx(s);
 	session->sessions.tqe_next = NULL;
 	session->sessions.tqe_prev = NULL;
 
 	/* complete any throttled CCBs */
 	while ((ccb = TAILQ_FIRST(&session->ccbs_throttled)) != NULL) {
-		ccb->status = ISCSI_STATUS_LOGOUT;
-		TAILQ_REMOVE(&session->ccbs_throttled, ccb, chain);
-		complete_ccb(ccb);
+		throttle_ccb(ccb, FALSE);
+		wake_ccb(ccb, ISCSI_STATUS_LOGOUT);
 	}
 
 	/*
@@ -595,7 +614,7 @@
 				  PTHREADOBJ p)
 {
 	connection_t *connection;
-	int rc;
+	int rc, s;
 
 	DEB(1, ("Create Connection for Session %d\n", session->id));
 
@@ -709,15 +728,14 @@
 		return -1;
 	}
 
-	CS_BEGIN;
+	s = splbio();
 	connection->state = ST_FULL_FEATURE;
-
 	TAILQ_INSERT_TAIL(&session->conn_list, connection, connections);
 	connection->in_session = TRUE;
 	session->total_connections++;
 	session->active_connections++;
 	session->mru_connection = connection;
-	CS_END;
+	splx(s);
 
 	DEBC(connection, 5, ("Connection created successfully!\n"));
 	return 0;
@@ -742,7 +760,7 @@
 recreate_connection(iscsi_login_parameters_t *par, session_t *session,
 					connection_t *connection, PTHREADOBJ p)
 {
-	int rc;
+	int rc, s;
 	ccb_t *ccb;
 	ccb_list_t old_waiting;
 
@@ -777,10 +795,9 @@
 	session->active_connections++;
 
 	TAILQ_INIT(&old_waiting);
-	while ((ccb = TAILQ_FIRST(&connection->ccbs_waiting)) != NULL) {
-		TAILQ_REMOVE(&connection->ccbs_waiting, ccb, chain);
-		TAILQ_INSERT_TAIL(&old_waiting, ccb, chain);
-	}
+	s = splbio();
+	TAILQ_CONCAT(&old_waiting, &connection->ccbs_waiting, chain);
+	splx(s);
 
 	init_sernum(&connection->StatSN_buf);
 	wakeup(connection);
@@ -789,8 +806,7 @@
 		DEBOUT(("Login failed (rc %d)\n", rc));
 		while ((ccb = TAILQ_FIRST(&old_waiting)) != NULL) {
 			TAILQ_REMOVE(&old_waiting, ccb, chain);
-			ccb->status = rc;
-			complete_ccb(ccb);
+			wake_ccb(ccb, rc);
 		}
 		/* Don't attempt to recover, there seems to be something amiss */
 		kill_connection(connection, rc, NO_LOGOUT, FALSE);
@@ -800,14 +816,15 @@
 
 	DEBC(connection, 9, ("Re-Login successful\n"));
 	par->status = ISCSI_STATUS_SUCCESS;
-	CS_BEGIN;
+
+	s = splbio();
 	connection->state = ST_FULL_FEATURE;
 	session->mru_connection = connection;
-	CS_END;
+	splx(s);
 
 	while ((ccb = TAILQ_FIRST(&old_waiting)) != NULL) {
 		TAILQ_REMOVE(&old_waiting, ccb, chain);
-		TAILQ_INSERT_TAIL(&connection->ccbs_waiting, ccb, chain);
+		suspend_ccb(ccb, TRUE);
 
 		rc = send_task_management(connection, ccb, NULL, TASK_REASSIGN);
 		/* if we get an error on reassign, restart the original request */
@@ -926,7 +943,7 @@
 login(iscsi_login_parameters_t *par, PTHREADOBJ p)
 {
 	session_t *session;
-	int rc;
+	int rc, s;
 
 	DEB(99, ("ISCSI: login\n"));
 
@@ -967,9 +984,9 @@
 		return;
 	}
 
-	CS_BEGIN;
+	s = splbio();
 	TAILQ_INSERT_HEAD(&iscsi_sessions, session, sessions);
-	CS_END;
+	splx(s);
 
 	/* Session established, map LUNs? */
 	if (par->login_type == ISCSI_LOGINTYPE_MAP) {
Index: iscsi_main.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_main.c,v
retrieving revision 1.6
diff -u -r1.6 iscsi_main.c
--- iscsi_main.c	12 Aug 2012 13:26:18 -0000	1.6
+++ iscsi_main.c	11 Sep 2012 19:22:21 -0000
@@ -361,6 +361,7 @@
 	struct scsipi_xfer *xs;
 	session_t *session;
 	int flags;
+	struct scsipi_xfer_mode *xm;
 
 	session = (session_t *) adapt;	/* adapter is first field in session */
 
@@ -400,6 +401,9 @@
 
 	case ADAPTER_REQ_SET_XFER_MODE:
 		DEB(5, ("ISCSI: scsipi_request SET_XFER_MODE\n"));
+		xm = (struct scsipi_xfer_mode *)arg;
+		xm->xm_mode |= PERIPH_CAP_TQING;
+		scsipi_async_event(chan, ASYNC_EVENT_XFER_MODE, xm);
 		return;
 
 	default:
@@ -478,8 +482,6 @@
 		scsipi_done(xs);
 		DEB(99, ("scsipi_done returned\n"));
 	}
-
-	free_ccb(ccb);
 }
 
 /* Kernel Module support */
Index: iscsi_rcv.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_rcv.c,v
retrieving revision 1.3
diff -u -r1.3 iscsi_rcv.c
--- iscsi_rcv.c	24 Jun 2012 17:01:35 -0000	1.3
+++ iscsi_rcv.c	11 Sep 2012 19:22:22 -0000
@@ -278,7 +278,9 @@
 		nlen = req_ccb->text_len + pdu->temp_data_len;
 		/* Note: allocate extra 2 bytes for text terminator */
 		if ((newp = malloc(nlen + 2, M_TEMP, M_WAITOK)) == NULL) {
+			DEBOUT(("Collect Text Data: Out of Memory, ccb = %p\n", req_ccb));
 			req_ccb->status = ISCSI_STATUS_NO_RESOURCES;
+			/* XXX where is CCB freed? */
 			return 1;
 		}
 		memcpy(newp, req_ccb->text_data, req_ccb->text_len);
@@ -374,8 +376,7 @@
 	uint32_t sn = ntohl(nw_sn);
 	ccb_t *ccb, *nxt;
 
-	for (ccb = TAILQ_FIRST(&conn->ccbs_waiting); ccb != NULL; ccb = nxt) {
-		nxt = TAILQ_NEXT(ccb, chain);
+	TAILQ_FOREACH_SAFE(ccb, &conn->ccbs_waiting, chain, nxt) {
 		DEBC(conn, 10,
 			("CheckCmdSN - CmdSN=%d, ExpCmdSn=%d, waiting=%p, flags=%x\n",
 			ccb->CmdSN, sn, ccb->pdu_waiting, ccb->flags));
@@ -429,10 +430,7 @@
 		DEBC(conn, 1, ("Login problem - Class = %x, Detail = %x\n",
 				pdu->pdu.p.login_rsp.StatusClass,
 				pdu->pdu.p.login_rsp.StatusDetail));
-
-		req_ccb->status = ISCSI_STATUS_LOGIN_FAILED;
-		/* XXX */
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, ISCSI_STATUS_LOGIN_FAILED);
 		return 0;
 	}
 
@@ -511,6 +509,7 @@
 {
 	bool otherconn;
 	uint8_t response;
+	uint32_t status;
 
 	otherconn = (req_ccb != NULL) ? (req_ccb->flags & CCBF_OTHERCONN) != 0 : 1;
 	response = pdu->pdu.OpcodeSpecific [0];
@@ -526,16 +525,16 @@
 
 	switch (response) {
 	case 0:
-		req_ccb->status = ISCSI_STATUS_SUCCESS;
+		status = ISCSI_STATUS_SUCCESS;
 		break;
 	case 1:
-		req_ccb->status = ISCSI_STATUS_LOGOUT_CID_NOT_FOUND;
+		status = ISCSI_STATUS_LOGOUT_CID_NOT_FOUND;
 		break;
 	case 2:
-		req_ccb->status = ISCSI_STATUS_LOGOUT_RECOVERY_NS;
+		status = ISCSI_STATUS_LOGOUT_RECOVERY_NS;
 		break;
 	default:
-		req_ccb->status = ISCSI_STATUS_LOGOUT_ERROR;
+		status = ISCSI_STATUS_LOGOUT_ERROR;
 		break;
 	}
 
@@ -546,7 +545,7 @@
 		refconn->Time2Retain = ntohs(pdu->pdu.p.logout_rsp.Time2Retain);
 	}
 
-	wake_ccb(req_ccb);
+	wake_ccb(req_ccb, status);
 
 	if (!otherconn && conn->state == ST_LOGOUT_SENT) {
 		conn->terminating = ISCSI_STATUS_LOGOUT;
@@ -641,14 +640,13 @@
 		/* successful transfer, reset recover count */
 		conn->recover = 0;
 
-		if (done) {
-			wake_ccb(req_ccb);
-		}
-		if (check_StatSN(conn, pdu->pdu.p.data_in.StatSN, done)) {
+		if (done)
+			wake_ccb(req_ccb, ISCSI_STATUS_SUCCESS);
+		if (check_StatSN(conn, pdu->pdu.p.data_in.StatSN, done))
 			return -1;
-		}
+
 	} else if (done && (req_ccb->flags & CCBF_COMPLETE)) {
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, ISCSI_STATUS_SUCCESS);
 	}
 	/* else wait for command response */
 
@@ -699,6 +697,7 @@
 {
 	int len, rc;
 	bool done;
+	uint32_t status;
 
 	/* Read any provided data */
 	if (pdu->temp_data_len && req_ccb != NULL && req_ccb->sense_len_req) {
@@ -726,23 +725,23 @@
 	conn->recover = 0;	/* successful transfer, reset recover count */
 
 	if (pdu->pdu.OpcodeSpecific[0]) {	/* Response */
-		req_ccb->status = ISCSI_STATUS_TARGET_FAILURE;
+		status = ISCSI_STATUS_TARGET_FAILURE;
 	} else {
 		switch (pdu->pdu.OpcodeSpecific[1]) {	/* Status */
 		case 0x00:
-			/* success */
+			status = ISCSI_STATUS_SUCCESS;
 			break;
 
 		case 0x02:
-			req_ccb->status = ISCSI_STATUS_CHECK_CONDITION;
+			status = ISCSI_STATUS_CHECK_CONDITION;
 			break;
 
 		case 0x08:
-			req_ccb->status = ISCSI_STATUS_TARGET_BUSY;
+			status = ISCSI_STATUS_TARGET_BUSY;
 			break;
 
 		default:
-			req_ccb->status = ISCSI_STATUS_TARGET_ERROR;
+			status = ISCSI_STATUS_TARGET_ERROR;
 			break;
 		}
 	}
@@ -750,7 +749,7 @@
 	if (pdu->pdu.Flags & (FLAG_OVERFLOW | FLAG_UNDERFLOW))
 		req_ccb->residual = ntohl(pdu->pdu.p.response.ResidualCount);
 
-	done = req_ccb->status || sn_empty(&req_ccb->DataSN_buf);
+	done = status || sn_empty(&req_ccb->DataSN_buf);
 
 	DEBC(conn, 10, ("Rx Command Response rsp = %x, status = %x\n",
 			pdu->pdu.OpcodeSpecific[0], pdu->pdu.OpcodeSpecific[1]));
@@ -758,7 +757,7 @@
 	rc = check_StatSN(conn, pdu->pdu.p.response.StatSN, done);
 
 	if (done)
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, status);
 
 	return rc;
 }
@@ -836,6 +835,7 @@
 {
 	pdu_header_t *hpdu;
 	ccb_t *req_ccb;
+	uint32_t status;
 
 	DEBOUT(("Received Reject PDU, reason = %x, data_len = %d\n",
 			pdu->pdu.OpcodeSpecific[0], pdu->temp_data_len));
@@ -864,23 +864,23 @@
 
 		case REJECT_SNACK:
 		case REJECT_PROTOCOL_ERROR:
-			req_ccb->status = ISCSI_STATUS_PROTOCOL_ERROR;
+			status = ISCSI_STATUS_PROTOCOL_ERROR;
 			break;
 
 		case REJECT_CMD_NOT_SUPPORTED:
-			req_ccb->status = ISCSI_STATUS_CMD_NOT_SUPPORTED;
+			status = ISCSI_STATUS_CMD_NOT_SUPPORTED;
 			break;
 
 		case REJECT_INVALID_PDU_FIELD:
-			req_ccb->status = ISCSI_STATUS_PDU_ERROR;
+			status = ISCSI_STATUS_PDU_ERROR;
 			break;
 
 		default:
-			req_ccb->status = ISCSI_STATUS_GENERAL_ERROR;
+			status = ISCSI_STATUS_GENERAL_ERROR;
 			break;
 		}
 
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, status);
 		handle_connection_error(conn, ISCSI_STATUS_PROTOCOL_ERROR,
 							LOGOUT_CONNECTION);
 	}
@@ -901,6 +901,7 @@
 STATIC int
 receive_task_management_pdu(connection_t *conn, pdu_t *pdu, ccb_t *req_ccb)
 {
+	uint32_t status;
 
 	DEBC(conn, 2, ("Received Task Management PDU, response %d, req_ccb %p\n",
 			pdu->pdu.OpcodeSpecific[0], req_ccb));
@@ -908,34 +909,34 @@
 	if (req_ccb != NULL) {
 		switch (pdu->pdu.OpcodeSpecific[0]) {	/* Response */
 		case 0:
-			req_ccb->status = ISCSI_STATUS_SUCCESS;
+			status = ISCSI_STATUS_SUCCESS;
 			break;
 		case 1:
-			req_ccb->status = ISCSI_STATUS_TASK_NOT_FOUND;
+			status = ISCSI_STATUS_TASK_NOT_FOUND;
 			break;
 		case 2:
-			req_ccb->status = ISCSI_STATUS_LUN_NOT_FOUND;
+			status = ISCSI_STATUS_LUN_NOT_FOUND;
 			break;
 		case 3:
-			req_ccb->status = ISCSI_STATUS_TASK_ALLEGIANT;
+			status = ISCSI_STATUS_TASK_ALLEGIANT;
 			break;
 		case 4:
-			req_ccb->status = ISCSI_STATUS_CANT_REASSIGN;
+			status = ISCSI_STATUS_CANT_REASSIGN;
 			break;
 		case 5:
-			req_ccb->status = ISCSI_STATUS_FUNCTION_UNSUPPORTED;
+			status = ISCSI_STATUS_FUNCTION_UNSUPPORTED;
 			break;
 		case 6:
-			req_ccb->status = ISCSI_STATUS_FUNCTION_NOT_AUTHORIZED;
+			status = ISCSI_STATUS_FUNCTION_NOT_AUTHORIZED;
 			break;
 		case 255:
-			req_ccb->status = ISCSI_STATUS_FUNCTION_REJECTED;
+			status = ISCSI_STATUS_FUNCTION_REJECTED;
 			break;
 		default:
-			req_ccb->status = ISCSI_STATUS_UNKNOWN_REASON;
+			status = ISCSI_STATUS_UNKNOWN_REASON;
 			break;
 		}
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, status);
 	}
 
 	check_StatSN(conn, pdu->pdu.p.task_rsp.StatSN, TRUE);
@@ -982,7 +983,7 @@
 		/* and advance StatSN */
 		check_CmdSN(conn, pdu->pdu.p.nop_in.ExpCmdSN);
 
-		wake_ccb(req_ccb);
+		wake_ccb(req_ccb, ISCSI_STATUS_SUCCESS);
 
 		check_StatSN(conn, pdu->pdu.p.nop_in.StatSN, TRUE);
 	}
@@ -1007,7 +1008,7 @@
 {
 	ccb_t *req_ccb;
 	ccb_list_t waiting;
-	int rc;
+	int rc, s;
 	uint32_t MaxCmdSN, digest;
 	session_t *sess = conn->session;
 
@@ -1131,22 +1132,16 @@
 	 */
 	if (MaxCmdSN != sess->MaxCmdSN) {
 		sess->MaxCmdSN = MaxCmdSN;
-#if 0
-/* XXX - agc */
-		if (TAILQ_FIRST(&sess->ccbs_throttled) == NULL ||
-			!sn_a_lt_b(sess->CmdSN, MaxCmdSN))
-			return 0;
-#else
 		if (TAILQ_FIRST(&sess->ccbs_throttled) == NULL)
 			return 0;
-#endif
 
 		DEBC(conn, 1, ("Unthrottling - MaxCmdSN = %d\n", MaxCmdSN));
 
-		CS_BEGIN;
 		TAILQ_INIT(&waiting);
+		s = splbio();
 		TAILQ_CONCAT(&waiting, &sess->ccbs_throttled, chain);
-		CS_END;
+		splx(s);
+
 		while ((req_ccb = TAILQ_FIRST(&waiting)) != NULL) {
 			TAILQ_REMOVE(&waiting, req_ccb, chain);
 			DEBC(conn, 1, ("Unthrottling - ccb = %p, disp = %d\n",
@@ -1181,7 +1176,7 @@
 
 	do {
 		while (!conn->terminating) {
-			pdu = get_pdu(conn);
+			pdu = get_pdu(conn, TRUE);
 			pdu->uio.uio_iov = pdu->io_vec;
 			UIO_SETUP_SYSSPACE(&pdu->uio);
 			pdu->uio.uio_iovcnt = 1;
Index: iscsi_send.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_send.c,v
retrieving revision 1.7
diff -u -r1.7 iscsi_send.c
--- iscsi_send.c	9 Sep 2012 06:06:29 -0000	1.7
+++ iscsi_send.c	11 Sep 2012 19:22:22 -0000
@@ -172,7 +172,7 @@
 
 	TAILQ_FOREACH(ccb, &oldconn->ccbs_waiting, chain) {
 		/* Copy PDU contents (PDUs are bound to connection) */
-		if ((pdu = get_pdu(conn)) == NULL) {
+		if ((pdu = get_pdu(conn, TRUE)) == NULL) {
 			break;
 		}
 		opdu = ccb->pdu_waiting;
@@ -214,15 +214,14 @@
 		DEBC(conn, 1, ("Error while copying PDUs in reassign_tasks!\n"));
 		/* give up recovering, the other connection is screwed up as well... */
 		while ((ccb = TAILQ_FIRST(&oldconn->ccbs_waiting)) != NULL) {
-			ccb->status = oldconn->terminating;
-			wake_ccb(ccb);
+			wake_ccb(ccb, oldconn->terminating);
 		}
 		return;
 	}
 
 	while ((ccb = TAILQ_FIRST(&oldconn->ccbs_waiting)) != NULL) {
 		TAILQ_REMOVE(&oldconn->ccbs_waiting, ccb, chain);
-		TAILQ_INSERT_TAIL(&conn->ccbs_waiting, ccb, chain);
+		suspend_ccb(ccb, TRUE);
 
 		if (!no_tm) {
 			rc = send_task_management(conn, ccb, NULL, TASK_REASSIGN);
@@ -353,13 +352,11 @@
 		sounlock((struct socket *) fp->f_data);
 
 		/* wake up any non-reassignable waiting CCBs */
-		for (ccb = TAILQ_FIRST(&conn->ccbs_waiting); ccb != NULL; ccb = nccb) {
-			nccb = TAILQ_NEXT(ccb, chain);
+		TAILQ_FOREACH_SAFE(ccb, &conn->ccbs_waiting, chain, nccb) {
 			if (!(ccb->flags & CCBF_REASSIGN) || ccb->pdu_waiting == NULL) {
 				DEBC(conn, 9, ("Terminating CCB %p (t=%p)\n",
 					ccb,&ccb->timeout));
-				ccb->status = conn->terminating;
-				wake_ccb(ccb);
+				wake_ccb(ccb, conn->terminating);
 			} else {
 				callout_stop(&ccb->timeout);
 				ccb->num_timeouts = 0;
@@ -377,7 +374,7 @@
 			}
 		}
 
-        /* If there's another connection available, transfer pending tasks */
+		/* If there's another connection available, transfer pending tasks */
 		if (sess->active_connections &&
 			TAILQ_FIRST(&conn->ccbs_waiting) != NULL) {
 			reassign_tasks (conn);
@@ -399,8 +396,7 @@
 
 	/* wake up any waiting CCBs */
 	while ((ccb = TAILQ_FIRST(&conn->ccbs_waiting)) != NULL) {
-		ccb->status = conn->terminating;
-		wake_ccb(ccb);
+		wake_ccb(ccb, conn->terminating);
 		/* NOTE: wake_ccb will remove the CCB from the queue */
 	}
 
@@ -441,6 +437,7 @@
 {
 	connection_t *conn = pdu->connection;
 	ccb_disp_t prev_cdisp = 0;
+	int s;
 
 	if (ccb != NULL) {
 		prev_cdisp = ccb->disp;
@@ -455,7 +452,7 @@
 	DEBC(conn, 10, ("Send_pdu: ccb=%p, pcd=%d, cdsp=%d, pdu=%p, pdsp=%d\n",
 			ccb, prev_cdisp, cdisp, pdu, pdisp));
 
-	CS_BEGIN;
+	s = splbio();
 	if (pdisp == PDUDISP_WAIT) {
 		ccb->pdu_waiting = pdu;
 
@@ -479,12 +476,12 @@
 		SET_CCB_TIMEOUT(conn, ccb, COMMAND_TIMEOUT);
 
 		if (prev_cdisp <= CCBDISP_NOWAIT)
-			TAILQ_INSERT_TAIL(&conn->ccbs_waiting, ccb, chain);
+			suspend_ccb(ccb, TRUE);
 
 		if (cdisp == CCBDISP_WAIT)
 			tsleep(ccb, PWAIT, "sendpdu", 0);
 	}
-	CS_END;
+	splx(s);
 }
 
 
@@ -671,11 +668,11 @@
 		set_negotiated_parameters(tx_ccb);
 
 		DEBC(conn, 5, ("Login Successful!\n"));
-		wake_ccb(tx_ccb);
+		wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
 		return;
 	}
 
-	tx_pdu = get_pdu(conn);
+	tx_pdu = get_pdu(conn, TRUE);
 	if (tx_pdu == NULL)
 		return;
 
@@ -699,8 +696,7 @@
 	}
 
 	if (rc > 0) {
-		tx_ccb->status = rc;
-		wake_ccb(tx_ccb);
+		wake_ccb(tx_ccb, rc);
 		free_pdu(tx_pdu);
 	} else {
 		init_login_pdu(conn, tx_pdu, next);
@@ -754,7 +750,7 @@
 {
 	pdu_t *tx_pdu;
 
-	tx_pdu = get_pdu(conn);
+	tx_pdu = get_pdu(conn, TRUE);
 	if (tx_pdu == NULL)
 		return;
 
@@ -786,7 +782,7 @@
 	ccb = get_ccb(conn, TRUE);
 	if (ccb == NULL)
 		return;
-	pdu = get_pdu(conn);
+	pdu = get_pdu(conn, TRUE);
 	if (pdu == NULL) {
 		free_ccb(ccb);
 		return;
@@ -830,10 +826,10 @@
 		tx_ccb->text_data = rx_pdu->temp_data;
 		tx_ccb->text_len = rx_pdu->temp_data_len;
 		rx_pdu->temp_data = NULL;
-		wake_ccb(tx_ccb);
+		wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
 	} else {
 		if (!(rx_pdu->pdu.Flags & FLAG_FINAL))
-			tx_pdu = get_pdu(conn);
+			tx_pdu = get_pdu(conn, TRUE);
 		else
 			tx_pdu = NULL;
 
@@ -850,7 +846,7 @@
 			send_pdu(tx_ccb, tx_pdu, CCBDISP_NOWAIT, PDUDISP_FREE);
 		} else {
 			set_negotiated_parameters(tx_ccb);
-			wake_ccb(tx_ccb);
+			wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
 		}
 	}
 }
@@ -886,7 +882,7 @@
 	ccb = get_ccb(conn, TRUE);
 	if (ccb == NULL)
 		return conn->terminating;
-	pdu = get_pdu(conn);
+	pdu = get_pdu(conn, TRUE);
 	if (pdu == NULL) {
 		free_ccb(ccb);
 		return conn->terminating;
@@ -939,7 +935,7 @@
 
 	if (rx_pdu != NULL) {
 		ccb = NULL;
-		ppdu = get_pdu(conn);
+		ppdu = get_pdu(conn, TRUE);
 		if (ppdu == NULL)
 			return 1;
 	} else {
@@ -948,7 +944,7 @@
 			DEBOUT(("Can't get CCB in send_nop_out\n"));
 			return 1;
 		}
-		ppdu = get_pdu_c(conn, FALSE);
+		ppdu = get_pdu(conn, FALSE);
 		if (ppdu == NULL) {
 			free_ccb(ccb);
 			DEBOUT(("Can't get PDU in send_nop_out\n"));
@@ -1001,7 +997,7 @@
 	PDEBC(conn, 1, ("SNACK Missing type = %d, BegRun %d RunLength %d\n",
 			 type, BegRun, RunLength));
 
-	ppdu = get_pdu(conn);
+	ppdu = get_pdu(conn, TRUE);
 	if (ppdu == NULL)
 		return;
 	pdu = &ppdu->pdu;
@@ -1041,7 +1037,7 @@
 
 	PDEBC(conn, 1, ("Send SNACK type = %d\n", type));
 
-	ppdu = get_pdu(conn);
+	ppdu = get_pdu(conn, TRUE);
 	if (ppdu == NULL)
 		return;
 	pdu = &ppdu->pdu;
@@ -1108,7 +1104,7 @@
 	/* only if terminating (which couldn't possibly happen here, but...) */
 	if (ccb == NULL)
 		return conn->terminating;
-	pdu = get_pdu(conn);
+	pdu = get_pdu(conn, TRUE);
 	if (pdu == NULL) {
 		free_ccb(ccb);
 		return conn->terminating;
@@ -1154,7 +1150,7 @@
 	/* can only happen if terminating... */
 	if (ccb == NULL)
 		return conn->terminating;
-	ppdu = get_pdu(conn);
+	ppdu = get_pdu(conn, TRUE);
 	if (ppdu == NULL) {
 		free_ccb(ccb);
 		return conn->terminating;
@@ -1221,7 +1217,7 @@
 	/* can only happen if terminating... */
 	if (ccb == NULL)
 		return conn->terminating;
-	ppdu = get_pdu(conn);
+	ppdu = get_pdu(conn, TRUE);
 	if (ppdu == NULL) {
 		free_ccb(ccb);
 		return conn->terminating;
@@ -1290,7 +1286,7 @@
 	while (totlen) {
 		len = min(totlen, conn->max_transfer);
 
-		tx_pdu = get_pdu_c(conn, waitok);
+		tx_pdu = get_pdu(conn, waitok);
 		if (tx_pdu == NULL) {
 			DEBOUT(("No PDU in send_data_out\n"));
 
@@ -1351,52 +1347,41 @@
 
 	PERF_BEGIN(ccb, !waitok);
 
-	if (!waitok) {
-		s = splbio();
-		if (/*CONSTCOND*/ISCSI_TROTTLING_ENABLED &&
-		    /*CONSTCOND*/ISCSI_SERVER_TRUSTED &&
-		    !sn_a_le_b(sess->CmdSN, sess->MaxCmdSN)) {
-			ccb->disp = disp;
-			TAILQ_INSERT_TAIL(&sess->ccbs_throttled, ccb, chain);
-			splx(s);
-			PDEBOUT(("Throttling S - CmdSN = %d, MaxCmdSN = %d\n",
-					 sess->CmdSN, sess->MaxCmdSN));
-			return;
-		}
-		splx(s);
-		ppdu = get_pdu_c(conn, FALSE);
-		if (ppdu == NULL) {
-			ccb->status = ISCSI_STATUS_NO_RESOURCES;
-			iscsi_done(ccb);
-			return;
-		}
-	} else {
-		s = splbio();
-		while (/*CONSTCOND*/ISCSI_TROTTLING_ENABLED &&
-		       /*CONSTCOND*/ISCSI_SERVER_TRUSTED &&
-		       !sn_a_le_b(sess->CmdSN, sess->MaxCmdSN)) {
-			ccb->disp = disp;
+	s = splbio();
+	if (/*CONSTCOND*/ISCSI_THROTTLING_ENABLED &&
+	    /*CONSTCOND*/ISCSI_SERVER_TRUSTED &&
+	    !sn_a_le_b(sess->CmdSN, sess->MaxCmdSN)) {
+
+		ccb->disp = disp;
+		if (waitok)
 			ccb->flags |= CCBF_WAITING;
-			TAILQ_INSERT_TAIL(&sess->ccbs_throttled, ccb, chain);
-			PDEBOUT(("Throttling W - CmdSN = %d, MaxCmdSN = %d\n",
-					 sess->CmdSN, sess->MaxCmdSN));
+		throttle_ccb(ccb, TRUE);
+
+		PDEBOUT(("Throttling S - CmdSN = %d, MaxCmdSN = %d\n",
+				 sess->CmdSN, sess->MaxCmdSN));
+		if (waitok)
 			tsleep(ccb, PWAIT, "waitMaxCmd", 0);
-			splbio();
-		}
+
 		splx(s);
-		ppdu = get_pdu(conn);
+		return;
 	}
+	splx(s);
+	ppdu = get_pdu(conn, FALSE);
+	if (ppdu == NULL) {
+		wake_ccb(ccb, ISCSI_STATUS_NO_RESOURCES);
+		return;
+	}
+
+	totlen = len = ccb->data_len;
 
 	pdu = &ppdu->pdu;
 	pdu->LUN = htonq(ccb->lun);
 	memcpy(pdu->p.command.SCSI_CDB, ccb->cmd, ccb->cmdlen);
-	totlen = len = ccb->data_len;
 	pdu->Opcode = IOP_SCSI_Command;
 	if (immed)
 		pdu->Opcode |= OP_IMMEDIATE;
 	pdu->p.command.ExpectedDataTransferLength = htonl(totlen);
 
-
 	if (totlen) {
 		if (ccb->data_in) {
 			pdu->Flags = FLAG_READ;
@@ -1436,7 +1421,6 @@
 	PERF_PDUSET(ppdu, ccb, PERF_BEGIN_PDUWRITECMD);
 
 	setup_tx_uio(ppdu, len, ccb->data_ptr, ccb->data_in);
-
 	send_pdu(ccb, ppdu, (totlen) ? CCBDISP_DEFER : disp, PDUDISP_WAIT);
 
 	if (totlen)
@@ -1634,9 +1618,8 @@
 		ccb->total_tries > MAX_CCB_TRIES ||
 		ccb->disp <= CCBDISP_FREE ||
 		!ccb->session->ErrorRecoveryLevel) {
-		ccb->status = ISCSI_STATUS_TIMEOUT;
-		complete_ccb(ccb);
 
+		wake_ccb(ccb, ISCSI_STATUS_TIMEOUT);
 		handle_connection_error(conn, ISCSI_STATUS_TIMEOUT, RECOVER_CONNECTION);
 	} else {
 		if (ccb->data_in && ccb->xfer_len < ccb->data_len) {
Index: iscsi_test.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_test.c,v
retrieving revision 1.2
diff -u -r1.2 iscsi_test.c
--- iscsi_test.c	29 Nov 2011 03:50:31 -0000	1.2
+++ iscsi_test.c	11 Sep 2012 19:22:22 -0000
@@ -294,7 +294,7 @@
 {
 	mod_desc_t *mod;
 	uint32_t mpoff, off;
-	int i, rc = 0;
+	int i, rc = 0, s;
 
 	tp->pdu_count[kind][rxtx]++;
 	tp->pdu_count[ANY_PDU][rxtx]++;
@@ -363,9 +363,10 @@
 
 		if (!off || (mpoff != 0 && mpoff < off)) {
 			/* This might happen in some cases. Just discard the modification. */
-			CS_BEGIN;
+			s = splbio();
 			TAILQ_REMOVE(&tp->mods, mod, link);
-			CS_END;
+			splx(s);
+
 			update_options(tp, mod);
 
 			if (mod->pars.options & ISCSITEST_OPT_WAIT_FOR_COMPLETION) {
@@ -413,8 +414,7 @@
 			if (ccb != NULL &&
 				(ccb->disp == CCBDISP_WAIT || ccb->disp == CCBDISP_SCSIPI)) {
 				/* simulate timeout */
-				ccb->status = ISCSI_STATUS_TIMEOUT;
-				wake_ccb(ccb);
+				wake_ccb(ccb, ISCSI_STATUS_TIMEOUT);
 			}
 		}
 
@@ -436,14 +436,13 @@
 		}
 	}
 
-	CS_BEGIN;
+	s = splbio();
 	TAILQ_REMOVE(&tp->mods, mod, link);
-
 	update_options(tp, mod);
-
 	/* we've modified a PDU - copy current count into last count */
 	memcpy(tp->pdu_last, tp->pdu_count, sizeof(tp->pdu_last));
-	CS_END;
+	splx(s);
+
 	if (mod->pars.options & ISCSITEST_OPT_WAIT_FOR_COMPLETION) {
 		wakeup(mod);
 	}
@@ -941,6 +940,7 @@
 test_cancel(iscsi_test_cancel_parameters_t *par)
 {
 	test_pars_t *tp;
+	int s;
 
 	if ((tp = find_test_id(par->test_id)) == NULL) {
 		par->status = ISCSI_STATUS_INVALID_ID;
@@ -948,11 +948,12 @@
 	}
 	DEB(1, ("Test Cancel, id %d\n", par->test_id));
 
-	CS_BEGIN;
+	s = splbio();
 	if (tp->connection)
 		tp->connection->test_pars = NULL;
 	TAILQ_REMOVE(&test_list, tp, link);
-	CS_END;
+	splx(s);
+
 	free_negs(tp);
 	free_mods(tp, ISCSI_STATUS_TEST_CANCELED);
 	free(tp, M_TEMP);
@@ -980,6 +981,7 @@
 	void *pdu_ptr = par->pdu_ptr;
 	struct uio *uio;
 	uint32_t i, pad, dsl, size;
+	int s;
 
 	if ((tp = find_test_id(par->test_id)) == NULL) {
 		par->status = ISCSI_STATUS_INVALID_ID;
@@ -994,7 +996,7 @@
 		par->status = ISCSI_STATUS_TEST_INACTIVE;
 		return;
 	}
-	if ((pdu = get_pdu(conn)) == NULL) {
+	if ((pdu = get_pdu(conn, TRUE)) == NULL) {
 		par->status = ISCSI_STATUS_TEST_CONNECTION_CLOSED;
 		return;
 	}
@@ -1068,8 +1070,8 @@
 	pdu->disp = PDUDISP_SIGNAL;
 	pdu->flags = PDUF_BUSY | PDUF_NOUPDATE;
 
-	CS_BEGIN;
-    /* Enqueue for sending */
+	s = splbio();
+	/* Enqueue for sending */
 	if (pdu->pdu.Opcode & OP_IMMEDIATE)
 		TAILQ_INSERT_HEAD(&conn->pdus_to_send, pdu, send_chain);
 	else
@@ -1077,7 +1079,8 @@
 
 	wakeup(&conn->pdus_to_send);
 	tsleep(pdu, PINOD, "test_send_pdu", 0);
-	CS_END;
+	splx(s);
+
 	unmap_databuf(p, pdu_ptr, psize);
 	par->status = ISCSI_STATUS_SUCCESS;
 	if (par->options & ISCSITEST_KILL_CONNECTION)
Index: iscsi_utils.c
===================================================================
RCS file: /cvsroot/src/sys/dev/iscsi/iscsi_utils.c,v
retrieving revision 1.4
diff -u -r1.4 iscsi_utils.c
--- iscsi_utils.c	25 Jun 2012 20:34:26 -0000	1.4
+++ iscsi_utils.c	11 Sep 2012 19:22:22 -0000
@@ -219,14 +219,15 @@
 {
 	ccb_t *ccb;
 	session_t *sess = conn->session;
+	int s;
 
 	do {
-		CS_BEGIN;
+		s = splbio();
 		ccb = TAILQ_FIRST(&sess->ccb_pool);
-		if (ccb != NULL) {
+		if (ccb != NULL)
 			TAILQ_REMOVE(&sess->ccb_pool, ccb, chain);
-		}
-		CS_END;
+		s = splx(s);
+
 		DEB(100, ("get_ccb: ccb = %p, waitok = %d\n", ccb, waitok));
 		if (ccb == NULL) {
 			if (!waitok || conn->terminating) {
@@ -262,6 +263,9 @@
 {
 	session_t *sess = ccb->session;
 	pdu_t *pdu;
+	int s;
+
+	KASSERT((ccb->flags & (CCBF_THROTTLING|CCBF_WAITQUEUE)) == 0);
 
 	ccb->connection->usecount--;
 	ccb->connection = NULL;
@@ -281,9 +285,10 @@
 		free_pdu(pdu);
 	}
 
-	CS_BEGIN;
+	s = splbio();
 	TAILQ_INSERT_TAIL(&sess->ccb_pool, ccb, chain);
-	CS_END;
+	splx(s);
+
 	wakeup(&sess->ccb_pool);
 }
 
@@ -319,132 +324,128 @@
 	}
 }
 
-
 /*
- * wake_ccb:
- *    Wake up (or dispose of) a CCB. Depending on the CCB's disposition,
- *    either wake up the requesting thread, signal SCSIPI that we're done,
- *    or just free the CCB for CCBDISP_FREE.
- *
- *    Parameter:  The CCB to handle.
+ * suspend_ccb:
+ *    Put CCB on wait queue
  */
-
 void
-wake_ccb(ccb_t *ccb)
+suspend_ccb(ccb_t *ccb, bool yes)
 {
-	ccb_disp_t disp;
 	connection_t *conn;
 	int s;
-#ifdef ISCSI_DEBUG
-	static ccb_t *lastccb = NULL;
-	static int lastdisp = -1;
-#endif
-
-	/* Just in case */
-	if (ccb == NULL)
-		return;
 
 	conn = ccb->connection;
-
-#ifdef ISCSI_DEBUG
-	if (ccb != lastccb || ccb->disp != lastdisp) {
-		DEBC(conn, 9, ("Wake CCB, ccb = %p, disp = %d\n",
-			ccb, (ccb) ? ccb->disp : 0));
-		lastccb = ccb;
-		lastdisp = (ccb) ? ccb->disp : 0;
-	}
-#endif
-
-	callout_stop(&ccb->timeout);
-
 	s = splbio();
-	disp = ccb->disp;
-	if (disp <= CCBDISP_NOWAIT ||
-		(disp == CCBDISP_DEFER && conn->state <= ST_WINDING_DOWN)) {
-		splx(s);
-		return;
+	if (yes) {
+		KASSERT((ccb->flags & (CCBF_THROTTLING|CCBF_WAITQUEUE)) == 0);
+		TAILQ_INSERT_TAIL(&conn->ccbs_waiting, ccb, chain);
+		ccb->flags |= CCBF_WAITQUEUE;
+	} else if (ccb->flags & CCBF_WAITQUEUE) {
+		KASSERT((ccb->flags & CCBF_THROTTLING) == 0);
+		TAILQ_REMOVE(&conn->ccbs_waiting, ccb, chain);
+		ccb->flags &= ~CCBF_WAITQUEUE;
 	}
-
-	TAILQ_REMOVE(&conn->ccbs_waiting, ccb, chain);
-
-	/* change the disposition so nobody tries this again */
-	ccb->disp = CCBDISP_BUSY;
 	splx(s);
+}
 
-	PERF_END(ccb);
-
-	switch (disp) {
-	case CCBDISP_WAIT:
-		wakeup(ccb);
-		break;
-
-	case CCBDISP_SCSIPI:
-		iscsi_done(ccb);
-		break;
-
-	case CCBDISP_DEFER:
-		break;
+/*
+ * throttle_ccb:
+ *    Put CCB on throttling queue
+ */
+void
+throttle_ccb(ccb_t *ccb, bool yes)
+{
+	session_t *sess;
+	int s;
 
-	default:
-		free_ccb(ccb);
-		break;
+	sess = ccb->session;
+	s = splbio();
+	if (yes) {
+		KASSERT((ccb->flags & (CCBF_THROTTLING|CCBF_WAITQUEUE)) == 0);
+		TAILQ_INSERT_TAIL(&sess->ccbs_throttled, ccb, chain);
+		ccb->flags |= CCBF_THROTTLING;
+	} else if (ccb->flags & CCBF_THROTTLING) {
+		KASSERT((ccb->flags & CCBF_WAITQUEUE) == 0);
+		TAILQ_REMOVE(&sess->ccbs_throttled, ccb, chain);
+		ccb->flags &= ~CCBF_THROTTLING;
 	}
+	splx(s);
 }
 
 
 /*
- * complete_ccb:
- *    Same as wake_ccb, but the CCB is not assumed to be in the waiting list.
+ * wake_ccb:
+ *    Wake up (or dispose of) a CCB. Depending on the CCB's disposition,
+ *    either wake up the requesting thread, signal SCSIPI that we're done,
+ *    or just free the CCB for CCBDISP_FREE.
  *
- *    Parameter:  The CCB to handle.
+ *    Parameter:  The CCB to handle and the new status of the CCB
  */
 
 void
-complete_ccb(ccb_t *ccb)
+wake_ccb(ccb_t *ccb, uint32_t status)
 {
 	ccb_disp_t disp;
+	connection_t *conn;
 	int s;
 
-	/* Just in case */
-	if (ccb == NULL)
-		return;
+	conn = ccb->connection;
+
+#ifdef ISCSI_DEBUG
+	DEBC(conn, 9, ("CCB done, ccb = %p, disp = %d\n",
+		ccb, ccb->disp));
+#endif
 
 	callout_stop(&ccb->timeout);
 
 	s = splbio();
 	disp = ccb->disp;
-	if (disp <= CCBDISP_NOWAIT || disp == CCBDISP_DEFER) {
+	if (disp <= CCBDISP_NOWAIT ||
+		(disp == CCBDISP_DEFER && conn->state <= ST_WINDING_DOWN)) {
 		splx(s);
 		return;
 	}
+
+	suspend_ccb(ccb, FALSE);
+	throttle_ccb(ccb, FALSE);
+
 	/* change the disposition so nobody tries this again */
 	ccb->disp = CCBDISP_BUSY;
+	ccb->status = status;
 	splx(s);
 
 	PERF_END(ccb);
 
 	switch (disp) {
+	case CCBDISP_FREE:
+		free_ccb(ccb);
+		break;
+
 	case CCBDISP_WAIT:
 		wakeup(ccb);
 		break;
 
 	case CCBDISP_SCSIPI:
 		iscsi_done(ccb);
+		free_ccb(ccb);
+		break;
+
+	case CCBDISP_DEFER:
 		break;
 
 	default:
+		DEBC(conn, 1, ("CCB done, ccb = %p, invalid disposition %d", ccb, disp));
 		free_ccb(ccb);
 		break;
 	}
 }
 
-
 /*****************************************************************************
  * PDU management functions
  *****************************************************************************/
 
 /*
- * get_pdu_c:
+ * get_pdu:
  *    Get a PDU for the SCSI operation.
  *
  *    Parameter:
@@ -455,17 +456,18 @@
  */
 
 pdu_t *
-get_pdu_c(connection_t *conn, bool waitok)
+get_pdu(connection_t *conn, bool waitok)
 {
 	pdu_t *pdu;
+	int s;
 
 	do {
-		CS_BEGIN;
+		s = splbio();
 		pdu = TAILQ_FIRST(&conn->pdu_pool);
-		if (pdu != NULL) {
+		if (pdu != NULL)
 			TAILQ_REMOVE(&conn->pdu_pool, pdu, chain);
-		}
-		CS_END;
+		splx(s);
+
 		DEB(100, ("get_pdu_c: pdu = %p, waitok = %d\n", pdu, waitok));
 		if (pdu == NULL) {
 			if (!waitok || conn->terminating)
@@ -483,46 +485,6 @@
 }
 
 /*
- * get_pdu:
- *    Get a PDU for the SCSI operation, waits if none is available.
- *    Same as get_pdu_c, but with wait always OK.
- *    Duplicated code because this is the more common case.
- *
- *    Parameter:  The connection this PDU should be associated with.
- *
- *    Returns:    The PDU.
- */
-
-pdu_t *
-get_pdu(connection_t *conn)
-{
-	pdu_t *pdu;
-
-	do {
-		CS_BEGIN;
-		pdu = TAILQ_FIRST(&conn->pdu_pool);
-		if (pdu != NULL) {
-			TAILQ_REMOVE(&conn->pdu_pool, pdu, chain);
-		}
-		CS_END;
-		DEB(100, ("get_pdu: pdu = %p\n", pdu));
-		if (pdu == NULL) {
-			if (conn->terminating)
-				return NULL;
-
-			PDEBOUT(("Waiting for PDU!\n"));
-			tsleep(&conn->pdu_pool, PWAIT, "get_pdu", 0);
-		}
-	} while (pdu == NULL);
-
-	memset(pdu, 0, sizeof(pdu_t));
-	pdu->connection = conn;
-	pdu->disp = PDUDISP_FREE;
-
-	return pdu;
-}
-
-/*
  * free_pdu:
  *    Put a PDU back onto the free list.
  *
@@ -534,6 +496,7 @@
 {
 	connection_t *conn = pdu->connection;
 	pdu_disp_t pdisp;
+	int s;
 
 	if (PDUDISP_UNUSED == (pdisp = pdu->disp))
 		return;
@@ -551,9 +514,10 @@
 	if (pdu->temp_data)
 		free(pdu->temp_data, M_TEMP);
 
-	CS_BEGIN;
+	s = splbio();
 	TAILQ_INSERT_TAIL(&conn->pdu_pool, pdu, chain);
-	CS_END;
+	splx(s);
+
 	wakeup(&conn->pdu_pool);
 }