Commit | Line | Data |
---|---|---|
c942fddf | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
e48354ce NB |
2 | /******************************************************************************* |
3 | * This file contains error recovery level one used by the iSCSI Target driver. | |
4 | * | |
4c76251e | 5 | * (c) Copyright 2007-2013 Datera, Inc. |
e48354ce NB |
6 | * |
7 | * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> | |
8 | * | |
e48354ce NB |
9 | ******************************************************************************/ |
10 | ||
11 | #include <linux/list.h> | |
8dcf07be | 12 | #include <linux/slab.h> |
e48354ce NB |
13 | #include <scsi/iscsi_proto.h> |
14 | #include <target/target_core_base.h> | |
c4795fb2 | 15 | #include <target/target_core_fabric.h> |
3e1c81a9 | 16 | #include <target/iscsi/iscsi_transport.h> |
e48354ce | 17 | |
67f091f2 | 18 | #include <target/iscsi/iscsi_target_core.h> |
e48354ce NB |
19 | #include "iscsi_target_seq_pdu_list.h" |
20 | #include "iscsi_target_datain_values.h" | |
21 | #include "iscsi_target_device.h" | |
22 | #include "iscsi_target_tpg.h" | |
23 | #include "iscsi_target_util.h" | |
24 | #include "iscsi_target_erl0.h" | |
25 | #include "iscsi_target_erl1.h" | |
26 | #include "iscsi_target_erl2.h" | |
27 | #include "iscsi_target.h" | |
28 | ||
919765e9 | 29 | #define OFFLOAD_BUF_SIZE 32768U |
e48354ce NB |
30 | |
31 | /* | |
32 | * Used to dump excess datain payload for certain error recovery | |
33 | * situations. Receive in OFFLOAD_BUF_SIZE max of datain per rx_data(). | |
34 | * | |
35 | * dump_padding_digest denotes if padding and data digests need | |
36 | * to be dumped. | |
37 | */ | |
38 | int iscsit_dump_data_payload( | |
39 | struct iscsi_conn *conn, | |
40 | u32 buf_len, | |
41 | int dump_padding_digest) | |
42 | { | |
658c3122 | 43 | char *buf; |
e48354ce | 44 | int ret = DATAOUT_WITHIN_COMMAND_RECOVERY, rx_got; |
658c3122 | 45 | u32 length, offset = 0, size; |
e48354ce NB |
46 | struct kvec iov; |
47 | ||
3e1c81a9 NB |
48 | if (conn->sess->sess_ops->RDMAExtensions) |
49 | return 0; | |
50 | ||
658c3122 BVA |
51 | if (dump_padding_digest) { |
52 | buf_len = ALIGN(buf_len, 4); | |
53 | if (conn->conn_ops->DataDigest) | |
54 | buf_len += ISCSI_CRC_LEN; | |
55 | } | |
56 | ||
919765e9 | 57 | length = min(buf_len, OFFLOAD_BUF_SIZE); |
e48354ce NB |
58 | |
59 | buf = kzalloc(length, GFP_ATOMIC); | |
60 | if (!buf) { | |
61 | pr_err("Unable to allocate %u bytes for offload" | |
62 | " buffer.\n", length); | |
63 | return -1; | |
64 | } | |
65 | memset(&iov, 0, sizeof(struct kvec)); | |
66 | ||
67 | while (offset < buf_len) { | |
919765e9 | 68 | size = min(buf_len - offset, length); |
e48354ce NB |
69 | |
70 | iov.iov_len = size; | |
71 | iov.iov_base = buf; | |
72 | ||
73 | rx_got = rx_data(conn, &iov, 1, size); | |
74 | if (rx_got != size) { | |
75 | ret = DATAOUT_CANNOT_RECOVER; | |
658c3122 | 76 | break; |
e48354ce NB |
77 | } |
78 | ||
79 | offset += size; | |
80 | } | |
81 | ||
e48354ce NB |
82 | kfree(buf); |
83 | return ret; | |
84 | } | |
85 | ||
86 | /* | |
87 | * Used for retransmitting R2Ts from a R2T SNACK request. | |
88 | */ | |
89 | static int iscsit_send_recovery_r2t_for_snack( | |
90 | struct iscsi_cmd *cmd, | |
91 | struct iscsi_r2t *r2t) | |
92 | { | |
93 | /* | |
94 | * If the struct iscsi_r2t has not been sent yet, we can safely | |
95 | * ignore retransmission | |
96 | * of the R2TSN in question. | |
97 | */ | |
98 | spin_lock_bh(&cmd->r2t_lock); | |
99 | if (!r2t->sent_r2t) { | |
100 | spin_unlock_bh(&cmd->r2t_lock); | |
101 | return 0; | |
102 | } | |
103 | r2t->sent_r2t = 0; | |
104 | spin_unlock_bh(&cmd->r2t_lock); | |
105 | ||
106 | iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T); | |
107 | ||
108 | return 0; | |
109 | } | |
110 | ||
111 | static int iscsit_handle_r2t_snack( | |
112 | struct iscsi_cmd *cmd, | |
113 | unsigned char *buf, | |
114 | u32 begrun, | |
115 | u32 runlength) | |
116 | { | |
117 | u32 last_r2tsn; | |
118 | struct iscsi_r2t *r2t; | |
119 | ||
120 | /* | |
121 | * Make sure the initiator is not requesting retransmission | |
122 | * of R2TSNs already acknowledged by a TMR TASK_REASSIGN. | |
123 | */ | |
124 | if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && | |
125 | (begrun <= cmd->acked_data_sn)) { | |
126 | pr_err("ITT: 0x%08x, R2T SNACK requesting" | |
127 | " retransmission of R2TSN: 0x%08x to 0x%08x but already" | |
128 | " acked to R2TSN: 0x%08x by TMR TASK_REASSIGN," | |
129 | " protocol error.\n", cmd->init_task_tag, begrun, | |
130 | (begrun + runlength), cmd->acked_data_sn); | |
131 | ||
5a342521 | 132 | return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); |
e48354ce NB |
133 | } |
134 | ||
135 | if (runlength) { | |
136 | if ((begrun + runlength) > cmd->r2t_sn) { | |
137 | pr_err("Command ITT: 0x%08x received R2T SNACK" | |
138 | " with BegRun: 0x%08x, RunLength: 0x%08x, exceeds" | |
139 | " current R2TSN: 0x%08x, protocol error.\n", | |
140 | cmd->init_task_tag, begrun, runlength, cmd->r2t_sn); | |
ba159914 NB |
141 | return iscsit_reject_cmd(cmd, |
142 | ISCSI_REASON_BOOKMARK_INVALID, buf); | |
e48354ce NB |
143 | } |
144 | last_r2tsn = (begrun + runlength); | |
145 | } else | |
146 | last_r2tsn = cmd->r2t_sn; | |
147 | ||
148 | while (begrun < last_r2tsn) { | |
149 | r2t = iscsit_get_holder_for_r2tsn(cmd, begrun); | |
150 | if (!r2t) | |
151 | return -1; | |
152 | if (iscsit_send_recovery_r2t_for_snack(cmd, r2t) < 0) | |
153 | return -1; | |
154 | ||
155 | begrun++; | |
156 | } | |
157 | ||
158 | return 0; | |
159 | } | |
160 | ||
161 | /* | |
162 | * Generates Offsets and NextBurstLength based on Begrun and Runlength | |
163 | * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. | |
164 | * | |
165 | * For DataSequenceInOrder=Yes and DataPDUInOrder=[Yes,No] only. | |
166 | * | |
167 | * FIXME: How is this handled for a RData SNACK? | |
168 | */ | |
169 | int iscsit_create_recovery_datain_values_datasequenceinorder_yes( | |
170 | struct iscsi_cmd *cmd, | |
171 | struct iscsi_datain_req *dr) | |
172 | { | |
173 | u32 data_sn = 0, data_sn_count = 0; | |
174 | u32 pdu_start = 0, seq_no = 0; | |
175 | u32 begrun = dr->begrun; | |
176 | struct iscsi_conn *conn = cmd->conn; | |
177 | ||
178 | while (begrun > data_sn++) { | |
179 | data_sn_count++; | |
180 | if ((dr->next_burst_len + | |
181 | conn->conn_ops->MaxRecvDataSegmentLength) < | |
182 | conn->sess->sess_ops->MaxBurstLength) { | |
183 | dr->read_data_done += | |
184 | conn->conn_ops->MaxRecvDataSegmentLength; | |
185 | dr->next_burst_len += | |
186 | conn->conn_ops->MaxRecvDataSegmentLength; | |
187 | } else { | |
188 | dr->read_data_done += | |
189 | (conn->sess->sess_ops->MaxBurstLength - | |
190 | dr->next_burst_len); | |
191 | dr->next_burst_len = 0; | |
192 | pdu_start += data_sn_count; | |
193 | data_sn_count = 0; | |
194 | seq_no++; | |
195 | } | |
196 | } | |
197 | ||
198 | if (!conn->sess->sess_ops->DataPDUInOrder) { | |
199 | cmd->seq_no = seq_no; | |
200 | cmd->pdu_start = pdu_start; | |
201 | cmd->pdu_send_order = data_sn_count; | |
202 | } | |
203 | ||
204 | return 0; | |
205 | } | |
206 | ||
207 | /* | |
208 | * Generates Offsets and NextBurstLength based on Begrun and Runlength | |
209 | * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN. | |
210 | * | |
211 | * For DataSequenceInOrder=No and DataPDUInOrder=[Yes,No] only. | |
212 | * | |
213 | * FIXME: How is this handled for a RData SNACK? | |
214 | */ | |
215 | int iscsit_create_recovery_datain_values_datasequenceinorder_no( | |
216 | struct iscsi_cmd *cmd, | |
217 | struct iscsi_datain_req *dr) | |
218 | { | |
219 | int found_seq = 0, i; | |
220 | u32 data_sn, read_data_done = 0, seq_send_order = 0; | |
221 | u32 begrun = dr->begrun; | |
222 | u32 runlength = dr->runlength; | |
223 | struct iscsi_conn *conn = cmd->conn; | |
224 | struct iscsi_seq *first_seq = NULL, *seq = NULL; | |
225 | ||
226 | if (!cmd->seq_list) { | |
227 | pr_err("struct iscsi_cmd->seq_list is NULL!\n"); | |
228 | return -1; | |
229 | } | |
230 | ||
231 | /* | |
232 | * Calculate read_data_done for all sequences containing a | |
233 | * first_datasn and last_datasn less than the BegRun. | |
234 | * | |
235 | * Locate the struct iscsi_seq the BegRun lies within and calculate | |
236 | * NextBurstLenghth up to the DataSN based on MaxRecvDataSegmentLength. | |
237 | * | |
238 | * Also use struct iscsi_seq->seq_send_order to determine where to start. | |
239 | */ | |
240 | for (i = 0; i < cmd->seq_count; i++) { | |
241 | seq = &cmd->seq_list[i]; | |
242 | ||
243 | if (!seq->seq_send_order) | |
244 | first_seq = seq; | |
245 | ||
246 | /* | |
247 | * No data has been transferred for this DataIN sequence, so the | |
248 | * seq->first_datasn and seq->last_datasn have not been set. | |
249 | */ | |
250 | if (!seq->sent) { | |
e48354ce NB |
251 | pr_err("Ignoring non-sent sequence 0x%08x ->" |
252 | " 0x%08x\n\n", seq->first_datasn, | |
253 | seq->last_datasn); | |
e48354ce NB |
254 | continue; |
255 | } | |
256 | ||
257 | /* | |
258 | * This DataIN sequence is precedes the received BegRun, add the | |
259 | * total xfer_len of the sequence to read_data_done and reset | |
260 | * seq->pdu_send_order. | |
261 | */ | |
262 | if ((seq->first_datasn < begrun) && | |
263 | (seq->last_datasn < begrun)) { | |
e48354ce NB |
264 | pr_err("Pre BegRun sequence 0x%08x ->" |
265 | " 0x%08x\n", seq->first_datasn, | |
266 | seq->last_datasn); | |
8b1e1244 | 267 | |
e48354ce NB |
268 | read_data_done += cmd->seq_list[i].xfer_len; |
269 | seq->next_burst_len = seq->pdu_send_order = 0; | |
270 | continue; | |
271 | } | |
272 | ||
273 | /* | |
274 | * The BegRun lies within this DataIN sequence. | |
275 | */ | |
276 | if ((seq->first_datasn <= begrun) && | |
277 | (seq->last_datasn >= begrun)) { | |
e48354ce NB |
278 | pr_err("Found sequence begrun: 0x%08x in" |
279 | " 0x%08x -> 0x%08x\n", begrun, | |
280 | seq->first_datasn, seq->last_datasn); | |
8b1e1244 | 281 | |
e48354ce NB |
282 | seq_send_order = seq->seq_send_order; |
283 | data_sn = seq->first_datasn; | |
284 | seq->next_burst_len = seq->pdu_send_order = 0; | |
285 | found_seq = 1; | |
286 | ||
287 | /* | |
288 | * For DataPDUInOrder=Yes, while the first DataSN of | |
289 | * the sequence is less than the received BegRun, add | |
290 | * the MaxRecvDataSegmentLength to read_data_done and | |
291 | * to the sequence's next_burst_len; | |
292 | * | |
293 | * For DataPDUInOrder=No, while the first DataSN of the | |
294 | * sequence is less than the received BegRun, find the | |
295 | * struct iscsi_pdu of the DataSN in question and add the | |
296 | * MaxRecvDataSegmentLength to read_data_done and to the | |
297 | * sequence's next_burst_len; | |
298 | */ | |
299 | if (conn->sess->sess_ops->DataPDUInOrder) { | |
300 | while (data_sn < begrun) { | |
301 | seq->pdu_send_order++; | |
302 | read_data_done += | |
303 | conn->conn_ops->MaxRecvDataSegmentLength; | |
304 | seq->next_burst_len += | |
305 | conn->conn_ops->MaxRecvDataSegmentLength; | |
306 | data_sn++; | |
307 | } | |
308 | } else { | |
309 | int j; | |
310 | struct iscsi_pdu *pdu; | |
311 | ||
312 | while (data_sn < begrun) { | |
313 | seq->pdu_send_order++; | |
314 | ||
315 | for (j = 0; j < seq->pdu_count; j++) { | |
316 | pdu = &cmd->pdu_list[ | |
317 | seq->pdu_start + j]; | |
318 | if (pdu->data_sn == data_sn) { | |
319 | read_data_done += | |
320 | pdu->length; | |
321 | seq->next_burst_len += | |
322 | pdu->length; | |
323 | } | |
324 | } | |
325 | data_sn++; | |
326 | } | |
327 | } | |
328 | continue; | |
329 | } | |
330 | ||
331 | /* | |
332 | * This DataIN sequence is larger than the received BegRun, | |
333 | * reset seq->pdu_send_order and continue. | |
334 | */ | |
335 | if ((seq->first_datasn > begrun) || | |
336 | (seq->last_datasn > begrun)) { | |
e48354ce NB |
337 | pr_err("Post BegRun sequence 0x%08x -> 0x%08x\n", |
338 | seq->first_datasn, seq->last_datasn); | |
8b1e1244 | 339 | |
e48354ce NB |
340 | seq->next_burst_len = seq->pdu_send_order = 0; |
341 | continue; | |
342 | } | |
343 | } | |
344 | ||
345 | if (!found_seq) { | |
346 | if (!begrun) { | |
347 | if (!first_seq) { | |
348 | pr_err("ITT: 0x%08x, Begrun: 0x%08x" | |
349 | " but first_seq is NULL\n", | |
350 | cmd->init_task_tag, begrun); | |
351 | return -1; | |
352 | } | |
353 | seq_send_order = first_seq->seq_send_order; | |
354 | seq->next_burst_len = seq->pdu_send_order = 0; | |
355 | goto done; | |
356 | } | |
357 | ||
358 | pr_err("Unable to locate struct iscsi_seq for ITT: 0x%08x," | |
359 | " BegRun: 0x%08x, RunLength: 0x%08x while" | |
360 | " DataSequenceInOrder=No and DataPDUInOrder=%s.\n", | |
361 | cmd->init_task_tag, begrun, runlength, | |
362 | (conn->sess->sess_ops->DataPDUInOrder) ? "Yes" : "No"); | |
363 | return -1; | |
364 | } | |
365 | ||
366 | done: | |
367 | dr->read_data_done = read_data_done; | |
368 | dr->seq_send_order = seq_send_order; | |
369 | ||
370 | return 0; | |
371 | } | |
372 | ||
373 | static int iscsit_handle_recovery_datain( | |
374 | struct iscsi_cmd *cmd, | |
375 | unsigned char *buf, | |
376 | u32 begrun, | |
377 | u32 runlength) | |
378 | { | |
379 | struct iscsi_conn *conn = cmd->conn; | |
380 | struct iscsi_datain_req *dr; | |
381 | struct se_cmd *se_cmd = &cmd->se_cmd; | |
382 | ||
7d680f3b | 383 | if (!(se_cmd->transport_state & CMD_T_COMPLETE)) { |
e48354ce NB |
384 | pr_err("Ignoring ITT: 0x%08x Data SNACK\n", |
385 | cmd->init_task_tag); | |
386 | return 0; | |
387 | } | |
388 | ||
389 | /* | |
390 | * Make sure the initiator is not requesting retransmission | |
391 | * of DataSNs already acknowledged by a Data ACK SNACK. | |
392 | */ | |
393 | if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) && | |
394 | (begrun <= cmd->acked_data_sn)) { | |
395 | pr_err("ITT: 0x%08x, Data SNACK requesting" | |
396 | " retransmission of DataSN: 0x%08x to 0x%08x but" | |
397 | " already acked to DataSN: 0x%08x by Data ACK SNACK," | |
398 | " protocol error.\n", cmd->init_task_tag, begrun, | |
399 | (begrun + runlength), cmd->acked_data_sn); | |
400 | ||
ba159914 | 401 | return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); |
e48354ce NB |
402 | } |
403 | ||
404 | /* | |
405 | * Make sure BegRun and RunLength in the Data SNACK are sane. | |
406 | * Note: (cmd->data_sn - 1) will carry the maximum DataSN sent. | |
407 | */ | |
408 | if ((begrun + runlength) > (cmd->data_sn - 1)) { | |
409 | pr_err("Initiator requesting BegRun: 0x%08x, RunLength" | |
410 | ": 0x%08x greater than maximum DataSN: 0x%08x.\n", | |
411 | begrun, runlength, (cmd->data_sn - 1)); | |
ba159914 NB |
412 | return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, |
413 | buf); | |
e48354ce NB |
414 | } |
415 | ||
416 | dr = iscsit_allocate_datain_req(); | |
417 | if (!dr) | |
ba159914 NB |
418 | return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, |
419 | buf); | |
e48354ce NB |
420 | |
421 | dr->data_sn = dr->begrun = begrun; | |
422 | dr->runlength = runlength; | |
423 | dr->generate_recovery_values = 1; | |
424 | dr->recovery = DATAIN_WITHIN_COMMAND_RECOVERY; | |
425 | ||
426 | iscsit_attach_datain_req(cmd, dr); | |
427 | ||
428 | cmd->i_state = ISTATE_SEND_DATAIN; | |
429 | iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); | |
430 | ||
431 | return 0; | |
432 | } | |
433 | ||
434 | int iscsit_handle_recovery_datain_or_r2t( | |
435 | struct iscsi_conn *conn, | |
436 | unsigned char *buf, | |
66c7db68 | 437 | itt_t init_task_tag, |
e48354ce NB |
438 | u32 targ_xfer_tag, |
439 | u32 begrun, | |
440 | u32 runlength) | |
441 | { | |
442 | struct iscsi_cmd *cmd; | |
443 | ||
444 | cmd = iscsit_find_cmd_from_itt(conn, init_task_tag); | |
445 | if (!cmd) | |
446 | return 0; | |
447 | ||
448 | /* | |
449 | * FIXME: This will not work for bidi commands. | |
450 | */ | |
451 | switch (cmd->data_direction) { | |
452 | case DMA_TO_DEVICE: | |
453 | return iscsit_handle_r2t_snack(cmd, buf, begrun, runlength); | |
454 | case DMA_FROM_DEVICE: | |
455 | return iscsit_handle_recovery_datain(cmd, buf, begrun, | |
456 | runlength); | |
457 | default: | |
458 | pr_err("Unknown cmd->data_direction: 0x%02x\n", | |
459 | cmd->data_direction); | |
460 | return -1; | |
461 | } | |
462 | ||
463 | return 0; | |
464 | } | |
465 | ||
466 | /* #warning FIXME: Status SNACK needs to be dependent on OPCODE!!! */ | |
467 | int iscsit_handle_status_snack( | |
468 | struct iscsi_conn *conn, | |
66c7db68 | 469 | itt_t init_task_tag, |
e48354ce NB |
470 | u32 targ_xfer_tag, |
471 | u32 begrun, | |
472 | u32 runlength) | |
473 | { | |
474 | struct iscsi_cmd *cmd = NULL; | |
475 | u32 last_statsn; | |
476 | int found_cmd; | |
477 | ||
752d8680 NB |
478 | if (!begrun) { |
479 | begrun = conn->exp_statsn; | |
480 | } else if (conn->exp_statsn > begrun) { | |
e48354ce NB |
481 | pr_err("Got Status SNACK Begrun: 0x%08x, RunLength:" |
482 | " 0x%08x but already got ExpStatSN: 0x%08x on CID:" | |
483 | " %hu.\n", begrun, runlength, conn->exp_statsn, | |
484 | conn->cid); | |
485 | return 0; | |
486 | } | |
487 | ||
488 | last_statsn = (!runlength) ? conn->stat_sn : (begrun + runlength); | |
489 | ||
490 | while (begrun < last_statsn) { | |
491 | found_cmd = 0; | |
492 | ||
493 | spin_lock_bh(&conn->cmd_lock); | |
2fbb471e | 494 | list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { |
e48354ce NB |
495 | if (cmd->stat_sn == begrun) { |
496 | found_cmd = 1; | |
497 | break; | |
498 | } | |
499 | } | |
500 | spin_unlock_bh(&conn->cmd_lock); | |
501 | ||
502 | if (!found_cmd) { | |
503 | pr_err("Unable to find StatSN: 0x%08x for" | |
504 | " a Status SNACK, assuming this was a" | |
505 | " protactic SNACK for an untransmitted" | |
506 | " StatSN, ignoring.\n", begrun); | |
507 | begrun++; | |
508 | continue; | |
509 | } | |
510 | ||
511 | spin_lock_bh(&cmd->istate_lock); | |
512 | if (cmd->i_state == ISTATE_SEND_DATAIN) { | |
513 | spin_unlock_bh(&cmd->istate_lock); | |
514 | pr_err("Ignoring Status SNACK for BegRun:" | |
515 | " 0x%08x, RunLength: 0x%08x, assuming this was" | |
516 | " a protactic SNACK for an untransmitted" | |
517 | " StatSN\n", begrun, runlength); | |
518 | begrun++; | |
519 | continue; | |
520 | } | |
521 | spin_unlock_bh(&cmd->istate_lock); | |
522 | ||
523 | cmd->i_state = ISTATE_SEND_STATUS_RECOVERY; | |
524 | iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); | |
525 | begrun++; | |
526 | } | |
527 | ||
528 | return 0; | |
529 | } | |
530 | ||
531 | int iscsit_handle_data_ack( | |
532 | struct iscsi_conn *conn, | |
533 | u32 targ_xfer_tag, | |
534 | u32 begrun, | |
535 | u32 runlength) | |
536 | { | |
537 | struct iscsi_cmd *cmd = NULL; | |
538 | ||
539 | cmd = iscsit_find_cmd_from_ttt(conn, targ_xfer_tag); | |
540 | if (!cmd) { | |
541 | pr_err("Data ACK SNACK for TTT: 0x%08x is" | |
542 | " invalid.\n", targ_xfer_tag); | |
543 | return -1; | |
544 | } | |
545 | ||
546 | if (begrun <= cmd->acked_data_sn) { | |
547 | pr_err("ITT: 0x%08x Data ACK SNACK BegRUN: 0x%08x is" | |
548 | " less than the already acked DataSN: 0x%08x.\n", | |
549 | cmd->init_task_tag, begrun, cmd->acked_data_sn); | |
550 | return -1; | |
551 | } | |
552 | ||
553 | /* | |
554 | * For Data ACK SNACK, BegRun is the next expected DataSN. | |
555 | * (see iSCSI v19: 10.16.6) | |
556 | */ | |
557 | cmd->cmd_flags |= ICF_GOT_DATACK_SNACK; | |
558 | cmd->acked_data_sn = (begrun - 1); | |
559 | ||
560 | pr_debug("Received Data ACK SNACK for ITT: 0x%08x," | |
561 | " updated acked DataSN to 0x%08x.\n", | |
562 | cmd->init_task_tag, cmd->acked_data_sn); | |
563 | ||
564 | return 0; | |
565 | } | |
566 | ||
567 | static int iscsit_send_recovery_r2t( | |
568 | struct iscsi_cmd *cmd, | |
569 | u32 offset, | |
570 | u32 xfer_len) | |
571 | { | |
572 | int ret; | |
573 | ||
574 | spin_lock_bh(&cmd->r2t_lock); | |
575 | ret = iscsit_add_r2t_to_list(cmd, offset, xfer_len, 1, 0); | |
576 | spin_unlock_bh(&cmd->r2t_lock); | |
577 | ||
578 | return ret; | |
579 | } | |
580 | ||
581 | int iscsit_dataout_datapduinorder_no_fbit( | |
582 | struct iscsi_cmd *cmd, | |
583 | struct iscsi_pdu *pdu) | |
584 | { | |
585 | int i, send_recovery_r2t = 0, recovery = 0; | |
586 | u32 length = 0, offset = 0, pdu_count = 0, xfer_len = 0; | |
587 | struct iscsi_conn *conn = cmd->conn; | |
588 | struct iscsi_pdu *first_pdu = NULL; | |
589 | ||
590 | /* | |
591 | * Get an struct iscsi_pdu pointer to the first PDU, and total PDU count | |
592 | * of the DataOUT sequence. | |
593 | */ | |
594 | if (conn->sess->sess_ops->DataSequenceInOrder) { | |
595 | for (i = 0; i < cmd->pdu_count; i++) { | |
596 | if (cmd->pdu_list[i].seq_no == pdu->seq_no) { | |
597 | if (!first_pdu) | |
598 | first_pdu = &cmd->pdu_list[i]; | |
5a342521 BVA |
599 | xfer_len += cmd->pdu_list[i].length; |
600 | pdu_count++; | |
e48354ce NB |
601 | } else if (pdu_count) |
602 | break; | |
603 | } | |
604 | } else { | |
605 | struct iscsi_seq *seq = cmd->seq_ptr; | |
606 | ||
607 | first_pdu = &cmd->pdu_list[seq->pdu_start]; | |
608 | pdu_count = seq->pdu_count; | |
609 | } | |
610 | ||
611 | if (!first_pdu || !pdu_count) | |
612 | return DATAOUT_CANNOT_RECOVER; | |
613 | ||
614 | /* | |
615 | * Loop through the ending DataOUT Sequence checking each struct iscsi_pdu. | |
616 | * The following ugly logic does batching of not received PDUs. | |
617 | */ | |
618 | for (i = 0; i < pdu_count; i++) { | |
619 | if (first_pdu[i].status == ISCSI_PDU_RECEIVED_OK) { | |
620 | if (!send_recovery_r2t) | |
621 | continue; | |
622 | ||
623 | if (iscsit_send_recovery_r2t(cmd, offset, length) < 0) | |
624 | return DATAOUT_CANNOT_RECOVER; | |
625 | ||
626 | send_recovery_r2t = length = offset = 0; | |
627 | continue; | |
628 | } | |
629 | /* | |
630 | * Set recovery = 1 for any missing, CRC failed, or timed | |
631 | * out PDUs to let the DataOUT logic know that this sequence | |
632 | * has not been completed yet. | |
633 | * | |
634 | * Also, only send a Recovery R2T for ISCSI_PDU_NOT_RECEIVED. | |
635 | * We assume if the PDU either failed CRC or timed out | |
636 | * that a Recovery R2T has already been sent. | |
637 | */ | |
638 | recovery = 1; | |
639 | ||
640 | if (first_pdu[i].status != ISCSI_PDU_NOT_RECEIVED) | |
641 | continue; | |
642 | ||
643 | if (!offset) | |
644 | offset = first_pdu[i].offset; | |
645 | length += first_pdu[i].length; | |
646 | ||
647 | send_recovery_r2t = 1; | |
648 | } | |
649 | ||
650 | if (send_recovery_r2t) | |
651 | if (iscsit_send_recovery_r2t(cmd, offset, length) < 0) | |
652 | return DATAOUT_CANNOT_RECOVER; | |
653 | ||
654 | return (!recovery) ? DATAOUT_NORMAL : DATAOUT_WITHIN_COMMAND_RECOVERY; | |
655 | } | |
656 | ||
657 | static int iscsit_recalculate_dataout_values( | |
658 | struct iscsi_cmd *cmd, | |
659 | u32 pdu_offset, | |
660 | u32 pdu_length, | |
661 | u32 *r2t_offset, | |
662 | u32 *r2t_length) | |
663 | { | |
664 | int i; | |
665 | struct iscsi_conn *conn = cmd->conn; | |
666 | struct iscsi_pdu *pdu = NULL; | |
667 | ||
668 | if (conn->sess->sess_ops->DataSequenceInOrder) { | |
669 | cmd->data_sn = 0; | |
670 | ||
671 | if (conn->sess->sess_ops->DataPDUInOrder) { | |
672 | *r2t_offset = cmd->write_data_done; | |
673 | *r2t_length = (cmd->seq_end_offset - | |
674 | cmd->write_data_done); | |
675 | return 0; | |
676 | } | |
677 | ||
678 | *r2t_offset = cmd->seq_start_offset; | |
679 | *r2t_length = (cmd->seq_end_offset - cmd->seq_start_offset); | |
680 | ||
681 | for (i = 0; i < cmd->pdu_count; i++) { | |
682 | pdu = &cmd->pdu_list[i]; | |
683 | ||
684 | if (pdu->status != ISCSI_PDU_RECEIVED_OK) | |
685 | continue; | |
686 | ||
687 | if ((pdu->offset >= cmd->seq_start_offset) && | |
688 | ((pdu->offset + pdu->length) <= | |
689 | cmd->seq_end_offset)) { | |
690 | if (!cmd->unsolicited_data) | |
691 | cmd->next_burst_len -= pdu->length; | |
692 | else | |
693 | cmd->first_burst_len -= pdu->length; | |
694 | ||
695 | cmd->write_data_done -= pdu->length; | |
696 | pdu->status = ISCSI_PDU_NOT_RECEIVED; | |
697 | } | |
698 | } | |
699 | } else { | |
700 | struct iscsi_seq *seq = NULL; | |
701 | ||
702 | seq = iscsit_get_seq_holder(cmd, pdu_offset, pdu_length); | |
703 | if (!seq) | |
704 | return -1; | |
705 | ||
706 | *r2t_offset = seq->orig_offset; | |
707 | *r2t_length = seq->xfer_len; | |
708 | ||
709 | cmd->write_data_done -= (seq->offset - seq->orig_offset); | |
710 | if (cmd->immediate_data) | |
711 | cmd->first_burst_len = cmd->write_data_done; | |
712 | ||
713 | seq->data_sn = 0; | |
714 | seq->offset = seq->orig_offset; | |
715 | seq->next_burst_len = 0; | |
716 | seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY; | |
717 | ||
718 | if (conn->sess->sess_ops->DataPDUInOrder) | |
719 | return 0; | |
720 | ||
721 | for (i = 0; i < seq->pdu_count; i++) { | |
722 | pdu = &cmd->pdu_list[i+seq->pdu_start]; | |
723 | ||
724 | if (pdu->status != ISCSI_PDU_RECEIVED_OK) | |
725 | continue; | |
726 | ||
727 | pdu->status = ISCSI_PDU_NOT_RECEIVED; | |
728 | } | |
729 | } | |
730 | ||
731 | return 0; | |
732 | } | |
733 | ||
734 | int iscsit_recover_dataout_sequence( | |
735 | struct iscsi_cmd *cmd, | |
736 | u32 pdu_offset, | |
737 | u32 pdu_length) | |
738 | { | |
739 | u32 r2t_length = 0, r2t_offset = 0; | |
740 | ||
741 | spin_lock_bh(&cmd->istate_lock); | |
742 | cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; | |
743 | spin_unlock_bh(&cmd->istate_lock); | |
744 | ||
745 | if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, | |
746 | &r2t_offset, &r2t_length) < 0) | |
747 | return DATAOUT_CANNOT_RECOVER; | |
748 | ||
749 | iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length); | |
750 | ||
751 | return DATAOUT_WITHIN_COMMAND_RECOVERY; | |
752 | } | |
753 | ||
754 | static struct iscsi_ooo_cmdsn *iscsit_allocate_ooo_cmdsn(void) | |
755 | { | |
756 | struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL; | |
757 | ||
758 | ooo_cmdsn = kmem_cache_zalloc(lio_ooo_cache, GFP_ATOMIC); | |
759 | if (!ooo_cmdsn) { | |
760 | pr_err("Unable to allocate memory for" | |
761 | " struct iscsi_ooo_cmdsn.\n"); | |
762 | return NULL; | |
763 | } | |
764 | INIT_LIST_HEAD(&ooo_cmdsn->ooo_list); | |
765 | ||
766 | return ooo_cmdsn; | |
767 | } | |
768 | ||
e48354ce NB |
769 | static int iscsit_attach_ooo_cmdsn( |
770 | struct iscsi_session *sess, | |
771 | struct iscsi_ooo_cmdsn *ooo_cmdsn) | |
772 | { | |
773 | struct iscsi_ooo_cmdsn *ooo_tail, *ooo_tmp; | |
618baaf7 BVA |
774 | |
775 | lockdep_assert_held(&sess->cmdsn_mutex); | |
776 | ||
e48354ce NB |
777 | /* |
778 | * We attach the struct iscsi_ooo_cmdsn entry to the out of order | |
779 | * list in increasing CmdSN order. | |
780 | * This allows iscsi_execute_ooo_cmdsns() to detect any | |
781 | * additional CmdSN holes while performing delayed execution. | |
782 | */ | |
783 | if (list_empty(&sess->sess_ooo_cmdsn_list)) | |
784 | list_add_tail(&ooo_cmdsn->ooo_list, | |
785 | &sess->sess_ooo_cmdsn_list); | |
786 | else { | |
787 | ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, | |
788 | typeof(*ooo_tail), ooo_list); | |
789 | /* | |
790 | * CmdSN is greater than the tail of the list. | |
791 | */ | |
3eccfdb0 | 792 | if (iscsi_sna_lt(ooo_tail->cmdsn, ooo_cmdsn->cmdsn)) |
e48354ce NB |
793 | list_add_tail(&ooo_cmdsn->ooo_list, |
794 | &sess->sess_ooo_cmdsn_list); | |
795 | else { | |
796 | /* | |
797 | * CmdSN is either lower than the head, or somewhere | |
798 | * in the middle. | |
799 | */ | |
800 | list_for_each_entry(ooo_tmp, &sess->sess_ooo_cmdsn_list, | |
801 | ooo_list) { | |
3eccfdb0 | 802 | if (iscsi_sna_lt(ooo_tmp->cmdsn, ooo_cmdsn->cmdsn)) |
e48354ce NB |
803 | continue; |
804 | ||
3eccfdb0 | 805 | /* Insert before this entry */ |
e48354ce | 806 | list_add(&ooo_cmdsn->ooo_list, |
3eccfdb0 | 807 | ooo_tmp->ooo_list.prev); |
e48354ce NB |
808 | break; |
809 | } | |
810 | } | |
811 | } | |
812 | ||
813 | return 0; | |
814 | } | |
815 | ||
816 | /* | |
817 | * Removes an struct iscsi_ooo_cmdsn from a session's list, | |
818 | * called with struct iscsi_session->cmdsn_mutex held. | |
819 | */ | |
820 | void iscsit_remove_ooo_cmdsn( | |
821 | struct iscsi_session *sess, | |
822 | struct iscsi_ooo_cmdsn *ooo_cmdsn) | |
823 | { | |
824 | list_del(&ooo_cmdsn->ooo_list); | |
825 | kmem_cache_free(lio_ooo_cache, ooo_cmdsn); | |
826 | } | |
827 | ||
828 | void iscsit_clear_ooo_cmdsns_for_conn(struct iscsi_conn *conn) | |
829 | { | |
830 | struct iscsi_ooo_cmdsn *ooo_cmdsn; | |
831 | struct iscsi_session *sess = conn->sess; | |
832 | ||
833 | mutex_lock(&sess->cmdsn_mutex); | |
834 | list_for_each_entry(ooo_cmdsn, &sess->sess_ooo_cmdsn_list, ooo_list) { | |
835 | if (ooo_cmdsn->cid != conn->cid) | |
836 | continue; | |
837 | ||
838 | ooo_cmdsn->cmd = NULL; | |
839 | } | |
840 | mutex_unlock(&sess->cmdsn_mutex); | |
841 | } | |
842 | ||
e48354ce NB |
843 | int iscsit_execute_ooo_cmdsns(struct iscsi_session *sess) |
844 | { | |
845 | int ooo_count = 0; | |
846 | struct iscsi_cmd *cmd = NULL; | |
847 | struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; | |
848 | ||
618baaf7 BVA |
849 | lockdep_assert_held(&sess->cmdsn_mutex); |
850 | ||
e48354ce NB |
851 | list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, |
852 | &sess->sess_ooo_cmdsn_list, ooo_list) { | |
853 | if (ooo_cmdsn->cmdsn != sess->exp_cmd_sn) | |
854 | continue; | |
855 | ||
856 | if (!ooo_cmdsn->cmd) { | |
857 | sess->exp_cmd_sn++; | |
858 | iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); | |
859 | continue; | |
860 | } | |
861 | ||
862 | cmd = ooo_cmdsn->cmd; | |
863 | cmd->i_state = cmd->deferred_i_state; | |
864 | ooo_count++; | |
865 | sess->exp_cmd_sn++; | |
866 | pr_debug("Executing out of order CmdSN: 0x%08x," | |
867 | " incremented ExpCmdSN to 0x%08x.\n", | |
868 | cmd->cmd_sn, sess->exp_cmd_sn); | |
869 | ||
870 | iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn); | |
871 | ||
872 | if (iscsit_execute_cmd(cmd, 1) < 0) | |
873 | return -1; | |
874 | ||
875 | continue; | |
876 | } | |
877 | ||
878 | return ooo_count; | |
879 | } | |
880 | ||
881 | /* | |
882 | * Called either: | |
883 | * | |
884 | * 1. With sess->cmdsn_mutex held from iscsi_execute_ooo_cmdsns() | |
885 | * or iscsi_check_received_cmdsn(). | |
886 | * 2. With no locks held directly from iscsi_handle_XXX_pdu() functions | |
887 | * for immediate commands. | |
888 | */ | |
889 | int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo) | |
890 | { | |
891 | struct se_cmd *se_cmd = &cmd->se_cmd; | |
3e1c81a9 | 892 | struct iscsi_conn *conn = cmd->conn; |
e48354ce NB |
893 | int lr = 0; |
894 | ||
895 | spin_lock_bh(&cmd->istate_lock); | |
896 | if (ooo) | |
897 | cmd->cmd_flags &= ~ICF_OOO_CMDSN; | |
898 | ||
899 | switch (cmd->iscsi_opcode) { | |
900 | case ISCSI_OP_SCSI_CMD: | |
901 | /* | |
902 | * Go ahead and send the CHECK_CONDITION status for | |
de103c93 | 903 | * any SCSI CDB exceptions that may have occurred. |
e48354ce | 904 | */ |
de103c93 CH |
905 | if (cmd->sense_reason) { |
906 | if (cmd->sense_reason == TCM_RESERVATION_CONFLICT) { | |
e48354ce NB |
907 | cmd->i_state = ISTATE_SEND_STATUS; |
908 | spin_unlock_bh(&cmd->istate_lock); | |
909 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, | |
910 | cmd->i_state); | |
911 | return 0; | |
912 | } | |
913 | spin_unlock_bh(&cmd->istate_lock); | |
aaa00cc9 | 914 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
e48354ce | 915 | return 0; |
e48354ce | 916 | return transport_send_check_condition_and_sense(se_cmd, |
de103c93 | 917 | cmd->sense_reason, 0); |
e48354ce NB |
918 | } |
919 | /* | |
920 | * Special case for delayed CmdSN with Immediate | |
921 | * Data and/or Unsolicited Data Out attached. | |
922 | */ | |
923 | if (cmd->immediate_data) { | |
924 | if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { | |
925 | spin_unlock_bh(&cmd->istate_lock); | |
67441b68 CH |
926 | target_execute_cmd(&cmd->se_cmd); |
927 | return 0; | |
e48354ce NB |
928 | } |
929 | spin_unlock_bh(&cmd->istate_lock); | |
930 | ||
931 | if (!(cmd->cmd_flags & | |
932 | ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { | |
aaa00cc9 | 933 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
e48354ce NB |
934 | return 0; |
935 | ||
936 | iscsit_set_dataout_sequence_values(cmd); | |
3e1c81a9 | 937 | conn->conn_transport->iscsit_get_dataout(conn, cmd, false); |
e48354ce NB |
938 | } |
939 | return 0; | |
940 | } | |
941 | /* | |
942 | * The default handler. | |
943 | */ | |
944 | spin_unlock_bh(&cmd->istate_lock); | |
945 | ||
946 | if ((cmd->data_direction == DMA_TO_DEVICE) && | |
947 | !(cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) { | |
aaa00cc9 | 948 | if (cmd->se_cmd.transport_state & CMD_T_ABORTED) |
e48354ce NB |
949 | return 0; |
950 | ||
0300b114 | 951 | iscsit_set_unsolicited_dataout(cmd); |
e48354ce NB |
952 | } |
953 | return transport_handle_cdb_direct(&cmd->se_cmd); | |
954 | ||
955 | case ISCSI_OP_NOOP_OUT: | |
956 | case ISCSI_OP_TEXT: | |
957 | spin_unlock_bh(&cmd->istate_lock); | |
958 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); | |
959 | break; | |
960 | case ISCSI_OP_SCSI_TMFUNC: | |
de103c93 | 961 | if (cmd->se_cmd.se_tmr_req->response) { |
e48354ce NB |
962 | spin_unlock_bh(&cmd->istate_lock); |
963 | iscsit_add_cmd_to_response_queue(cmd, cmd->conn, | |
964 | cmd->i_state); | |
965 | return 0; | |
966 | } | |
967 | spin_unlock_bh(&cmd->istate_lock); | |
968 | ||
969 | return transport_generic_handle_tmr(&cmd->se_cmd); | |
970 | case ISCSI_OP_LOGOUT: | |
971 | spin_unlock_bh(&cmd->istate_lock); | |
972 | switch (cmd->logout_reason) { | |
973 | case ISCSI_LOGOUT_REASON_CLOSE_SESSION: | |
974 | lr = iscsit_logout_closesession(cmd, cmd->conn); | |
975 | break; | |
976 | case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: | |
977 | lr = iscsit_logout_closeconnection(cmd, cmd->conn); | |
978 | break; | |
979 | case ISCSI_LOGOUT_REASON_RECOVERY: | |
980 | lr = iscsit_logout_removeconnforrecovery(cmd, cmd->conn); | |
981 | break; | |
982 | default: | |
983 | pr_err("Unknown iSCSI Logout Request Code:" | |
984 | " 0x%02x\n", cmd->logout_reason); | |
985 | return -1; | |
986 | } | |
987 | ||
988 | return lr; | |
989 | default: | |
990 | spin_unlock_bh(&cmd->istate_lock); | |
991 | pr_err("Cannot perform out of order execution for" | |
992 | " unknown iSCSI Opcode: 0x%02x\n", cmd->iscsi_opcode); | |
993 | return -1; | |
994 | } | |
995 | ||
996 | return 0; | |
997 | } | |
998 | ||
999 | void iscsit_free_all_ooo_cmdsns(struct iscsi_session *sess) | |
1000 | { | |
1001 | struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp; | |
1002 | ||
1003 | mutex_lock(&sess->cmdsn_mutex); | |
1004 | list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp, | |
1005 | &sess->sess_ooo_cmdsn_list, ooo_list) { | |
1006 | ||
1007 | list_del(&ooo_cmdsn->ooo_list); | |
1008 | kmem_cache_free(lio_ooo_cache, ooo_cmdsn); | |
1009 | } | |
1010 | mutex_unlock(&sess->cmdsn_mutex); | |
1011 | } | |
1012 | ||
1013 | int iscsit_handle_ooo_cmdsn( | |
1014 | struct iscsi_session *sess, | |
1015 | struct iscsi_cmd *cmd, | |
1016 | u32 cmdsn) | |
1017 | { | |
1018 | int batch = 0; | |
1019 | struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL, *ooo_tail = NULL; | |
1020 | ||
1021 | cmd->deferred_i_state = cmd->i_state; | |
1022 | cmd->i_state = ISTATE_DEFERRED_CMD; | |
1023 | cmd->cmd_flags |= ICF_OOO_CMDSN; | |
1024 | ||
1025 | if (list_empty(&sess->sess_ooo_cmdsn_list)) | |
1026 | batch = 1; | |
1027 | else { | |
1028 | ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev, | |
1029 | typeof(*ooo_tail), ooo_list); | |
1030 | if (ooo_tail->cmdsn != (cmdsn - 1)) | |
1031 | batch = 1; | |
1032 | } | |
1033 | ||
1034 | ooo_cmdsn = iscsit_allocate_ooo_cmdsn(); | |
1035 | if (!ooo_cmdsn) | |
561bf158 | 1036 | return -ENOMEM; |
e48354ce NB |
1037 | |
1038 | ooo_cmdsn->cmd = cmd; | |
1039 | ooo_cmdsn->batch_count = (batch) ? | |
1040 | (cmdsn - sess->exp_cmd_sn) : 1; | |
1041 | ooo_cmdsn->cid = cmd->conn->cid; | |
1042 | ooo_cmdsn->exp_cmdsn = sess->exp_cmd_sn; | |
1043 | ooo_cmdsn->cmdsn = cmdsn; | |
1044 | ||
1045 | if (iscsit_attach_ooo_cmdsn(sess, ooo_cmdsn) < 0) { | |
1046 | kmem_cache_free(lio_ooo_cache, ooo_cmdsn); | |
561bf158 | 1047 | return -ENOMEM; |
e48354ce NB |
1048 | } |
1049 | ||
561bf158 | 1050 | return 0; |
e48354ce NB |
1051 | } |
1052 | ||
1053 | static int iscsit_set_dataout_timeout_values( | |
1054 | struct iscsi_cmd *cmd, | |
1055 | u32 *offset, | |
1056 | u32 *length) | |
1057 | { | |
1058 | struct iscsi_conn *conn = cmd->conn; | |
1059 | struct iscsi_r2t *r2t; | |
1060 | ||
1061 | if (cmd->unsolicited_data) { | |
1062 | *offset = 0; | |
1063 | *length = (conn->sess->sess_ops->FirstBurstLength > | |
ebf1d95c AG |
1064 | cmd->se_cmd.data_length) ? |
1065 | cmd->se_cmd.data_length : | |
e48354ce NB |
1066 | conn->sess->sess_ops->FirstBurstLength; |
1067 | return 0; | |
1068 | } | |
1069 | ||
1070 | spin_lock_bh(&cmd->r2t_lock); | |
1071 | if (list_empty(&cmd->cmd_r2t_list)) { | |
1072 | pr_err("cmd->cmd_r2t_list is empty!\n"); | |
1073 | spin_unlock_bh(&cmd->r2t_lock); | |
1074 | return -1; | |
1075 | } | |
1076 | ||
1077 | list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) { | |
1078 | if (r2t->sent_r2t && !r2t->recovery_r2t && !r2t->seq_complete) { | |
1079 | *offset = r2t->offset; | |
1080 | *length = r2t->xfer_len; | |
1081 | spin_unlock_bh(&cmd->r2t_lock); | |
1082 | return 0; | |
1083 | } | |
1084 | } | |
1085 | spin_unlock_bh(&cmd->r2t_lock); | |
1086 | ||
1087 | pr_err("Unable to locate any incomplete DataOUT" | |
1088 | " sequences for ITT: 0x%08x.\n", cmd->init_task_tag); | |
1089 | ||
1090 | return -1; | |
1091 | } | |
1092 | ||
1093 | /* | |
1094 | * NOTE: Called from interrupt (timer) context. | |
1095 | */ | |
f7c9564a | 1096 | void iscsit_handle_dataout_timeout(struct timer_list *t) |
e48354ce NB |
1097 | { |
1098 | u32 pdu_length = 0, pdu_offset = 0; | |
1099 | u32 r2t_length = 0, r2t_offset = 0; | |
f7c9564a | 1100 | struct iscsi_cmd *cmd = from_timer(cmd, t, dataout_timer); |
e48354ce NB |
1101 | struct iscsi_conn *conn = cmd->conn; |
1102 | struct iscsi_session *sess = NULL; | |
1103 | struct iscsi_node_attrib *na; | |
1104 | ||
1105 | iscsit_inc_conn_usage_count(conn); | |
1106 | ||
1107 | spin_lock_bh(&cmd->dataout_timeout_lock); | |
1108 | if (cmd->dataout_timer_flags & ISCSI_TF_STOP) { | |
1109 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1110 | iscsit_dec_conn_usage_count(conn); | |
1111 | return; | |
1112 | } | |
1113 | cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; | |
1114 | sess = conn->sess; | |
1115 | na = iscsit_tpg_get_node_attrib(sess); | |
1116 | ||
1117 | if (!sess->sess_ops->ErrorRecoveryLevel) { | |
d9a771fd DD |
1118 | pr_err("Unable to recover from DataOut timeout while" |
1119 | " in ERL=0, closing iSCSI connection for I_T Nexus" | |
1120 | " %s,i,0x%6phN,%s,t,0x%02x\n", | |
1121 | sess->sess_ops->InitiatorName, sess->isid, | |
1122 | sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); | |
e48354ce NB |
1123 | goto failure; |
1124 | } | |
1125 | ||
1126 | if (++cmd->dataout_timeout_retries == na->dataout_timeout_retries) { | |
d9a771fd DD |
1127 | pr_err("Command ITT: 0x%08x exceeded max retries" |
1128 | " for DataOUT timeout %u, closing iSCSI connection for" | |
1129 | " I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n", | |
1130 | cmd->init_task_tag, na->dataout_timeout_retries, | |
1131 | sess->sess_ops->InitiatorName, sess->isid, | |
1132 | sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt); | |
e48354ce NB |
1133 | goto failure; |
1134 | } | |
1135 | ||
1136 | cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY; | |
1137 | ||
1138 | if (conn->sess->sess_ops->DataSequenceInOrder) { | |
1139 | if (conn->sess->sess_ops->DataPDUInOrder) { | |
1140 | pdu_offset = cmd->write_data_done; | |
1141 | if ((pdu_offset + (conn->sess->sess_ops->MaxBurstLength - | |
ebf1d95c AG |
1142 | cmd->next_burst_len)) > cmd->se_cmd.data_length) |
1143 | pdu_length = (cmd->se_cmd.data_length - | |
e48354ce NB |
1144 | cmd->write_data_done); |
1145 | else | |
1146 | pdu_length = (conn->sess->sess_ops->MaxBurstLength - | |
1147 | cmd->next_burst_len); | |
1148 | } else { | |
1149 | pdu_offset = cmd->seq_start_offset; | |
1150 | pdu_length = (cmd->seq_end_offset - | |
1151 | cmd->seq_start_offset); | |
1152 | } | |
1153 | } else { | |
1154 | if (iscsit_set_dataout_timeout_values(cmd, &pdu_offset, | |
1155 | &pdu_length) < 0) | |
1156 | goto failure; | |
1157 | } | |
1158 | ||
1159 | if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length, | |
1160 | &r2t_offset, &r2t_length) < 0) | |
1161 | goto failure; | |
1162 | ||
1163 | pr_debug("Command ITT: 0x%08x timed out waiting for" | |
1164 | " completion of %sDataOUT Sequence Offset: %u, Length: %u\n", | |
1165 | cmd->init_task_tag, (cmd->unsolicited_data) ? "Unsolicited " : | |
1166 | "", r2t_offset, r2t_length); | |
1167 | ||
1168 | if (iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length) < 0) | |
1169 | goto failure; | |
1170 | ||
1171 | iscsit_start_dataout_timer(cmd, conn); | |
1172 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1173 | iscsit_dec_conn_usage_count(conn); | |
1174 | ||
1175 | return; | |
1176 | ||
1177 | failure: | |
1178 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
33b3f8ca | 1179 | iscsit_fill_cxn_timeout_err_stats(sess); |
e48354ce NB |
1180 | iscsit_cause_connection_reinstatement(conn, 0); |
1181 | iscsit_dec_conn_usage_count(conn); | |
1182 | } | |
1183 | ||
1184 | void iscsit_mod_dataout_timer(struct iscsi_cmd *cmd) | |
1185 | { | |
1186 | struct iscsi_conn *conn = cmd->conn; | |
1187 | struct iscsi_session *sess = conn->sess; | |
e8904dc5 | 1188 | struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); |
e48354ce NB |
1189 | |
1190 | spin_lock_bh(&cmd->dataout_timeout_lock); | |
1191 | if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { | |
1192 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1193 | return; | |
1194 | } | |
1195 | ||
1196 | mod_timer(&cmd->dataout_timer, | |
1197 | (get_jiffies_64() + na->dataout_timeout * HZ)); | |
1198 | pr_debug("Updated DataOUT timer for ITT: 0x%08x", | |
1199 | cmd->init_task_tag); | |
1200 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1201 | } | |
1202 | ||
e48354ce NB |
1203 | void iscsit_start_dataout_timer( |
1204 | struct iscsi_cmd *cmd, | |
1205 | struct iscsi_conn *conn) | |
1206 | { | |
1207 | struct iscsi_session *sess = conn->sess; | |
e8904dc5 | 1208 | struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess); |
e48354ce | 1209 | |
618baaf7 BVA |
1210 | lockdep_assert_held(&cmd->dataout_timeout_lock); |
1211 | ||
e48354ce NB |
1212 | if (cmd->dataout_timer_flags & ISCSI_TF_RUNNING) |
1213 | return; | |
1214 | ||
1215 | pr_debug("Starting DataOUT timer for ITT: 0x%08x on" | |
1216 | " CID: %hu.\n", cmd->init_task_tag, conn->cid); | |
1217 | ||
e48354ce NB |
1218 | cmd->dataout_timer_flags &= ~ISCSI_TF_STOP; |
1219 | cmd->dataout_timer_flags |= ISCSI_TF_RUNNING; | |
8a47aa9d | 1220 | mod_timer(&cmd->dataout_timer, jiffies + na->dataout_timeout * HZ); |
e48354ce NB |
1221 | } |
1222 | ||
1223 | void iscsit_stop_dataout_timer(struct iscsi_cmd *cmd) | |
1224 | { | |
1225 | spin_lock_bh(&cmd->dataout_timeout_lock); | |
1226 | if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) { | |
1227 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1228 | return; | |
1229 | } | |
1230 | cmd->dataout_timer_flags |= ISCSI_TF_STOP; | |
1231 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1232 | ||
1233 | del_timer_sync(&cmd->dataout_timer); | |
1234 | ||
1235 | spin_lock_bh(&cmd->dataout_timeout_lock); | |
1236 | cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING; | |
1237 | pr_debug("Stopped DataOUT Timer for ITT: 0x%08x\n", | |
1238 | cmd->init_task_tag); | |
1239 | spin_unlock_bh(&cmd->dataout_timeout_lock); | |
1240 | } | |
3e1c81a9 | 1241 | EXPORT_SYMBOL(iscsit_stop_dataout_timer); |