[linux-block.git] / net / rxrpc / call_event.c

// SPDX-License-Identifier: GPL-2.0-or-later
/* Management of Tx window, Tx resend, ACKs and out-of-sequence reception
 *
 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/circ_buf.h>
#include <linux/net.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/udp.h>
#include <net/sock.h>
#include <net/af_rxrpc.h>
#include "ar-internal.h"

/*
 * Propose a PING ACK be sent.
 */
void rxrpc_propose_ping(struct rxrpc_call *call, u32 serial,
			enum rxrpc_propose_ack_trace why)
{
	ktime_t delay = ms_to_ktime(READ_ONCE(rxrpc_idle_ack_delay));
	ktime_t now = ktime_get_real();
	ktime_t ping_at = ktime_add(now, delay);

	trace_rxrpc_propose_ack(call, why, RXRPC_ACK_PING, serial);
	if (ktime_before(ping_at, call->ping_at)) {
		call->ping_at = ping_at;
		trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_ping);
	}
}

/*
 * Propose a DELAY ACK be sent in the future.
 */
void rxrpc_propose_delay_ACK(struct rxrpc_call *call, rxrpc_serial_t serial,
			     enum rxrpc_propose_ack_trace why)
{
	ktime_t now = ktime_get_real(), delay;

	trace_rxrpc_propose_ack(call, why, RXRPC_ACK_DELAY, serial);

	if (call->srtt_us)
		delay = (call->srtt_us >> 3) * NSEC_PER_USEC;
	else
		delay = ms_to_ktime(READ_ONCE(rxrpc_soft_ack_delay));
	ktime_add_ms(delay, call->tx_backoff);

	call->delay_ack_at = ktime_add(now, delay);
	trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_delayed_ack);
}

/*
 * Retransmit one or more packets.
 */
static bool rxrpc_retransmit_data(struct rxrpc_call *call,
				  struct rxrpc_send_data_req *req)
{
	struct rxrpc_txqueue *tq = req->tq;
	unsigned int ix = req->seq & RXRPC_TXQ_MASK;
	struct rxrpc_txbuf *txb = tq->bufs[ix];

	_enter("%x,%x,%x,%x", tq->qbase, req->seq, ix, txb->debug_id);

	req->retrans = true;
	trace_rxrpc_retransmit(call, req, txb);

	txb->flags |= RXRPC_TXBUF_RESENT;
	rxrpc_send_data_packet(call, req);
	rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);

	req->tq		= NULL;
	req->n		= 0;
	req->did_send	= true;
	req->now	= ktime_get_real();
	return true;
}

/*
 * Perform retransmission of NAK'd and unack'd packets.
 */
static void rxrpc_resend(struct rxrpc_call *call)
{
	struct rxrpc_send_data_req req = {
		.now	= ktime_get_real(),
		.trace	= rxrpc_txdata_retransmit,
	};
	struct rxrpc_txqueue *tq;

	_enter("{%d,%d}", call->tx_bottom, call->tx_top);

	trace_rxrpc_resend(call, call->acks_highest_serial);

	/* Scan the transmission queue, looking for lost packets. */
	for (tq = call->tx_queue; tq; tq = tq->next) {
		unsigned long lost = tq->segment_lost;

		if (after(tq->qbase, call->tx_transmitted))
			break;

		_debug("retr %16lx %u c=%08x [%x]",
		       tq->segment_acked, tq->nr_reported_acks, call->debug_id, tq->qbase);
		_debug("lost %16lx", lost);

		trace_rxrpc_resend_lost(call, tq, lost);
		while (lost) {
			unsigned int ix = __ffs(lost);
			struct rxrpc_txbuf *txb = tq->bufs[ix];

			__clear_bit(ix, &lost);
			rxrpc_see_txbuf(txb, rxrpc_txbuf_see_lost);

			req.tq  = tq;
			req.seq = tq->qbase + ix;
			req.n   = 1;
			rxrpc_retransmit_data(call, &req);
		}
	}

	rxrpc_get_rto_backoff(call, req.did_send);
	_leave("");
}

/*
 * Resend the highest-seq DATA packet so far transmitted for RACK-TLP [RFC8985 7.3].
 */
void rxrpc_resend_tlp(struct rxrpc_call *call)
{
	struct rxrpc_send_data_req req = {
		.now		= ktime_get_real(),
		.seq		= call->tx_transmitted,
		.n		= 1,
		.tlp_probe	= true,
		.trace		= rxrpc_txdata_tlp_retransmit,
	};

	/* There's a chance it'll be on the tail segment of the queue. */
	req.tq = READ_ONCE(call->tx_qtail);
	if (req.tq &&
	    before(call->tx_transmitted, req.tq->qbase + RXRPC_NR_TXQUEUE)) {
		rxrpc_retransmit_data(call, &req);
		return;
	}

	for (req.tq = call->tx_queue; req.tq; req.tq = req.tq->next) {
		if (after_eq(call->tx_transmitted, req.tq->qbase) &&
		    before(call->tx_transmitted, req.tq->qbase + RXRPC_NR_TXQUEUE)) {
			rxrpc_retransmit_data(call, &req);
			return;
		}
	}
}

/*
 * Start transmitting the reply to a service.  This cancels the need to ACK the
 * request if we haven't yet done so.
 */
static void rxrpc_begin_service_reply(struct rxrpc_call *call)
{
	rxrpc_set_call_state(call, RXRPC_CALL_SERVER_SEND_REPLY);
	if (call->ackr_reason == RXRPC_ACK_DELAY)
		call->ackr_reason = 0;
	call->delay_ack_at = KTIME_MAX;
	trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack);
}

/*
 * Close the transmission phase.  After this point there is no more data to be
 * transmitted in the call.
 */
static void rxrpc_close_tx_phase(struct rxrpc_call *call)
{
	_debug("________awaiting reply/ACK__________");

	switch (__rxrpc_call_state(call)) {
	case RXRPC_CALL_CLIENT_SEND_REQUEST:
		rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY);
		break;
	case RXRPC_CALL_SERVER_SEND_REPLY:
		rxrpc_set_call_state(call, RXRPC_CALL_SERVER_AWAIT_ACK);
		break;
	default:
		break;
	}
}

/*
 * Transmit some as-yet untransmitted data, to a maximum of the supplied limit.
 */
static void rxrpc_transmit_fresh_data(struct rxrpc_call *call, unsigned int limit,
				      enum rxrpc_txdata_trace trace)
{
	int space = rxrpc_tx_window_space(call);

	if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
		if (call->send_top == call->tx_top)
			return;
		rxrpc_expose_client_call(call);
	}

	while (space > 0) {
		struct rxrpc_send_data_req req = {
			.now	= ktime_get_real(),
			.seq	= call->tx_transmitted + 1,
			.n	= 0,
			.trace	= trace,
		};
		struct rxrpc_txqueue *tq;
		struct rxrpc_txbuf *txb;
		rxrpc_seq_t send_top, seq;
		int limit = min(space, max(call->peer->pmtud_jumbo, 1));

		/* Order send_top before the contents of the new txbufs and
		 * txqueue pointers
		 */
		send_top = smp_load_acquire(&call->send_top);
		if (call->tx_top == send_top)
			break;

		trace_rxrpc_transmit(call, send_top, space);

		tq = call->tx_qtail;
		seq = call->tx_top;
		trace_rxrpc_tq(call, tq, seq, rxrpc_tq_decant);

		do {
			int ix;

			seq++;
			ix = seq & RXRPC_TXQ_MASK;
			if (!ix) {
				tq = tq->next;
				trace_rxrpc_tq(call, tq, seq, rxrpc_tq_decant_advance);
			}
			if (!req.tq)
				req.tq = tq;
			txb = tq->bufs[ix];
			req.n++;
			if (!txb->jumboable)
				break;
		} while (req.n < limit && before(seq, send_top));

		if (txb->flags & RXRPC_LAST_PACKET) {
			rxrpc_close_tx_phase(call);
			tq = NULL;
		}
		call->tx_qtail = tq;
		call->tx_top = seq;

		space -= req.n;
		rxrpc_send_data_packet(call, &req);
	}
}

void rxrpc_transmit_some_data(struct rxrpc_call *call, unsigned int limit,
			      enum rxrpc_txdata_trace trace)
{
	switch (__rxrpc_call_state(call)) {
	case RXRPC_CALL_SERVER_ACK_REQUEST:
		if (call->tx_bottom == READ_ONCE(call->send_top))
			return;
		rxrpc_begin_service_reply(call);
		fallthrough;

	case RXRPC_CALL_SERVER_SEND_REPLY:
	case RXRPC_CALL_CLIENT_SEND_REQUEST:
		if (!rxrpc_tx_window_space(call))
			return;
		if (call->tx_bottom == READ_ONCE(call->send_top)) {
			rxrpc_inc_stat(call->rxnet, stat_tx_data_underflow);
			return;
		}
		rxrpc_transmit_fresh_data(call, limit, trace);
		break;
	default:
		return;
	}
}

/*
 * Ping the other end to fill our RTT cache and to retrieve the rwind
 * and MTU parameters.
 */
static void rxrpc_send_initial_ping(struct rxrpc_call *call)
{
	if (call->rtt_count < 3 ||
	    ktime_before(ktime_add_ms(call->rtt_last_req, 1000),
			 ktime_get_real()))
		rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
			       rxrpc_propose_ack_ping_for_params);
}

/*
 * Handle retransmission and deferred ACK/abort generation.
 */
bool rxrpc_input_call_event(struct rxrpc_call *call)
{
	struct sk_buff *skb;
	ktime_t now, t;
	bool did_receive = false, saw_ack = false;
	s32 abort_code;

	rxrpc_see_call(call, rxrpc_call_see_input);

	//printk("\n--------------------\n");
	_enter("{%d,%s,%lx}",
	       call->debug_id, rxrpc_call_states[__rxrpc_call_state(call)],
	       call->events);

	/* Handle abort request locklessly, vs rxrpc_propose_abort(). */
	abort_code = smp_load_acquire(&call->send_abort);
	if (abort_code) {
		rxrpc_abort_call(call, 0, call->send_abort, call->send_abort_err,
				 call->send_abort_why);
		goto out;
	}

	do {
		skb = __skb_dequeue(&call->rx_queue);
		if (skb) {
			struct rxrpc_skb_priv *sp = rxrpc_skb(skb);

			if (__rxrpc_call_is_complete(call) ||
			    skb->mark == RXRPC_SKB_MARK_ERROR) {
				rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
				goto out;
			}

			saw_ack |= sp->hdr.type == RXRPC_PACKET_TYPE_ACK;

			rxrpc_input_call_packet(call, skb);
			rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
			did_receive = true;
		}

		t = ktime_sub(call->rack_timo_at, ktime_get_real());
		if (t <= 0) {
			trace_rxrpc_timer_exp(call, t,
					      rxrpc_timer_trace_rack_off + call->rack_timer_mode);
			call->rack_timo_at = KTIME_MAX;
			rxrpc_rack_timer_expired(call, t);
		}

	} while (!skb_queue_empty(&call->rx_queue));

	/* If we see our async-event poke, check for timeout trippage. */
	now = ktime_get_real();
	t = ktime_sub(call->expect_rx_by, now);
	if (t <= 0) {
		trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_expect_rx);
		goto expired;
	}

	t = ktime_sub(call->expect_req_by, now);
	if (t <= 0) {
		call->expect_req_by = KTIME_MAX;
		if (__rxrpc_call_state(call) == RXRPC_CALL_SERVER_RECV_REQUEST) {
			trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_idle);
			goto expired;
		}
	}

	t = ktime_sub(READ_ONCE(call->expect_term_by), now);
	if (t <= 0) {
		trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_hard);
		goto expired;
	}

	t = ktime_sub(call->delay_ack_at, now);
	if (t <= 0) {
		trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_delayed_ack);
		call->delay_ack_at = KTIME_MAX;
		rxrpc_send_ACK(call, RXRPC_ACK_DELAY, 0,
			       rxrpc_propose_ack_delayed_ack);
	}

	t = ktime_sub(call->ping_at, now);
	if (t <= 0) {
		trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_ping);
		call->ping_at = KTIME_MAX;
		rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
			       rxrpc_propose_ack_ping_for_keepalive);
	}

	now = ktime_get_real();
	t = ktime_sub(call->keepalive_at, now);
	if (t <= 0) {
		trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_keepalive);
		call->keepalive_at = KTIME_MAX;
		rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
			       rxrpc_propose_ack_ping_for_keepalive);
	}

	if (test_and_clear_bit(RXRPC_CALL_EV_INITIAL_PING, &call->events))
		rxrpc_send_initial_ping(call);

	rxrpc_transmit_some_data(call, UINT_MAX, rxrpc_txdata_new_data);

	if (saw_ack)
		rxrpc_congestion_degrade(call);

	if (did_receive &&
	    (__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_SEND_REQUEST ||
	     __rxrpc_call_state(call) == RXRPC_CALL_SERVER_SEND_REPLY)) {
		t = ktime_sub(call->rack_timo_at, ktime_get_real());
		trace_rxrpc_rack(call, t);
	}

	/* Process events */
	if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events))
		rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
			       rxrpc_propose_ack_ping_for_lost_ack);

	if (call->tx_nr_lost > 0 &&
	    __rxrpc_call_state(call) != RXRPC_CALL_CLIENT_RECV_REPLY &&
	    !test_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags))
		rxrpc_resend(call);

	if (test_and_clear_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags))
		rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0,
			       rxrpc_propose_ack_rx_idle);

	if (call->ackr_nr_unacked > 2) {
		if (call->rtt_count < 3)
			rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
				       rxrpc_propose_ack_ping_for_rtt);
		else if (ktime_before(ktime_add_ms(call->rtt_last_req, 1000),
				      ktime_get_real()))
			rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
				       rxrpc_propose_ack_ping_for_old_rtt);
		else
			rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0,
				       rxrpc_propose_ack_input_data);
	}

	/* Make sure the timer is restarted */
	if (!__rxrpc_call_is_complete(call)) {
		ktime_t next = READ_ONCE(call->expect_term_by), delay;

#define set(T) { ktime_t _t = (T); if (ktime_before(_t, next)) next = _t; }

		set(call->expect_req_by);
		set(call->expect_rx_by);
		set(call->delay_ack_at);
		set(call->rack_timo_at);
		set(call->keepalive_at);
		set(call->ping_at);

		now = ktime_get_real();
		delay = ktime_sub(next, now);
		if (delay <= 0) {
			rxrpc_poke_call(call, rxrpc_call_poke_timer_now);
		} else {
			unsigned long nowj = jiffies, delayj, nextj;

			delayj = umax(nsecs_to_jiffies(delay), 1);
			nextj = nowj + delayj;
			if (time_before(nextj, call->timer.expires) ||
			    !timer_pending(&call->timer)) {
				trace_rxrpc_timer_restart(call, delay, delayj);
				timer_reduce(&call->timer, nextj);
			}
		}
	}

out:
	if (__rxrpc_call_is_complete(call)) {
		timer_delete_sync(&call->timer);
		if (!test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))
			rxrpc_disconnect_call(call);
		if (call->security)
			call->security->free_call_crypto(call);
	} else {
		if (did_receive &&
		    call->peer->ackr_adv_pmtud &&
		    call->peer->pmtud_pending)
			rxrpc_send_probe_for_pmtud(call);
	}
	_leave("");
	return true;

expired:
	if (test_bit(RXRPC_CALL_RX_HEARD, &call->flags) &&
	    (int)call->conn->hi_serial - (int)call->rx_serial > 0) {
		trace_rxrpc_call_reset(call);
		rxrpc_abort_call(call, 0, RX_CALL_DEAD, -ECONNRESET,
				 rxrpc_abort_call_reset);
	} else {
		rxrpc_abort_call(call, 0, RX_CALL_TIMEOUT, -ETIME,
				 rxrpc_abort_call_timeout);
	}
	goto out;
}
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
17926a79 DH	2	/* Management of Tx window, Tx resend, ACKs and out-of-sequence reception
	3	*
	4	* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
	5	* Written by David Howells (dhowells@redhat.com)
17926a79 DH	6	*/
17926a79 DH	7
9b6d5398 JP	8	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9b6d5398 JP	9
17926a79 DH	10	#include <linux/module.h>
	11	#include <linux/circ_buf.h>
	12	#include <linux/net.h>
	13	#include <linux/skbuff.h>
5a0e3ad6	14	#include <linux/slab.h>
17926a79 DH	15	#include <linux/udp.h>
	16	#include <net/sock.h>
	17	#include <net/af_rxrpc.h>
	18	#include "ar-internal.h"
	19
a5af7e1f DH	20	/*
	21	* Propose a PING ACK be sent.
	22	*/
72f0c6fb DH	23	void rxrpc_propose_ping(struct rxrpc_call *call, u32 serial,
72f0c6fb DH	24	enum rxrpc_propose_ack_trace why)
a5af7e1f	25	{
153f90a0 DH	26	ktime_t delay = ms_to_ktime(READ_ONCE(rxrpc_idle_ack_delay));
	27	ktime_t now = ktime_get_real();
	28	ktime_t ping_at = ktime_add(now, delay);
a5af7e1f	29
153f90a0 DH	30	trace_rxrpc_propose_ack(call, why, RXRPC_ACK_PING, serial);
153f90a0 DH	31	if (ktime_before(ping_at, call->ping_at)) {
d73f3a74	32	call->ping_at = ping_at;
153f90a0	33	trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_ping);
a5af7e1f DH	34	}
	35	}
	36
17926a79	37	/*
530403d9	38	* Propose a DELAY ACK be sent in the future.
17926a79	39	*/
4e76bd40 DH	40	void rxrpc_propose_delay_ACK(struct rxrpc_call *call, rxrpc_serial_t serial,
4e76bd40 DH	41	enum rxrpc_propose_ack_trace why)
17926a79	42	{
153f90a0	43	ktime_t now = ktime_get_real(), delay;
17926a79	44
153f90a0 DH	45	trace_rxrpc_propose_ack(call, why, RXRPC_ACK_DELAY, serial);
153f90a0 DH	46
b40ef2b8 DH	47	if (call->srtt_us)
b40ef2b8 DH	48	delay = (call->srtt_us >> 3) * NSEC_PER_USEC;
72f0c6fb	49	else
153f90a0 DH	50	delay = ms_to_ktime(READ_ONCE(rxrpc_soft_ack_delay));
153f90a0 DH	51	ktime_add_ms(delay, call->tx_backoff);
9c7ad434	52
153f90a0 DH	53	call->delay_ack_at = ktime_add(now, delay);
153f90a0 DH	54	trace_rxrpc_timer_set(call, delay, rxrpc_timer_trace_delayed_ack);
17926a79 DH	55	}
17926a79 DH	56
b341a026 DH	57	/*
	58	* Retransmit one or more packets.
	59	*/
9b052c6b	60	static bool rxrpc_retransmit_data(struct rxrpc_call *call,
7c482665	61	struct rxrpc_send_data_req *req)
b341a026 DH	62	{
	63	struct rxrpc_txqueue *tq = req->tq;
	64	unsigned int ix = req->seq & RXRPC_TXQ_MASK;
	65	struct rxrpc_txbuf *txb = tq->bufs[ix];
b341a026 DH	66
	67	_enter("%x,%x,%x,%x", tq->qbase, req->seq, ix, txb->debug_id);
	68
7c482665 DH	69	req->retrans = true;
7c482665 DH	70	trace_rxrpc_retransmit(call, req, txb);
b341a026 DH	71
	72	txb->flags \|= RXRPC_TXBUF_RESENT;
	73	rxrpc_send_data_packet(call, req);
	74	rxrpc_inc_stat(call->rxnet, stat_tx_data_retrans);
	75
	76	req->tq = NULL;
	77	req->n = 0;
	78	req->did_send = true;
	79	req->now = ktime_get_real();
9b052c6b	80	return true;
b341a026 DH	81	}
b341a026 DH	82
17926a79	83	/*
248f219c	84	* Perform retransmission of NAK'd and unack'd packets.
17926a79	85	*/
7c482665	86	static void rxrpc_resend(struct rxrpc_call *call)
17926a79	87	{
b341a026 DH	88	struct rxrpc_send_data_req req = {
b341a026 DH	89	.now = ktime_get_real(),
372d12d1	90	.trace = rxrpc_txdata_retransmit,
b341a026	91	};
7c482665	92	struct rxrpc_txqueue *tq;
17926a79	93
692c4caa	94	_enter("{%d,%d}", call->tx_bottom, call->tx_top);
17926a79	95
7c482665	96	trace_rxrpc_resend(call, call->acks_highest_serial);
50235c4b	97
7c482665 DH	98	/* Scan the transmission queue, looking for lost packets. */
	99	for (tq = call->tx_queue; tq; tq = tq->next) {
	100	unsigned long lost = tq->segment_lost;
5e6ef4f1	101
9b052c6b DH	102	if (after(tq->qbase, call->tx_transmitted))
9b052c6b DH	103	break;
17926a79	104
9b052c6b DH	105	_debug("retr %16lx %u c=%08x [%x]",
9b052c6b DH	106	tq->segment_acked, tq->nr_reported_acks, call->debug_id, tq->qbase);
7c482665	107	_debug("lost %16lx", lost);
9b052c6b	108
7c482665 DH	109	trace_rxrpc_resend_lost(call, tq, lost);
	110	while (lost) {
	111	unsigned int ix = __ffs(lost);
9b052c6b DH	112	struct rxrpc_txbuf *txb = tq->bufs[ix];
9b052c6b DH	113
7c482665 DH	114	__clear_bit(ix, &lost);
7c482665 DH	115	rxrpc_see_txbuf(txb, rxrpc_txbuf_see_lost);
d57a3a15	116
b341a026	117	req.tq = tq;
9b052c6b	118	req.seq = tq->qbase + ix;
b341a026	119	req.n = 1;
7c482665	120	rxrpc_retransmit_data(call, &req);
b341a026	121	}
37473e41	122	}
248f219c	123
7c482665 DH	124	rxrpc_get_rto_backoff(call, req.did_send);
	125	_leave("");
	126	}
57494343	127
7c482665 DH	128	/*
	129	* Resend the highest-seq DATA packet so far transmitted for RACK-TLP [RFC8985 7.3].
	130	*/
	131	void rxrpc_resend_tlp(struct rxrpc_call *call)
	132	{
	133	struct rxrpc_send_data_req req = {
	134	.now = ktime_get_real(),
	135	.seq = call->tx_transmitted,
	136	.n = 1,
	137	.tlp_probe = true,
	138	.trace = rxrpc_txdata_tlp_retransmit,
	139	};
57494343	140
7c482665 DH	141	/* There's a chance it'll be on the tail segment of the queue. */
	142	req.tq = READ_ONCE(call->tx_qtail);
	143	if (req.tq &&
	144	before(call->tx_transmitted, req.tq->qbase + RXRPC_NR_TXQUEUE)) {
	145	rxrpc_retransmit_data(call, &req);
	146	return;
dfa7d920	147	}
248f219c	148
7c482665 DH	149	for (req.tq = call->tx_queue; req.tq; req.tq = req.tq->next) {
	150	if (after_eq(call->tx_transmitted, req.tq->qbase) &&
	151	before(call->tx_transmitted, req.tq->qbase + RXRPC_NR_TXQUEUE)) {
	152	rxrpc_retransmit_data(call, &req);
	153	return;
	154	}
	155	}
17926a79 DH	156	}
17926a79 DH	157
2d689424 DH	158	/*
	159	* Start transmitting the reply to a service. This cancels the need to ACK the
	160	* request if we haven't yet done so.
	161	*/
	162	static void rxrpc_begin_service_reply(struct rxrpc_call *call)
	163	{
96b4059f	164	rxrpc_set_call_state(call, RXRPC_CALL_SERVER_SEND_REPLY);
96b4059f DH	165	if (call->ackr_reason == RXRPC_ACK_DELAY)
96b4059f DH	166	call->ackr_reason = 0;
153f90a0 DH	167	call->delay_ack_at = KTIME_MAX;
153f90a0 DH	168	trace_rxrpc_timer_can(call, rxrpc_timer_trace_delayed_ack);
2d689424 DH	169	}
	170
	171	/*
	172	* Close the transmission phase. After this point there is no more data to be
	173	* transmitted in the call.
	174	*/
	175	static void rxrpc_close_tx_phase(struct rxrpc_call *call)
	176	{
	177	_debug("________awaiting reply/ACK__________");
	178
96b4059f	179	switch (__rxrpc_call_state(call)) {
2d689424	180	case RXRPC_CALL_CLIENT_SEND_REQUEST:
96b4059f	181	rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_AWAIT_REPLY);
2d689424 DH	182	break;
2d689424 DH	183	case RXRPC_CALL_SERVER_SEND_REPLY:
96b4059f	184	rxrpc_set_call_state(call, RXRPC_CALL_SERVER_AWAIT_ACK);
2d689424 DH	185	break;
	186	default:
	187	break;
	188	}
2d689424 DH	189	}
2d689424 DH	190
cf37b598	191	/*
7c482665	192	* Transmit some as-yet untransmitted data, to a maximum of the supplied limit.
cf37b598	193	*/
7c482665	194	static void rxrpc_transmit_fresh_data(struct rxrpc_call *call, unsigned int limit,
372d12d1	195	enum rxrpc_txdata_trace trace)
cf37b598	196	{
b7313009	197	int space = rxrpc_tx_window_space(call);
cf37b598	198
a343b174	199	if (!test_bit(RXRPC_CALL_EXPOSED, &call->flags)) {
b341a026	200	if (call->send_top == call->tx_top)
a343b174	201	return;
cf37b598	202	rxrpc_expose_client_call(call);
a343b174	203	}
cf37b598	204
b7313009	205	while (space > 0) {
b341a026 DH	206	struct rxrpc_send_data_req req = {
	207	.now = ktime_get_real(),
	208	.seq = call->tx_transmitted + 1,
	209	.n = 0,
372d12d1	210	.trace = trace,
b341a026 DH	211	};
	212	struct rxrpc_txqueue *tq;
	213	struct rxrpc_txbuf *txb;
	214	rxrpc_seq_t send_top, seq;
fe24a549	215	int limit = min(space, max(call->peer->pmtud_jumbo, 1));
b341a026 DH	216
	217	/* Order send_top before the contents of the new txbufs and
	218	* txqueue pointers
	219	*/
	220	send_top = smp_load_acquire(&call->send_top);
	221	if (call->tx_top == send_top)
b7313009 DH	222	break;
b7313009 DH	223
b341a026 DH	224	trace_rxrpc_transmit(call, send_top, space);
	225
	226	tq = call->tx_qtail;
	227	seq = call->tx_top;
	228	trace_rxrpc_tq(call, tq, seq, rxrpc_tq_decant);
149d002b	229
b7313009	230	do {
b341a026 DH	231	int ix;
	232
	233	seq++;
	234	ix = seq & RXRPC_TXQ_MASK;
	235	if (!ix) {
	236	tq = tq->next;
	237	trace_rxrpc_tq(call, tq, seq, rxrpc_tq_decant_advance);
	238	}
	239	if (!req.tq)
	240	req.tq = tq;
	241	txb = tq->bufs[ix];
	242	req.n++;
b7313009 DH	243	if (!txb->jumboable)
b7313009 DH	244	break;
b341a026	245	} while (req.n < limit && before(seq, send_top));
b7313009	246
b341a026	247	if (txb->flags & RXRPC_LAST_PACKET) {
2d689424	248	rxrpc_close_tx_phase(call);
b341a026 DH	249	tq = NULL;
	250	}
	251	call->tx_qtail = tq;
	252	call->tx_top = seq;
2d689424	253
b341a026 DH	254	space -= req.n;
b341a026 DH	255	rxrpc_send_data_packet(call, &req);
cf37b598 DH	256	}
	257	}
	258
7c482665 DH	259	void rxrpc_transmit_some_data(struct rxrpc_call *call, unsigned int limit,
7c482665 DH	260	enum rxrpc_txdata_trace trace)
cf37b598	261	{
96b4059f	262	switch (__rxrpc_call_state(call)) {
cf37b598	263	case RXRPC_CALL_SERVER_ACK_REQUEST:
b341a026	264	if (call->tx_bottom == READ_ONCE(call->send_top))
cf37b598	265	return;
2d689424	266	rxrpc_begin_service_reply(call);
cf37b598 DH	267	fallthrough;
	268
	269	case RXRPC_CALL_SERVER_SEND_REPLY:
cf37b598	270	case RXRPC_CALL_CLIENT_SEND_REQUEST:
b7313009	271	if (!rxrpc_tx_window_space(call))
cf37b598	272	return;
b341a026	273	if (call->tx_bottom == READ_ONCE(call->send_top)) {
32cf8edb	274	rxrpc_inc_stat(call->rxnet, stat_tx_data_underflow);
cf37b598	275	return;
32cf8edb	276	}
7c482665	277	rxrpc_transmit_fresh_data(call, limit, trace);
cf37b598 DH	278	break;
	279	default:
	280	return;
	281	}
	282	}
	283
5e6ef4f1 DH	284	/*
	285	* Ping the other end to fill our RTT cache and to retrieve the rwind
	286	* and MTU parameters.
	287	*/
	288	static void rxrpc_send_initial_ping(struct rxrpc_call *call)
	289	{
b40ef2b8 DH	290	if (call->rtt_count < 3 \|\|
b40ef2b8 DH	291	ktime_before(ktime_add_ms(call->rtt_last_req, 1000),
5e6ef4f1 DH	292	ktime_get_real()))
	293	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
	294	rxrpc_propose_ack_ping_for_params);
	295	}
	296
17926a79	297	/*
248f219c	298	* Handle retransmission and deferred ACK/abort generation.
17926a79	299	*/
9e3cccd1	300	bool rxrpc_input_call_event(struct rxrpc_call *call)
17926a79	301	{
9e3cccd1	302	struct sk_buff *skb;
153f90a0	303	ktime_t now, t;
7c482665	304	bool did_receive = false, saw_ack = false;
a343b174	305	s32 abort_code;
17926a79	306
cb0fc0c9	307	rxrpc_see_call(call, rxrpc_call_see_input);
e34d4234	308
17926a79	309	//printk("\n--------------------\n");
248f219c	310	_enter("{%d,%s,%lx}",
96b4059f DH	311	call->debug_id, rxrpc_call_states[__rxrpc_call_state(call)],
96b4059f DH	312	call->events);
17926a79	313
a343b174 DH	314	/* Handle abort request locklessly, vs rxrpc_propose_abort(). */
	315	abort_code = smp_load_acquire(&call->send_abort);
	316	if (abort_code) {
57af281e DH	317	rxrpc_abort_call(call, 0, call->send_abort, call->send_abort_err,
57af281e DH	318	call->send_abort_why);
a343b174 DH	319	goto out;
	320	}
	321
7c482665 DH	322	do {
	323	skb = __skb_dequeue(&call->rx_queue);
	324	if (skb) {
	325	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
	326
	327	if (__rxrpc_call_is_complete(call) \|\|
	328	skb->mark == RXRPC_SKB_MARK_ERROR) {
	329	rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
	330	goto out;
	331	}
9e3cccd1	332
7c482665 DH	333	saw_ack \|= sp->hdr.type == RXRPC_PACKET_TYPE_ACK;
	334
	335	rxrpc_input_call_packet(call, skb);
9e3cccd1	336	rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
7c482665	337	did_receive = true;
9e3cccd1 DH	338	}
9e3cccd1 DH	339
7c482665 DH	340	t = ktime_sub(call->rack_timo_at, ktime_get_real());
	341	if (t <= 0) {
	342	trace_rxrpc_timer_exp(call, t,
	343	rxrpc_timer_trace_rack_off + call->rack_timer_mode);
	344	call->rack_timo_at = KTIME_MAX;
	345	rxrpc_rack_timer_expired(call, t);
	346	}
17926a79	347
7c482665	348	} while (!skb_queue_empty(&call->rx_queue));
3e0b83ee	349
5e6ef4f1	350	/* If we see our async-event poke, check for timeout trippage. */
153f90a0 DH	351	now = ktime_get_real();
	352	t = ktime_sub(call->expect_rx_by, now);
	353	if (t <= 0) {
	354	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_expect_rx);
	355	goto expired;
a158bdd3 DH	356	}
a158bdd3 DH	357
153f90a0 DH	358	t = ktime_sub(call->expect_req_by, now);
	359	if (t <= 0) {
	360	call->expect_req_by = KTIME_MAX;
	361	if (__rxrpc_call_state(call) == RXRPC_CALL_SERVER_RECV_REQUEST) {
	362	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_idle);
	363	goto expired;
	364	}
a158bdd3 DH	365	}
a158bdd3 DH	366
153f90a0 DH	367	t = ktime_sub(READ_ONCE(call->expect_term_by), now);
	368	if (t <= 0) {
	369	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_hard);
	370	goto expired;
a158bdd3 DH	371	}
a158bdd3 DH	372
153f90a0 DH	373	t = ktime_sub(call->delay_ack_at, now);
	374	if (t <= 0) {
	375	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_delayed_ack);
	376	call->delay_ack_at = KTIME_MAX;
e7870cf1	377	rxrpc_send_ACK(call, RXRPC_ACK_DELAY, 0,
12a66e77	378	rxrpc_propose_ack_delayed_ack);
a158bdd3 DH	379	}
a158bdd3 DH	380
153f90a0 DH	381	t = ktime_sub(call->ping_at, now);
	382	if (t <= 0) {
	383	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_ping);
	384	call->ping_at = KTIME_MAX;
72f0c6fb DH	385	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
72f0c6fb DH	386	rxrpc_propose_ack_ping_for_keepalive);
415f44e4 DH	387	}
415f44e4 DH	388
153f90a0 DH	389	now = ktime_get_real();
	390	t = ktime_sub(call->keepalive_at, now);
	391	if (t <= 0) {
	392	trace_rxrpc_timer_exp(call, t, rxrpc_timer_trace_keepalive);
	393	call->keepalive_at = KTIME_MAX;
	394	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
	395	rxrpc_propose_ack_ping_for_keepalive);
	396	}
	397
7c482665 DH	398	if (test_and_clear_bit(RXRPC_CALL_EV_INITIAL_PING, &call->events))
	399	rxrpc_send_initial_ping(call);
	400
	401	rxrpc_transmit_some_data(call, UINT_MAX, rxrpc_txdata_new_data);
	402
9e3cccd1 DH	403	if (saw_ack)
9e3cccd1 DH	404	rxrpc_congestion_degrade(call);
5086d9a9	405
7c482665 DH	406	if (did_receive &&
	407	(__rxrpc_call_state(call) == RXRPC_CALL_CLIENT_SEND_REQUEST \|\|
	408	__rxrpc_call_state(call) == RXRPC_CALL_SERVER_SEND_REPLY)) {
	409	t = ktime_sub(call->rack_timo_at, ktime_get_real());
	410	trace_rxrpc_rack(call, t);
	411	}
5e6ef4f1	412
a158bdd3	413	/* Process events */
5e6ef4f1	414	if (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events))
72f0c6fb DH	415	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
72f0c6fb DH	416	rxrpc_propose_ack_ping_for_lost_ack);
a5af7e1f	417
7c482665	418	if (call->tx_nr_lost > 0 &&
a711d976 DH	419	__rxrpc_call_state(call) != RXRPC_CALL_CLIENT_RECV_REPLY &&
a711d976 DH	420	!test_bit(RXRPC_CALL_TX_ALL_ACKED, &call->flags))
7c482665	421	rxrpc_resend(call);
5e6ef4f1 DH	422
	423	if (test_and_clear_bit(RXRPC_CALL_RX_IS_IDLE, &call->flags))
	424	rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0,
	425	rxrpc_propose_ack_rx_idle);
	426
5bbf9533	427	if (call->ackr_nr_unacked > 2) {
b40ef2b8	428	if (call->rtt_count < 3)
84e28aa5 DH	429	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
84e28aa5 DH	430	rxrpc_propose_ack_ping_for_rtt);
b40ef2b8	431	else if (ktime_before(ktime_add_ms(call->rtt_last_req, 1000),
84e28aa5 DH	432	ktime_get_real()))
	433	rxrpc_send_ACK(call, RXRPC_ACK_PING, 0,
	434	rxrpc_propose_ack_ping_for_old_rtt);
	435	else
	436	rxrpc_send_ACK(call, RXRPC_ACK_IDLE, 0,
	437	rxrpc_propose_ack_input_data);
	438	}
17926a79	439
a158bdd3	440	/* Make sure the timer is restarted */
96b4059f	441	if (!__rxrpc_call_is_complete(call)) {
153f90a0	442	ktime_t next = READ_ONCE(call->expect_term_by), delay;
a158bdd3	443
153f90a0	444	#define set(T) { ktime_t _t = (T); if (ktime_before(_t, next)) next = _t; }
3d7682af	445
5e6ef4f1	446	set(call->expect_req_by);
153f90a0	447	set(call->expect_rx_by);
5e6ef4f1	448	set(call->delay_ack_at);
7c482665	449	set(call->rack_timo_at);
5e6ef4f1 DH	450	set(call->keepalive_at);
5e6ef4f1 DH	451	set(call->ping_at);
a158bdd3	452
153f90a0 DH	453	now = ktime_get_real();
	454	delay = ktime_sub(next, now);
	455	if (delay <= 0) {
5e6ef4f1	456	rxrpc_poke_call(call, rxrpc_call_poke_timer_now);
153f90a0 DH	457	} else {
	458	unsigned long nowj = jiffies, delayj, nextj;
	459
29e03ec7	460	delayj = umax(nsecs_to_jiffies(delay), 1);
153f90a0 DH	461	nextj = nowj + delayj;
	462	if (time_before(nextj, call->timer.expires) \|\|
	463	!timer_pending(&call->timer)) {
	464	trace_rxrpc_timer_restart(call, delay, delayj);
	465	timer_reduce(&call->timer, nextj);
	466	}
	467	}
5e6ef4f1	468	}
17926a79	469
248f219c	470	out:
96b4059f	471	if (__rxrpc_call_is_complete(call)) {
8fa7292f	472	timer_delete_sync(&call->timer);
03fc55ad DH	473	if (!test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))
	474	rxrpc_disconnect_call(call);
	475	if (call->security)
	476	call->security->free_call_crypto(call);
eeaedc54	477	} else {
9e3cccd1	478	if (did_receive &&
eeaedc54 DH	479	call->peer->ackr_adv_pmtud &&
	480	call->peer->pmtud_pending)
	481	rxrpc_send_probe_for_pmtud(call);
03fc55ad	482	}
17926a79	483	_leave("");
57af281e	484	return true;
153f90a0 DH	485
	486	expired:
	487	if (test_bit(RXRPC_CALL_RX_HEARD, &call->flags) &&
	488	(int)call->conn->hi_serial - (int)call->rx_serial > 0) {
	489	trace_rxrpc_call_reset(call);
	490	rxrpc_abort_call(call, 0, RX_CALL_DEAD, -ECONNRESET,
	491	rxrpc_abort_call_reset);
	492	} else {
	493	rxrpc_abort_call(call, 0, RX_CALL_TIMEOUT, -ETIME,
	494	rxrpc_abort_call_timeout);
	495	}
	496	goto out;
17926a79	497	}