Commit | Line | Data |
---|---|---|
740ebe35 GT |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Multipath TCP | |
3 | * | |
4 | * Copyright (c) 2022, SUSE. | |
5 | */ | |
6 | ||
7 | #define pr_fmt(fmt) "MPTCP: " fmt | |
8 | ||
9 | #include <linux/kernel.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/list.h> | |
12 | #include <linux/rculist.h> | |
13 | #include <linux/spinlock.h> | |
14 | #include "protocol.h" | |
15 | ||
16 | static DEFINE_SPINLOCK(mptcp_sched_list_lock); | |
17 | static LIST_HEAD(mptcp_sched_list); | |
18 | ||
ed1ad86b GT |
19 | static int mptcp_sched_default_get_subflow(struct mptcp_sock *msk, |
20 | struct mptcp_sched_data *data) | |
21 | { | |
22 | struct sock *ssk; | |
23 | ||
24 | ssk = data->reinject ? mptcp_subflow_get_retrans(msk) : | |
25 | mptcp_subflow_get_send(msk); | |
26 | if (!ssk) | |
27 | return -EINVAL; | |
28 | ||
29 | mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true); | |
30 | return 0; | |
31 | } | |
32 | ||
33 | static struct mptcp_sched_ops mptcp_sched_default = { | |
34 | .get_subflow = mptcp_sched_default_get_subflow, | |
35 | .name = "default", | |
36 | .owner = THIS_MODULE, | |
37 | }; | |
38 | ||
740ebe35 GT |
39 | /* Must be called with rcu read lock held */ |
40 | struct mptcp_sched_ops *mptcp_sched_find(const char *name) | |
41 | { | |
42 | struct mptcp_sched_ops *sched, *ret = NULL; | |
43 | ||
44 | list_for_each_entry_rcu(sched, &mptcp_sched_list, list) { | |
45 | if (!strcmp(sched->name, name)) { | |
46 | ret = sched; | |
47 | break; | |
48 | } | |
49 | } | |
50 | ||
51 | return ret; | |
52 | } | |
53 | ||
54 | int mptcp_register_scheduler(struct mptcp_sched_ops *sched) | |
55 | { | |
56 | if (!sched->get_subflow) | |
57 | return -EINVAL; | |
58 | ||
59 | spin_lock(&mptcp_sched_list_lock); | |
60 | if (mptcp_sched_find(sched->name)) { | |
61 | spin_unlock(&mptcp_sched_list_lock); | |
62 | return -EEXIST; | |
63 | } | |
64 | list_add_tail_rcu(&sched->list, &mptcp_sched_list); | |
65 | spin_unlock(&mptcp_sched_list_lock); | |
66 | ||
67 | pr_debug("%s registered", sched->name); | |
68 | return 0; | |
69 | } | |
70 | ||
71 | void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched) | |
72 | { | |
ed1ad86b GT |
73 | if (sched == &mptcp_sched_default) |
74 | return; | |
75 | ||
740ebe35 GT |
76 | spin_lock(&mptcp_sched_list_lock); |
77 | list_del_rcu(&sched->list); | |
78 | spin_unlock(&mptcp_sched_list_lock); | |
79 | } | |
1730b2b2 | 80 | |
ed1ad86b GT |
81 | void mptcp_sched_init(void) |
82 | { | |
83 | mptcp_register_scheduler(&mptcp_sched_default); | |
84 | } | |
85 | ||
1730b2b2 GT |
86 | int mptcp_init_sched(struct mptcp_sock *msk, |
87 | struct mptcp_sched_ops *sched) | |
88 | { | |
89 | if (!sched) | |
ed1ad86b | 90 | sched = &mptcp_sched_default; |
1730b2b2 GT |
91 | |
92 | if (!bpf_try_module_get(sched, sched->owner)) | |
93 | return -EBUSY; | |
94 | ||
95 | msk->sched = sched; | |
96 | if (msk->sched->init) | |
97 | msk->sched->init(msk); | |
98 | ||
99 | pr_debug("sched=%s", msk->sched->name); | |
100 | ||
1730b2b2 GT |
101 | return 0; |
102 | } | |
103 | ||
104 | void mptcp_release_sched(struct mptcp_sock *msk) | |
105 | { | |
106 | struct mptcp_sched_ops *sched = msk->sched; | |
107 | ||
108 | if (!sched) | |
109 | return; | |
110 | ||
111 | msk->sched = NULL; | |
112 | if (sched->release) | |
113 | sched->release(msk); | |
114 | ||
115 | bpf_module_put(sched, sched->owner); | |
116 | } | |
fce68b03 GT |
117 | |
118 | void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow, | |
119 | bool scheduled) | |
120 | { | |
121 | WRITE_ONCE(subflow->scheduled, scheduled); | |
122 | } | |
07336a87 GT |
123 | |
124 | int mptcp_sched_get_send(struct mptcp_sock *msk) | |
125 | { | |
126 | struct mptcp_subflow_context *subflow; | |
127 | struct mptcp_sched_data data; | |
128 | ||
0fa1b378 GT |
129 | msk_owned_by_me(msk); |
130 | ||
131 | /* the following check is moved out of mptcp_subflow_get_send */ | |
132 | if (__mptcp_check_fallback(msk)) { | |
133 | if (msk->first && | |
134 | __tcp_can_send(msk->first) && | |
135 | sk_stream_memory_free(msk->first)) { | |
136 | mptcp_subflow_set_scheduled(mptcp_subflow_ctx(msk->first), true); | |
137 | return 0; | |
138 | } | |
139 | return -EINVAL; | |
140 | } | |
141 | ||
07336a87 GT |
142 | mptcp_for_each_subflow(msk, subflow) { |
143 | if (READ_ONCE(subflow->scheduled)) | |
144 | return 0; | |
145 | } | |
146 | ||
07336a87 | 147 | data.reinject = false; |
ed1ad86b GT |
148 | if (msk->sched == &mptcp_sched_default || !msk->sched) |
149 | return mptcp_sched_default_get_subflow(msk, &data); | |
07336a87 GT |
150 | return msk->sched->get_subflow(msk, &data); |
151 | } | |
152 | ||
153 | int mptcp_sched_get_retrans(struct mptcp_sock *msk) | |
154 | { | |
155 | struct mptcp_subflow_context *subflow; | |
156 | struct mptcp_sched_data data; | |
157 | ||
ee2708ae GT |
158 | msk_owned_by_me(msk); |
159 | ||
160 | /* the following check is moved out of mptcp_subflow_get_retrans */ | |
161 | if (__mptcp_check_fallback(msk)) | |
162 | return -EINVAL; | |
163 | ||
07336a87 GT |
164 | mptcp_for_each_subflow(msk, subflow) { |
165 | if (READ_ONCE(subflow->scheduled)) | |
166 | return 0; | |
167 | } | |
168 | ||
07336a87 | 169 | data.reinject = true; |
ed1ad86b GT |
170 | if (msk->sched == &mptcp_sched_default || !msk->sched) |
171 | return mptcp_sched_default_get_subflow(msk, &data); | |
07336a87 GT |
172 | return msk->sched->get_subflow(msk, &data); |
173 | } |