Commit | Line | Data |
---|---|---|
2b9feef2 AE |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. | |
8e7c89d8 | 4 | * Copyright (C) 2018-2023 Linaro Ltd. |
2b9feef2 AE |
5 | */ |
6 | ||
7 | #include <linux/types.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/bits.h> | |
10 | #include <linux/bitops.h> | |
11 | #include <linux/bitfield.h> | |
12 | #include <linux/io.h> | |
13 | #include <linux/build_bug.h> | |
14 | #include <linux/device.h> | |
15 | #include <linux/dma-mapping.h> | |
16 | ||
17 | #include "ipa.h" | |
18 | #include "ipa_version.h" | |
19 | #include "ipa_endpoint.h" | |
20 | #include "ipa_table.h" | |
21 | #include "ipa_reg.h" | |
22 | #include "ipa_mem.h" | |
23 | #include "ipa_cmd.h" | |
24 | #include "gsi.h" | |
25 | #include "gsi_trans.h" | |
26 | ||
27 | /** | |
28 | * DOC: IPA Filter and Route Tables | |
29 | * | |
4ea29143 AE |
30 | * The IPA has tables defined in its local (IPA-resident) memory that define |
31 | * filter and routing rules. An entry in either of these tables is a little | |
32 | * endian 64-bit "slot" that holds the address of a rule definition. (The | |
33 | * size of these slots is 64 bits regardless of the host DMA address size.) | |
34 | * | |
bd552493 AE |
35 | * Separate tables (both filter and route) are used for IPv4 and IPv6. There |
36 | * is normally another set of "hashed" filter and route tables, which are | |
4ea29143 AE |
37 | * used with a hash of message metadata. Hashed operation is not supported |
38 | * by all IPA hardware (IPA v4.2 doesn't support hashed tables). | |
39 | * | |
40 | * Rules can be in local memory or in DRAM (system memory). The offset of | |
41 | * an object (such as a route or filter table) in IPA-resident memory must | |
42 | * 128-byte aligned. An object in system memory (such as a route or filter | |
43 | * rule) must be at an 8-byte aligned address. We currently only place | |
44 | * route or filter rules in system memory. | |
45 | * | |
2b9feef2 AE |
46 | * A rule consists of a contiguous block of 32-bit values terminated with |
47 | * 32 zero bits. A special "zero entry" rule consisting of 64 zero bits | |
48 | * represents "no filtering" or "no routing," and is the reset value for | |
4ea29143 | 49 | * filter or route table rules. |
2b9feef2 AE |
50 | * |
51 | * Each filter rule is associated with an AP or modem TX endpoint, though | |
4ea29143 | 52 | * not all TX endpoints support filtering. The first 64-bit slot in a |
2b9feef2 | 53 | * filter table is a bitmap indicating which endpoints have entries in |
bd552493 AE |
54 | * the table. Each set bit in this bitmap indicates the presence of the |
55 | * address of a filter rule in the memory following the bitmap. Until IPA | |
56 | * v5.0, the low-order bit (bit 0) in this bitmap represents a special | |
57 | * global filter, which applies to all traffic. Otherwise the position of | |
58 | * each set bit represents an endpoint for which a filter rule is defined. | |
59 | * | |
60 | * The global rule is not used in current code, and support for it is | |
61 | * removed starting at IPA v5.0. For IPA v5.0+, the endpoint bitmap | |
62 | * position defines the endpoint ID--i.e. if bit 1 is set in the endpoint | |
63 | * bitmap, endpoint 1 has a filter rule. Older versions of IPA represent | |
64 | * the presence of a filter rule for endpoint X by bit (X + 1) being set. | |
65 | * I.e., bit 1 set indicates the presence of a filter rule for endpoint 0, | |
66 | * and bit 3 set means there is a filter rule present for endpoint 2. | |
67 | * | |
68 | * Each filter table entry has the address of a set of equations that | |
69 | * implement a filter rule. So following the endpoint bitmap there | |
70 | * will be such an address/entry for each endpoint with a set bit in | |
71 | * the bitmap. | |
2b9feef2 AE |
72 | * |
73 | * The AP initializes all entries in a filter table to refer to a "zero" | |
bd552493 AE |
74 | * rule. Once initialized, the modem and AP update the entries for |
75 | * endpoints they "own" directly. Currently the AP does not use the IPA | |
76 | * filtering functionality. | |
77 | * | |
78 | * This diagram shows an example of a filter table with an endpoint | |
79 | * bitmap as defined prior to IPA v5.0. | |
2b9feef2 AE |
80 | * |
81 | * IPA Filter Table | |
82 | * ---------------------- | |
83 | * endpoint bitmap | 0x0000000000000048 | Bits 3 and 6 set (endpoints 2 and 5) | |
84 | * |--------------------| | |
85 | * 1st endpoint | 0x000123456789abc0 | DMA address for modem endpoint 2 rule | |
86 | * |--------------------| | |
87 | * 2nd endpoint | 0x000123456789abf0 | DMA address for AP endpoint 5 rule | |
88 | * |--------------------| | |
89 | * (unused) | | (Unused space in filter table) | |
90 | * |--------------------| | |
91 | * . . . | |
92 | * |--------------------| | |
93 | * (unused) | | (Unused space in filter table) | |
94 | * ---------------------- | |
95 | * | |
96 | * The set of available route rules is divided about equally between the AP | |
97 | * and modem. The AP initializes all entries in a route table to refer to | |
98 | * a "zero entry". Once initialized, the modem and AP are responsible for | |
99 | * updating their own entries. All entries in a route table are usable, | |
100 | * though the AP currently does not use the IPA routing functionality. | |
101 | * | |
102 | * IPA Route Table | |
103 | * ---------------------- | |
104 | * 1st modem route | 0x0001234500001100 | DMA address for first route rule | |
105 | * |--------------------| | |
106 | * 2nd modem route | 0x0001234500001140 | DMA address for second route rule | |
107 | * |--------------------| | |
108 | * . . . | |
109 | * |--------------------| | |
110 | * Last modem route| 0x0001234500002280 | DMA address for Nth route rule | |
111 | * |--------------------| | |
112 | * 1st AP route | 0x0001234500001100 | DMA address for route rule (N+1) | |
113 | * |--------------------| | |
114 | * 2nd AP route | 0x0001234500001140 | DMA address for next route rule | |
115 | * |--------------------| | |
116 | * . . . | |
117 | * |--------------------| | |
118 | * Last AP route | 0x0001234500002280 | DMA address for last route rule | |
119 | * ---------------------- | |
120 | */ | |
121 | ||
2b9feef2 AE |
122 | /* Filter or route rules consist of a set of 32-bit values followed by a |
123 | * 32-bit all-zero rule list terminator. The "zero rule" is simply an | |
124 | * all-zero rule followed by the list terminator. | |
125 | */ | |
126 | #define IPA_ZERO_RULE_SIZE (2 * sizeof(__le32)) | |
127 | ||
2b9feef2 AE |
128 | /* Check things that can be validated at build time. */ |
129 | static void ipa_table_validate_build(void) | |
130 | { | |
d2fd2311 AE |
131 | /* Filter and route tables contain DMA addresses that refer |
132 | * to filter or route rules. But the size of a table entry | |
133 | * is 64 bits regardless of what the size of an AP DMA address | |
134 | * is. A fixed constant defines the size of an entry, and | |
135 | * code in ipa_table_init() uses a pointer to __le64 to | |
136 | * initialize tables. | |
2b9feef2 | 137 | */ |
4ea29143 | 138 | BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(__le64)); |
2b9feef2 AE |
139 | |
140 | /* A "zero rule" is used to represent no filtering or no routing. | |
141 | * It is a 64-bit block of zeroed memory. Code in ipa_table_init() | |
142 | * assumes that it can be written using a pointer to __le64. | |
143 | */ | |
144 | BUILD_BUG_ON(IPA_ZERO_RULE_SIZE != sizeof(__le64)); | |
cf139196 | 145 | } |
2b9feef2 | 146 | |
cf139196 AE |
147 | static const struct ipa_mem * |
148 | ipa_table_mem(struct ipa *ipa, bool filter, bool hashed, bool ipv6) | |
149 | { | |
150 | enum ipa_mem_id mem_id; | |
151 | ||
152 | mem_id = filter ? hashed ? ipv6 ? IPA_MEM_V6_FILTER_HASHED | |
153 | : IPA_MEM_V4_FILTER_HASHED | |
154 | : ipv6 ? IPA_MEM_V6_FILTER | |
155 | : IPA_MEM_V4_FILTER | |
156 | : hashed ? ipv6 ? IPA_MEM_V6_ROUTE_HASHED | |
157 | : IPA_MEM_V4_ROUTE_HASHED | |
158 | : ipv6 ? IPA_MEM_V6_ROUTE | |
159 | : IPA_MEM_V4_ROUTE; | |
160 | ||
161 | return ipa_mem_find(ipa, mem_id); | |
2b9feef2 AE |
162 | } |
163 | ||
0f97fbd4 | 164 | bool ipa_filtered_valid(struct ipa *ipa, u64 filtered) |
2b9feef2 AE |
165 | { |
166 | struct device *dev = &ipa->pdev->dev; | |
167 | u32 count; | |
168 | ||
0f97fbd4 | 169 | if (!filtered) { |
2b9feef2 AE |
170 | dev_err(dev, "at least one filtering endpoint is required\n"); |
171 | ||
172 | return false; | |
173 | } | |
174 | ||
0f97fbd4 | 175 | count = hweight64(filtered); |
f787d848 | 176 | if (count > ipa->filter_count) { |
0f97fbd4 | 177 | dev_err(dev, "too many filtering endpoints (%u > %u)\n", |
f787d848 | 178 | count, ipa->filter_count); |
2b9feef2 AE |
179 | |
180 | return false; | |
181 | } | |
182 | ||
183 | return true; | |
184 | } | |
185 | ||
2b9feef2 AE |
186 | /* Zero entry count means no table, so just return a 0 address */ |
187 | static dma_addr_t ipa_table_addr(struct ipa *ipa, bool filter_mask, u16 count) | |
188 | { | |
189 | u32 skip; | |
190 | ||
191 | if (!count) | |
192 | return 0; | |
193 | ||
f787d848 | 194 | WARN_ON(count > max_t(u32, ipa->filter_count, ipa->route_count)); |
2b9feef2 AE |
195 | |
196 | /* Skip over the zero rule and possibly the filter mask */ | |
197 | skip = filter_mask ? 1 : 2; | |
198 | ||
199 | return ipa->table_addr + skip * sizeof(*ipa->table_virt); | |
200 | } | |
201 | ||
202 | static void ipa_table_reset_add(struct gsi_trans *trans, bool filter, | |
6337b147 | 203 | bool hashed, bool ipv6, u16 first, u16 count) |
2b9feef2 AE |
204 | { |
205 | struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); | |
6337b147 | 206 | const struct ipa_mem *mem; |
2b9feef2 AE |
207 | dma_addr_t addr; |
208 | u32 offset; | |
209 | u16 size; | |
210 | ||
6337b147 AE |
211 | /* Nothing to do if the memory region is doesn't exist or is empty */ |
212 | mem = ipa_table_mem(ipa, filter, hashed, ipv6); | |
213 | if (!mem || !mem->size) | |
2b9feef2 AE |
214 | return; |
215 | ||
216 | if (filter) | |
217 | first++; /* skip over bitmap */ | |
218 | ||
4ea29143 AE |
219 | offset = mem->offset + first * sizeof(__le64); |
220 | size = count * sizeof(__le64); | |
2b9feef2 AE |
221 | addr = ipa_table_addr(ipa, false, count); |
222 | ||
223 | ipa_cmd_dma_shared_mem_add(trans, offset, size, addr, true); | |
224 | } | |
225 | ||
226 | /* Reset entries in a single filter table belonging to either the AP or | |
227 | * modem to refer to the zero entry. The memory region supplied will be | |
228 | * for the IPv4 and IPv6 non-hashed and hashed filter tables. | |
229 | */ | |
230 | static int | |
6337b147 | 231 | ipa_filter_reset_table(struct ipa *ipa, bool hashed, bool ipv6, bool modem) |
2b9feef2 | 232 | { |
0f97fbd4 | 233 | u64 ep_mask = ipa->filtered; |
2b9feef2 AE |
234 | struct gsi_trans *trans; |
235 | enum gsi_ee_id ee_id; | |
236 | ||
0f97fbd4 | 237 | trans = ipa_cmd_trans_alloc(ipa, hweight64(ep_mask)); |
2b9feef2 AE |
238 | if (!trans) { |
239 | dev_err(&ipa->pdev->dev, | |
240 | "no transaction for %s filter reset\n", | |
241 | modem ? "modem" : "AP"); | |
242 | return -EBUSY; | |
243 | } | |
244 | ||
245 | ee_id = modem ? GSI_EE_MODEM : GSI_EE_AP; | |
246 | while (ep_mask) { | |
247 | u32 endpoint_id = __ffs(ep_mask); | |
248 | struct ipa_endpoint *endpoint; | |
249 | ||
250 | ep_mask ^= BIT(endpoint_id); | |
251 | ||
252 | endpoint = &ipa->endpoint[endpoint_id]; | |
253 | if (endpoint->ee_id != ee_id) | |
254 | continue; | |
255 | ||
6337b147 | 256 | ipa_table_reset_add(trans, true, hashed, ipv6, endpoint_id, 1); |
2b9feef2 AE |
257 | } |
258 | ||
259 | gsi_trans_commit_wait(trans); | |
260 | ||
261 | return 0; | |
262 | } | |
263 | ||
264 | /* Theoretically, each filter table could have more filter slots to | |
265 | * update than the maximum number of commands in a transaction. So | |
266 | * we do each table separately. | |
267 | */ | |
268 | static int ipa_filter_reset(struct ipa *ipa, bool modem) | |
269 | { | |
270 | int ret; | |
271 | ||
6337b147 | 272 | ret = ipa_filter_reset_table(ipa, false, false, modem); |
2b9feef2 AE |
273 | if (ret) |
274 | return ret; | |
275 | ||
e11ec2b8 AE |
276 | ret = ipa_filter_reset_table(ipa, false, true, modem); |
277 | if (ret || !ipa_table_hash_support(ipa)) | |
2b9feef2 AE |
278 | return ret; |
279 | ||
e11ec2b8 | 280 | ret = ipa_filter_reset_table(ipa, true, false, modem); |
2b9feef2 AE |
281 | if (ret) |
282 | return ret; | |
2b9feef2 | 283 | |
e11ec2b8 | 284 | return ipa_filter_reset_table(ipa, true, true, modem); |
2b9feef2 AE |
285 | } |
286 | ||
287 | /* The AP routes and modem routes are each contiguous within the | |
288 | * table. We can update each table with a single command, and we | |
289 | * won't exceed the per-transaction command limit. | |
290 | * */ | |
291 | static int ipa_route_reset(struct ipa *ipa, bool modem) | |
292 | { | |
e11ec2b8 | 293 | bool hash_support = ipa_table_hash_support(ipa); |
8defab8b | 294 | u32 modem_route_count = ipa->modem_route_count; |
2b9feef2 AE |
295 | struct gsi_trans *trans; |
296 | u16 first; | |
297 | u16 count; | |
298 | ||
e11ec2b8 | 299 | trans = ipa_cmd_trans_alloc(ipa, hash_support ? 4 : 2); |
2b9feef2 AE |
300 | if (!trans) { |
301 | dev_err(&ipa->pdev->dev, | |
302 | "no transaction for %s route reset\n", | |
303 | modem ? "modem" : "AP"); | |
304 | return -EBUSY; | |
305 | } | |
306 | ||
307 | if (modem) { | |
fb4014ac | 308 | first = 0; |
8defab8b | 309 | count = modem_route_count; |
2b9feef2 | 310 | } else { |
8defab8b AE |
311 | first = modem_route_count; |
312 | count = ipa->route_count - modem_route_count; | |
2b9feef2 AE |
313 | } |
314 | ||
6337b147 | 315 | ipa_table_reset_add(trans, false, false, false, first, count); |
6337b147 | 316 | ipa_table_reset_add(trans, false, false, true, first, count); |
e11ec2b8 AE |
317 | |
318 | if (hash_support) { | |
319 | ipa_table_reset_add(trans, false, true, false, first, count); | |
320 | ipa_table_reset_add(trans, false, true, true, first, count); | |
321 | } | |
2b9feef2 AE |
322 | |
323 | gsi_trans_commit_wait(trans); | |
324 | ||
325 | return 0; | |
326 | } | |
327 | ||
328 | void ipa_table_reset(struct ipa *ipa, bool modem) | |
329 | { | |
330 | struct device *dev = &ipa->pdev->dev; | |
331 | const char *ee_name; | |
332 | int ret; | |
333 | ||
334 | ee_name = modem ? "modem" : "AP"; | |
335 | ||
336 | /* Report errors, but reset filter and route tables */ | |
337 | ret = ipa_filter_reset(ipa, modem); | |
338 | if (ret) | |
339 | dev_err(dev, "error %d resetting filter table for %s\n", | |
340 | ret, ee_name); | |
341 | ||
342 | ret = ipa_route_reset(ipa, modem); | |
343 | if (ret) | |
344 | dev_err(dev, "error %d resetting route table for %s\n", | |
345 | ret, ee_name); | |
346 | } | |
347 | ||
348 | int ipa_table_hash_flush(struct ipa *ipa) | |
349 | { | |
2b9feef2 | 350 | struct gsi_trans *trans; |
81772e44 | 351 | const struct reg *reg; |
2b9feef2 AE |
352 | u32 val; |
353 | ||
a266ad6b | 354 | if (!ipa_table_hash_support(ipa)) |
2b9feef2 AE |
355 | return 0; |
356 | ||
357 | trans = ipa_cmd_trans_alloc(ipa, 1); | |
358 | if (!trans) { | |
359 | dev_err(&ipa->pdev->dev, "no transaction for hash flush\n"); | |
360 | return -EBUSY; | |
361 | } | |
362 | ||
8e7c89d8 AE |
363 | if (ipa->version < IPA_VERSION_5_0) { |
364 | reg = ipa_reg(ipa, FILT_ROUT_HASH_FLUSH); | |
6a244b75 | 365 | |
f1470fd7 AE |
366 | val = reg_bit(reg, IPV6_ROUTER_HASH); |
367 | val |= reg_bit(reg, IPV6_FILTER_HASH); | |
368 | val |= reg_bit(reg, IPV4_ROUTER_HASH); | |
369 | val |= reg_bit(reg, IPV4_FILTER_HASH); | |
8e7c89d8 AE |
370 | } else { |
371 | reg = ipa_reg(ipa, FILT_ROUT_CACHE_FLUSH); | |
8e7c89d8 AE |
372 | |
373 | /* IPA v5.0+ uses a unified cache (both IPv4 and IPv6) */ | |
f1470fd7 AE |
374 | val = reg_bit(reg, ROUTER_CACHE); |
375 | val |= reg_bit(reg, FILTER_CACHE); | |
8e7c89d8 | 376 | } |
2b9feef2 | 377 | |
fc4cecf7 | 378 | ipa_cmd_register_write_add(trans, reg_offset(reg), val, val, false); |
2b9feef2 AE |
379 | |
380 | gsi_trans_commit_wait(trans); | |
381 | ||
382 | return 0; | |
383 | } | |
384 | ||
5cb76899 | 385 | static void ipa_table_init_add(struct gsi_trans *trans, bool filter, bool ipv6) |
2b9feef2 AE |
386 | { |
387 | struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); | |
5cb76899 AE |
388 | const struct ipa_mem *hash_mem; |
389 | enum ipa_cmd_opcode opcode; | |
390 | const struct ipa_mem *mem; | |
2b9feef2 AE |
391 | dma_addr_t hash_addr; |
392 | dma_addr_t addr; | |
15b4f993 | 393 | u32 hash_offset; |
dc901505 | 394 | u32 zero_offset; |
2b9feef2 | 395 | u16 hash_count; |
dc901505 | 396 | u32 zero_size; |
2b9feef2 AE |
397 | u16 hash_size; |
398 | u16 count; | |
399 | u16 size; | |
400 | ||
5cb76899 AE |
401 | opcode = filter ? ipv6 ? IPA_CMD_IP_V6_FILTER_INIT |
402 | : IPA_CMD_IP_V4_FILTER_INIT | |
403 | : ipv6 ? IPA_CMD_IP_V6_ROUTING_INIT | |
404 | : IPA_CMD_IP_V4_ROUTING_INIT; | |
405 | ||
15b4f993 | 406 | /* The non-hashed region will exist (see ipa_table_mem_valid()) */ |
5cb76899 AE |
407 | mem = ipa_table_mem(ipa, filter, false, ipv6); |
408 | hash_mem = ipa_table_mem(ipa, filter, true, ipv6); | |
15b4f993 | 409 | hash_offset = hash_mem ? hash_mem->offset : 0; |
5cb76899 | 410 | |
dc901505 | 411 | /* Compute the number of table entries to initialize */ |
2b9feef2 | 412 | if (filter) { |
dc901505 AE |
413 | /* The number of filtering endpoints determines number of |
414 | * entries in the filter table; we also add one more "slot" | |
415 | * to hold the bitmap itself. The size of the hashed filter | |
416 | * table is either the same as the non-hashed one, or zero. | |
417 | */ | |
0f97fbd4 | 418 | count = 1 + hweight64(ipa->filtered); |
5cb76899 | 419 | hash_count = hash_mem && hash_mem->size ? count : 0; |
2b9feef2 | 420 | } else { |
dc901505 AE |
421 | /* The size of a route table region determines the number |
422 | * of entries it has. | |
423 | */ | |
4ea29143 | 424 | count = mem->size / sizeof(__le64); |
15b4f993 | 425 | hash_count = hash_mem ? hash_mem->size / sizeof(__le64) : 0; |
2b9feef2 | 426 | } |
4ea29143 AE |
427 | size = count * sizeof(__le64); |
428 | hash_size = hash_count * sizeof(__le64); | |
2b9feef2 AE |
429 | |
430 | addr = ipa_table_addr(ipa, filter, count); | |
431 | hash_addr = ipa_table_addr(ipa, filter, hash_count); | |
432 | ||
433 | ipa_cmd_table_init_add(trans, opcode, size, mem->offset, addr, | |
15b4f993 | 434 | hash_size, hash_offset, hash_addr); |
dc901505 AE |
435 | if (!filter) |
436 | return; | |
437 | ||
438 | /* Zero the unused space in the filter table */ | |
439 | zero_offset = mem->offset + size; | |
440 | zero_size = mem->size - size; | |
441 | ipa_cmd_dma_shared_mem_add(trans, zero_offset, zero_size, | |
442 | ipa->zero_addr, true); | |
443 | if (!hash_size) | |
444 | return; | |
445 | ||
446 | /* Zero the unused space in the hashed filter table */ | |
15b4f993 | 447 | zero_offset = hash_offset + hash_size; |
dc901505 AE |
448 | zero_size = hash_mem->size - hash_size; |
449 | ipa_cmd_dma_shared_mem_add(trans, zero_offset, zero_size, | |
450 | ipa->zero_addr, true); | |
2b9feef2 AE |
451 | } |
452 | ||
453 | int ipa_table_setup(struct ipa *ipa) | |
454 | { | |
455 | struct gsi_trans *trans; | |
456 | ||
dc901505 AE |
457 | /* We will need at most 8 TREs: |
458 | * - IPv4: | |
459 | * - One for route table initialization (non-hashed and hashed) | |
460 | * - One for filter table initialization (non-hashed and hashed) | |
461 | * - One to zero unused entries in the non-hashed filter table | |
462 | * - One to zero unused entries in the hashed filter table | |
463 | * - IPv6: | |
464 | * - One for route table initialization (non-hashed and hashed) | |
465 | * - One for filter table initialization (non-hashed and hashed) | |
466 | * - One to zero unused entries in the non-hashed filter table | |
467 | * - One to zero unused entries in the hashed filter table | |
468 | * All platforms support at least 8 TREs in a transaction. | |
469 | */ | |
470 | trans = ipa_cmd_trans_alloc(ipa, 8); | |
2b9feef2 AE |
471 | if (!trans) { |
472 | dev_err(&ipa->pdev->dev, "no transaction for table setup\n"); | |
473 | return -EBUSY; | |
474 | } | |
475 | ||
5cb76899 AE |
476 | ipa_table_init_add(trans, false, false); |
477 | ipa_table_init_add(trans, false, true); | |
478 | ipa_table_init_add(trans, true, false); | |
479 | ipa_table_init_add(trans, true, true); | |
2b9feef2 AE |
480 | |
481 | gsi_trans_commit_wait(trans); | |
482 | ||
483 | return 0; | |
484 | } | |
485 | ||
2b9feef2 AE |
486 | /** |
487 | * ipa_filter_tuple_zero() - Zero an endpoint's hashed filter tuple | |
e3eea08e | 488 | * @endpoint: Endpoint whose filter hash tuple should be zeroed |
2b9feef2 AE |
489 | * |
490 | * Endpoint must be for the AP (not modem) and support filtering. Updates | |
491 | * the filter hash values without changing route ones. | |
492 | */ | |
493 | static void ipa_filter_tuple_zero(struct ipa_endpoint *endpoint) | |
494 | { | |
495 | u32 endpoint_id = endpoint->endpoint_id; | |
6bfb7538 | 496 | struct ipa *ipa = endpoint->ipa; |
81772e44 | 497 | const struct reg *reg; |
2b9feef2 AE |
498 | u32 offset; |
499 | u32 val; | |
500 | ||
a08cedc3 AE |
501 | if (ipa->version < IPA_VERSION_5_0) { |
502 | reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG); | |
503 | ||
fc4cecf7 | 504 | offset = reg_n_offset(reg, endpoint_id); |
a08cedc3 | 505 | val = ioread32(endpoint->ipa->reg_virt + offset); |
2b9feef2 | 506 | |
a08cedc3 | 507 | /* Zero all filter-related fields, preserving the rest */ |
f1470fd7 | 508 | val &= ~reg_fmask(reg, FILTER_HASH_MSK_ALL); |
a08cedc3 AE |
509 | } else { |
510 | /* IPA v5.0 separates filter and router cache configuration */ | |
511 | reg = ipa_reg(ipa, ENDP_FILTER_CACHE_CFG); | |
fc4cecf7 | 512 | offset = reg_n_offset(reg, endpoint_id); |
2b9feef2 | 513 | |
a08cedc3 AE |
514 | /* Zero all filter-related fields */ |
515 | val = 0; | |
516 | } | |
2b9feef2 AE |
517 | |
518 | iowrite32(val, endpoint->ipa->reg_virt + offset); | |
519 | } | |
520 | ||
74858b63 | 521 | /* Configure a hashed filter table; there is no ipa_filter_deconfig() */ |
2b9feef2 AE |
522 | static void ipa_filter_config(struct ipa *ipa, bool modem) |
523 | { | |
524 | enum gsi_ee_id ee_id = modem ? GSI_EE_MODEM : GSI_EE_AP; | |
0f97fbd4 | 525 | u64 ep_mask = ipa->filtered; |
2b9feef2 | 526 | |
a266ad6b | 527 | if (!ipa_table_hash_support(ipa)) |
2b9feef2 AE |
528 | return; |
529 | ||
530 | while (ep_mask) { | |
531 | u32 endpoint_id = __ffs(ep_mask); | |
532 | struct ipa_endpoint *endpoint; | |
533 | ||
534 | ep_mask ^= BIT(endpoint_id); | |
535 | ||
536 | endpoint = &ipa->endpoint[endpoint_id]; | |
537 | if (endpoint->ee_id == ee_id) | |
538 | ipa_filter_tuple_zero(endpoint); | |
539 | } | |
540 | } | |
541 | ||
8defab8b | 542 | static bool ipa_route_id_modem(struct ipa *ipa, u32 route_id) |
2b9feef2 | 543 | { |
8defab8b | 544 | return route_id < ipa->modem_route_count; |
2b9feef2 AE |
545 | } |
546 | ||
547 | /** | |
548 | * ipa_route_tuple_zero() - Zero a hashed route table entry tuple | |
e3eea08e | 549 | * @ipa: IPA pointer |
2b9feef2 AE |
550 | * @route_id: Route table entry whose hash tuple should be zeroed |
551 | * | |
552 | * Updates the route hash values without changing filter ones. | |
553 | */ | |
554 | static void ipa_route_tuple_zero(struct ipa *ipa, u32 route_id) | |
555 | { | |
81772e44 | 556 | const struct reg *reg; |
6bfb7538 | 557 | u32 offset; |
2b9feef2 AE |
558 | u32 val; |
559 | ||
a08cedc3 AE |
560 | if (ipa->version < IPA_VERSION_5_0) { |
561 | reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG); | |
fc4cecf7 | 562 | offset = reg_n_offset(reg, route_id); |
a08cedc3 AE |
563 | |
564 | val = ioread32(ipa->reg_virt + offset); | |
6bfb7538 | 565 | |
a08cedc3 | 566 | /* Zero all route-related fields, preserving the rest */ |
f1470fd7 | 567 | val &= ~reg_fmask(reg, ROUTER_HASH_MSK_ALL); |
a08cedc3 AE |
568 | } else { |
569 | /* IPA v5.0 separates filter and router cache configuration */ | |
570 | reg = ipa_reg(ipa, ENDP_ROUTER_CACHE_CFG); | |
fc4cecf7 | 571 | offset = reg_n_offset(reg, route_id); |
2b9feef2 | 572 | |
a08cedc3 AE |
573 | /* Zero all route-related fields */ |
574 | val = 0; | |
575 | } | |
2b9feef2 AE |
576 | |
577 | iowrite32(val, ipa->reg_virt + offset); | |
578 | } | |
579 | ||
74858b63 | 580 | /* Configure a hashed route table; there is no ipa_route_deconfig() */ |
2b9feef2 AE |
581 | static void ipa_route_config(struct ipa *ipa, bool modem) |
582 | { | |
583 | u32 route_id; | |
584 | ||
a266ad6b | 585 | if (!ipa_table_hash_support(ipa)) |
2b9feef2 AE |
586 | return; |
587 | ||
fc094058 | 588 | for (route_id = 0; route_id < ipa->route_count; route_id++) |
8defab8b | 589 | if (ipa_route_id_modem(ipa, route_id) == modem) |
2b9feef2 AE |
590 | ipa_route_tuple_zero(ipa, route_id); |
591 | } | |
592 | ||
74858b63 | 593 | /* Configure a filter and route tables; there is no ipa_table_deconfig() */ |
2b9feef2 AE |
594 | void ipa_table_config(struct ipa *ipa) |
595 | { | |
596 | ipa_filter_config(ipa, false); | |
597 | ipa_filter_config(ipa, true); | |
598 | ipa_route_config(ipa, false); | |
599 | ipa_route_config(ipa, true); | |
600 | } | |
601 | ||
fc094058 AE |
602 | /* Verify the sizes of all IPA table filter or routing table memory regions |
603 | * are valid. If valid, this records the size of the routing table. | |
604 | */ | |
8defab8b | 605 | bool ipa_table_mem_valid(struct ipa *ipa, bool filter) |
cf139196 AE |
606 | { |
607 | bool hash_support = ipa_table_hash_support(ipa); | |
cf139196 AE |
608 | const struct ipa_mem *mem_hashed; |
609 | const struct ipa_mem *mem_ipv4; | |
610 | const struct ipa_mem *mem_ipv6; | |
611 | u32 count; | |
612 | ||
613 | /* IPv4 and IPv6 non-hashed tables are expected to be defined and | |
614 | * have the same size. Both must have at least two entries (and | |
615 | * would normally have more than that). | |
616 | */ | |
617 | mem_ipv4 = ipa_table_mem(ipa, filter, false, false); | |
618 | if (!mem_ipv4) | |
619 | return false; | |
620 | ||
621 | mem_ipv6 = ipa_table_mem(ipa, filter, false, true); | |
622 | if (!mem_ipv6) | |
623 | return false; | |
624 | ||
625 | if (mem_ipv4->size != mem_ipv6->size) | |
626 | return false; | |
627 | ||
f787d848 | 628 | /* Compute and record the number of entries for each table type */ |
0439e674 AE |
629 | count = mem_ipv4->size / sizeof(__le64); |
630 | if (count < 2) | |
631 | return false; | |
f787d848 AE |
632 | if (filter) |
633 | ipa->filter_count = count - 1; /* Filter map in first entry */ | |
634 | else | |
0439e674 | 635 | ipa->route_count = count; |
fc094058 | 636 | |
5444b0ea AE |
637 | /* Table offset and size must fit in TABLE_INIT command fields */ |
638 | if (!ipa_cmd_table_init_valid(ipa, mem_ipv4, !filter)) | |
639 | return false; | |
640 | ||
cf139196 | 641 | /* Make sure the regions are big enough */ |
cf139196 AE |
642 | if (filter) { |
643 | /* Filter tables must able to hold the endpoint bitmap plus | |
644 | * an entry for each endpoint that supports filtering | |
645 | */ | |
0f97fbd4 | 646 | if (count < 1 + hweight64(ipa->filtered)) |
cf139196 AE |
647 | return false; |
648 | } else { | |
649 | /* Routing tables must be able to hold all modem entries, | |
650 | * plus at least one entry for the AP. | |
651 | */ | |
8defab8b | 652 | if (count < ipa->modem_route_count + 1) |
cf139196 AE |
653 | return false; |
654 | } | |
655 | ||
656 | /* If hashing is supported, hashed tables are expected to be defined, | |
657 | * and have the same size as non-hashed tables. If hashing is not | |
658 | * supported, hashed tables are expected to have zero size (or not | |
659 | * be defined). | |
660 | */ | |
661 | mem_hashed = ipa_table_mem(ipa, filter, true, false); | |
662 | if (hash_support) { | |
663 | if (!mem_hashed || mem_hashed->size != mem_ipv4->size) | |
664 | return false; | |
665 | } else { | |
666 | if (mem_hashed && mem_hashed->size) | |
667 | return false; | |
668 | } | |
669 | ||
670 | /* Same check for IPv6 tables */ | |
671 | mem_hashed = ipa_table_mem(ipa, filter, true, true); | |
672 | if (hash_support) { | |
673 | if (!mem_hashed || mem_hashed->size != mem_ipv6->size) | |
674 | return false; | |
675 | } else { | |
676 | if (mem_hashed && mem_hashed->size) | |
677 | return false; | |
678 | } | |
679 | ||
680 | return true; | |
681 | } | |
682 | ||
683 | /* Initialize a coherent DMA allocation containing initialized filter and | |
2b9feef2 AE |
684 | * route table data. This is used when initializing or resetting the IPA |
685 | * filter or route table. | |
686 | * | |
687 | * The first entry in a filter table contains a bitmap indicating which | |
688 | * endpoints contain entries in the table. In addition to that first entry, | |
f787d848 | 689 | * there is a fixed maximum number of entries that follow. Filter table |
2b9feef2 AE |
690 | * entries are 64 bits wide, and (other than the bitmap) contain the DMA |
691 | * address of a filter rule. A "zero rule" indicates no filtering, and | |
692 | * consists of 64 bits of zeroes. When a filter table is initialized (or | |
693 | * reset) its entries are made to refer to the zero rule. | |
694 | * | |
695 | * Each entry in a route table is the DMA address of a routing rule. For | |
696 | * routing there is also a 64-bit "zero rule" that means no routing, and | |
697 | * when a route table is initialized or reset, its entries are made to refer | |
698 | * to the zero rule. The zero rule is shared for route and filter tables. | |
699 | * | |
2b9feef2 AE |
700 | * +-------------------+ |
701 | * --> | zero rule | | |
702 | * / |-------------------| | |
703 | * | | filter mask | | |
704 | * |\ |-------------------| | |
705 | * | ---- zero rule address | \ | |
706 | * |\ |-------------------| | | |
f787d848 | 707 | * | ---- zero rule address | | Max IPA filter count |
fc094058 | 708 | * | |-------------------| > or IPA route count, |
2b9feef2 AE |
709 | * | ... | whichever is greater |
710 | * \ |-------------------| | | |
711 | * ---- zero rule address | / | |
712 | * +-------------------+ | |
713 | */ | |
714 | int ipa_table_init(struct ipa *ipa) | |
715 | { | |
2b9feef2 AE |
716 | struct device *dev = &ipa->pdev->dev; |
717 | dma_addr_t addr; | |
718 | __le64 le_addr; | |
719 | __le64 *virt; | |
720 | size_t size; | |
fc094058 | 721 | u32 count; |
2b9feef2 AE |
722 | |
723 | ipa_table_validate_build(); | |
724 | ||
f787d848 | 725 | count = max_t(u32, ipa->filter_count, ipa->route_count); |
fc094058 | 726 | |
19aaf72c AE |
727 | /* The IPA hardware requires route and filter table rules to be |
728 | * aligned on a 128-byte boundary. We put the "zero rule" at the | |
729 | * base of the table area allocated here. The DMA address returned | |
730 | * by dma_alloc_coherent() is guaranteed to be a power-of-2 number | |
731 | * of pages, which satisfies the rule alignment requirement. | |
732 | */ | |
4ea29143 | 733 | size = IPA_ZERO_RULE_SIZE + (1 + count) * sizeof(__le64); |
2b9feef2 AE |
734 | virt = dma_alloc_coherent(dev, size, &addr, GFP_KERNEL); |
735 | if (!virt) | |
736 | return -ENOMEM; | |
737 | ||
738 | ipa->table_virt = virt; | |
739 | ipa->table_addr = addr; | |
740 | ||
741 | /* First slot is the zero rule */ | |
742 | *virt++ = 0; | |
743 | ||
bd552493 AE |
744 | /* Next is the filter table bitmap. The "soft" bitmap value might |
745 | * need to be converted to the hardware representation by shifting | |
746 | * it left one position. Prior to IPA v5.0, bit 0 repesents global | |
747 | * filtering, which is possible but not used. IPA v5.0+ eliminated | |
748 | * that option, so there's no shifting required. | |
2b9feef2 | 749 | */ |
bd552493 | 750 | if (ipa->version < IPA_VERSION_5_0) |
0f97fbd4 | 751 | *virt++ = cpu_to_le64(ipa->filtered << 1); |
bd552493 | 752 | else |
0f97fbd4 | 753 | *virt++ = cpu_to_le64(ipa->filtered); |
2b9feef2 AE |
754 | |
755 | /* All the rest contain the DMA address of the zero rule */ | |
756 | le_addr = cpu_to_le64(addr); | |
757 | while (count--) | |
758 | *virt++ = le_addr; | |
759 | ||
760 | return 0; | |
761 | } | |
762 | ||
763 | void ipa_table_exit(struct ipa *ipa) | |
764 | { | |
f787d848 | 765 | u32 count = max_t(u32, 1 + ipa->filter_count, ipa->route_count); |
2b9feef2 AE |
766 | struct device *dev = &ipa->pdev->dev; |
767 | size_t size; | |
768 | ||
4ea29143 | 769 | size = IPA_ZERO_RULE_SIZE + (1 + count) * sizeof(__le64); |
2b9feef2 AE |
770 | |
771 | dma_free_coherent(dev, size, ipa->table_virt, ipa->table_addr); | |
772 | ipa->table_addr = 0; | |
773 | ipa->table_virt = NULL; | |
774 | } |