Commit | Line | Data |
---|---|---|
eb50fd3a | 1 | // SPDX-License-Identifier: GPL-2.0 |
355a7058 AB |
2 | /* |
3 | * Loopback bridge driver for the Greybus loopback module. | |
4 | * | |
5 | * Copyright 2014 Google Inc. | |
6 | * Copyright 2014 Linaro Ltd. | |
355a7058 | 7 | */ |
8923c5b5 JH |
8 | |
9 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
10 | ||
355a7058 AB |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> | |
85d678c0 | 13 | #include <linux/mutex.h> |
355a7058 AB |
14 | #include <linux/slab.h> |
15 | #include <linux/kthread.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/random.h> | |
5679f783 | 18 | #include <linux/sizes.h> |
cbd204b4 BD |
19 | #include <linux/cdev.h> |
20 | #include <linux/fs.h> | |
21 | #include <linux/kfifo.h> | |
aa27bf82 | 22 | #include <linux/debugfs.h> |
5015115d | 23 | #include <linux/list_sort.h> |
2e238d71 | 24 | #include <linux/spinlock.h> |
12927835 | 25 | #include <linux/workqueue.h> |
36f241ff | 26 | #include <linux/atomic.h> |
e854ff58 | 27 | #include <linux/pm_runtime.h> |
cbd204b4 | 28 | |
1ffc12be JH |
29 | #include <asm/div64.h> |
30 | ||
355a7058 | 31 | #include "greybus.h" |
fd58926e | 32 | #include "connection.h" |
355a7058 | 33 | |
fd489e1a BD |
34 | #define NSEC_PER_DAY 86400000000000ULL |
35 | ||
355a7058 | 36 | struct gb_loopback_stats { |
e051c82b AE |
37 | u32 min; |
38 | u32 max; | |
00af6583 | 39 | u64 sum; |
e13fc28f | 40 | u32 count; |
355a7058 AB |
41 | }; |
42 | ||
aa27bf82 BD |
43 | struct gb_loopback_device { |
44 | struct dentry *root; | |
45 | u32 count; | |
8e1d6c33 | 46 | size_t size_max; |
67d1eece | 47 | |
2e238d71 BD |
48 | /* We need to take a lock in atomic context */ |
49 | spinlock_t lock; | |
67d1eece | 50 | struct list_head list; |
12927835 BD |
51 | struct list_head list_op_async; |
52 | wait_queue_head_t wq; | |
aa27bf82 BD |
53 | }; |
54 | ||
55 | static struct gb_loopback_device gb_dev; | |
56 | ||
12927835 BD |
57 | struct gb_loopback_async_operation { |
58 | struct gb_loopback *gb; | |
59 | struct gb_operation *operation; | |
31408d16 | 60 | ktime_t ts; |
12927835 BD |
61 | int (*completion)(struct gb_loopback_async_operation *op_async); |
62 | }; | |
63 | ||
355a7058 AB |
64 | struct gb_loopback { |
65 | struct gb_connection *connection; | |
355a7058 | 66 | |
aa27bf82 | 67 | struct dentry *file; |
4b0ea00c | 68 | struct kfifo kfifo_lat; |
85d678c0 | 69 | struct mutex mutex; |
355a7058 | 70 | struct task_struct *task; |
67d1eece | 71 | struct list_head entry; |
079fa32b | 72 | struct device *dev; |
8e1d6c33 | 73 | wait_queue_head_t wq; |
36f241ff BD |
74 | wait_queue_head_t wq_completion; |
75 | atomic_t outstanding_operations; | |
355a7058 | 76 | |
67d1eece | 77 | /* Per connection stats */ |
31408d16 | 78 | ktime_t ts; |
355a7058 AB |
79 | struct gb_loopback_stats latency; |
80 | struct gb_loopback_stats throughput; | |
583cbf50 | 81 | struct gb_loopback_stats requests_per_second; |
1ec5843e | 82 | struct gb_loopback_stats apbridge_unipro_latency; |
e54b106d | 83 | struct gb_loopback_stats gbphy_firmware_latency; |
67d1eece | 84 | |
8e1d6c33 | 85 | int type; |
12927835 | 86 | int async; |
079fa32b | 87 | int id; |
8e1d6c33 BD |
88 | u32 size; |
89 | u32 iteration_max; | |
67d1eece | 90 | u32 iteration_count; |
b36f04fa | 91 | int us_wait; |
355a7058 | 92 | u32 error; |
12927835 BD |
93 | u32 requests_completed; |
94 | u32 requests_timedout; | |
95 | u32 timeout; | |
96 | u32 jiffy_timeout; | |
97 | u32 timeout_min; | |
98 | u32 timeout_max; | |
8e3fba55 | 99 | u32 outstanding_operations_max; |
8e1d6c33 BD |
100 | u32 lbid; |
101 | u64 elapsed_nsecs; | |
e6227ee6 | 102 | u32 apbridge_latency_ts; |
e54b106d | 103 | u32 gbphy_latency_ts; |
01480ba3 AB |
104 | |
105 | u32 send_count; | |
355a7058 AB |
106 | }; |
107 | ||
079fa32b AH |
108 | static struct class loopback_class = { |
109 | .name = "gb_loopback", | |
110 | .owner = THIS_MODULE, | |
111 | }; | |
112 | static DEFINE_IDA(loopback_ida); | |
113 | ||
12927835 BD |
114 | /* Min/max values in jiffies */ |
115 | #define GB_LOOPBACK_TIMEOUT_MIN 1 | |
116 | #define GB_LOOPBACK_TIMEOUT_MAX 10000 | |
117 | ||
cbd204b4 BD |
118 | #define GB_LOOPBACK_FIFO_DEFAULT 8192 |
119 | ||
1d3dfbd1 | 120 | static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT; |
cbd204b4 BD |
121 | module_param(kfifo_depth, uint, 0444); |
122 | ||
cbd204b4 BD |
123 | /* Maximum size of any one send data buffer we support */ |
124 | #define MAX_PACKET_SIZE (PAGE_SIZE * 2) | |
125 | ||
b36f04fa | 126 | #define GB_LOOPBACK_US_WAIT_MAX 1000000 |
355a7058 | 127 | |
355a7058 | 128 | /* interface sysfs attributes */ |
8e1d6c33 BD |
129 | #define gb_loopback_ro_attr(field) \ |
130 | static ssize_t field##_show(struct device *dev, \ | |
355a7058 AB |
131 | struct device_attribute *attr, \ |
132 | char *buf) \ | |
133 | { \ | |
079fa32b | 134 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 135 | return sprintf(buf, "%u\n", gb->field); \ |
355a7058 | 136 | } \ |
8e1d6c33 | 137 | static DEVICE_ATTR_RO(field) |
355a7058 | 138 | |
8e1d6c33 BD |
139 | #define gb_loopback_ro_stats_attr(name, field, type) \ |
140 | static ssize_t name##_##field##_show(struct device *dev, \ | |
355a7058 AB |
141 | struct device_attribute *attr, \ |
142 | char *buf) \ | |
143 | { \ | |
079fa32b | 144 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
1dc8d3d7 AB |
145 | /* Report 0 for min and max if no transfer successed */ \ |
146 | if (!gb->requests_completed) \ | |
147 | return sprintf(buf, "0\n"); \ | |
8e1d6c33 | 148 | return sprintf(buf, "%"#type"\n", gb->name.field); \ |
355a7058 | 149 | } \ |
8e1d6c33 | 150 | static DEVICE_ATTR_RO(name##_##field) |
355a7058 | 151 | |
8e1d6c33 BD |
152 | #define gb_loopback_ro_avg_attr(name) \ |
153 | static ssize_t name##_avg_show(struct device *dev, \ | |
7a135a96 AE |
154 | struct device_attribute *attr, \ |
155 | char *buf) \ | |
156 | { \ | |
f06272b2 | 157 | struct gb_loopback_stats *stats; \ |
f06272b2 | 158 | struct gb_loopback *gb; \ |
fb37f137 AB |
159 | u64 avg, rem; \ |
160 | u32 count; \ | |
079fa32b | 161 | gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 162 | stats = &gb->name; \ |
f06272b2 | 163 | count = stats->count ? stats->count : 1; \ |
58a527af | 164 | avg = stats->sum + count / 2000000; /* round closest */ \ |
f06272b2 | 165 | rem = do_div(avg, count); \ |
89ec14ce AB |
166 | rem *= 1000000; \ |
167 | do_div(rem, count); \ | |
fb37f137 | 168 | return sprintf(buf, "%llu.%06u\n", avg, (u32)rem); \ |
7a135a96 | 169 | } \ |
8e1d6c33 | 170 | static DEVICE_ATTR_RO(name##_avg) |
7a135a96 | 171 | |
8e1d6c33 BD |
172 | #define gb_loopback_stats_attrs(field) \ |
173 | gb_loopback_ro_stats_attr(field, min, u); \ | |
174 | gb_loopback_ro_stats_attr(field, max, u); \ | |
175 | gb_loopback_ro_avg_attr(field) | |
355a7058 AB |
176 | |
177 | #define gb_loopback_attr(field, type) \ | |
178 | static ssize_t field##_show(struct device *dev, \ | |
179 | struct device_attribute *attr, \ | |
180 | char *buf) \ | |
181 | { \ | |
079fa32b | 182 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
355a7058 AB |
183 | return sprintf(buf, "%"#type"\n", gb->field); \ |
184 | } \ | |
185 | static ssize_t field##_store(struct device *dev, \ | |
186 | struct device_attribute *attr, \ | |
187 | const char *buf, \ | |
188 | size_t len) \ | |
189 | { \ | |
190 | int ret; \ | |
079fa32b | 191 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 192 | mutex_lock(&gb->mutex); \ |
355a7058 | 193 | ret = sscanf(buf, "%"#type, &gb->field); \ |
355a7058 | 194 | if (ret != 1) \ |
85d678c0 BD |
195 | len = -EINVAL; \ |
196 | else \ | |
8e1d6c33 BD |
197 | gb_loopback_check_attr(gb, bundle); \ |
198 | mutex_unlock(&gb->mutex); \ | |
355a7058 AB |
199 | return len; \ |
200 | } \ | |
201 | static DEVICE_ATTR_RW(field) | |
202 | ||
f06272b2 BD |
203 | #define gb_dev_loopback_ro_attr(field, conn) \ |
204 | static ssize_t field##_show(struct device *dev, \ | |
67d1eece BD |
205 | struct device_attribute *attr, \ |
206 | char *buf) \ | |
207 | { \ | |
079fa32b | 208 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 209 | return sprintf(buf, "%u\n", gb->field); \ |
67d1eece BD |
210 | } \ |
211 | static DEVICE_ATTR_RO(field) | |
212 | ||
213 | #define gb_dev_loopback_rw_attr(field, type) \ | |
214 | static ssize_t field##_show(struct device *dev, \ | |
215 | struct device_attribute *attr, \ | |
216 | char *buf) \ | |
217 | { \ | |
079fa32b | 218 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 219 | return sprintf(buf, "%"#type"\n", gb->field); \ |
67d1eece BD |
220 | } \ |
221 | static ssize_t field##_store(struct device *dev, \ | |
222 | struct device_attribute *attr, \ | |
223 | const char *buf, \ | |
224 | size_t len) \ | |
225 | { \ | |
226 | int ret; \ | |
079fa32b | 227 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 BD |
228 | mutex_lock(&gb->mutex); \ |
229 | ret = sscanf(buf, "%"#type, &gb->field); \ | |
67d1eece BD |
230 | if (ret != 1) \ |
231 | len = -EINVAL; \ | |
232 | else \ | |
079fa32b | 233 | gb_loopback_check_attr(gb); \ |
8e1d6c33 | 234 | mutex_unlock(&gb->mutex); \ |
67d1eece BD |
235 | return len; \ |
236 | } \ | |
237 | static DEVICE_ATTR_RW(field) | |
238 | ||
8e1d6c33 | 239 | static void gb_loopback_reset_stats(struct gb_loopback *gb); |
079fa32b | 240 | static void gb_loopback_check_attr(struct gb_loopback *gb) |
355a7058 | 241 | { |
b36f04fa BD |
242 | if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX) |
243 | gb->us_wait = GB_LOOPBACK_US_WAIT_MAX; | |
8e1d6c33 BD |
244 | if (gb->size > gb_dev.size_max) |
245 | gb->size = gb_dev.size_max; | |
12927835 BD |
246 | gb->requests_timedout = 0; |
247 | gb->requests_completed = 0; | |
8e1d6c33 | 248 | gb->iteration_count = 0; |
01480ba3 | 249 | gb->send_count = 0; |
8e1d6c33 BD |
250 | gb->error = 0; |
251 | ||
252 | if (kfifo_depth < gb->iteration_max) { | |
079fa32b | 253 | dev_warn(gb->dev, |
8e1d6c33 BD |
254 | "cannot log bytes %u kfifo_depth %u\n", |
255 | gb->iteration_max, kfifo_depth); | |
cb60f496 | 256 | } |
8e1d6c33 | 257 | kfifo_reset_out(&gb->kfifo_lat); |
cb60f496 | 258 | |
8e1d6c33 | 259 | switch (gb->type) { |
a598f438 BD |
260 | case GB_LOOPBACK_TYPE_PING: |
261 | case GB_LOOPBACK_TYPE_TRANSFER: | |
384a7a3c | 262 | case GB_LOOPBACK_TYPE_SINK: |
12927835 BD |
263 | gb->jiffy_timeout = usecs_to_jiffies(gb->timeout); |
264 | if (!gb->jiffy_timeout) | |
265 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN; | |
266 | else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX) | |
267 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX; | |
8e1d6c33 BD |
268 | gb_loopback_reset_stats(gb); |
269 | wake_up(&gb->wq); | |
a598f438 BD |
270 | break; |
271 | default: | |
8e1d6c33 | 272 | gb->type = 0; |
a598f438 BD |
273 | break; |
274 | } | |
355a7058 AB |
275 | } |
276 | ||
277 | /* Time to send and receive one message */ | |
8e1d6c33 | 278 | gb_loopback_stats_attrs(latency); |
583cbf50 | 279 | /* Number of requests sent per second on this cport */ |
8e1d6c33 | 280 | gb_loopback_stats_attrs(requests_per_second); |
355a7058 | 281 | /* Quantity of data sent and received on this cport */ |
8e1d6c33 | 282 | gb_loopback_stats_attrs(throughput); |
1ec5843e | 283 | /* Latency across the UniPro link from APBridge's perspective */ |
8e1d6c33 | 284 | gb_loopback_stats_attrs(apbridge_unipro_latency); |
1ec5843e | 285 | /* Firmware induced overhead in the GPBridge */ |
e54b106d | 286 | gb_loopback_stats_attrs(gbphy_firmware_latency); |
8e1d6c33 | 287 | |
e140c75e | 288 | /* Number of errors encountered during loop */ |
8e1d6c33 | 289 | gb_loopback_ro_attr(error); |
12927835 BD |
290 | /* Number of requests successfully completed async */ |
291 | gb_loopback_ro_attr(requests_completed); | |
292 | /* Number of requests timed out async */ | |
293 | gb_loopback_ro_attr(requests_timedout); | |
294 | /* Timeout minimum in useconds */ | |
295 | gb_loopback_ro_attr(timeout_min); | |
296 | /* Timeout minimum in useconds */ | |
297 | gb_loopback_ro_attr(timeout_max); | |
355a7058 AB |
298 | |
299 | /* | |
799a3f03 | 300 | * Type of loopback message to send based on protocol type definitions |
355a7058 | 301 | * 0 => Don't send message |
799a3f03 | 302 | * 2 => Send ping message continuously (message without payload) |
006335a0 | 303 | * 3 => Send transfer message continuously (message with payload, |
799a3f03 BD |
304 | * payload returned in response) |
305 | * 4 => Send a sink message (message with payload, no payload in response) | |
355a7058 | 306 | */ |
67d1eece | 307 | gb_dev_loopback_rw_attr(type, d); |
355a7058 | 308 | /* Size of transfer message payload: 0-4096 bytes */ |
67d1eece | 309 | gb_dev_loopback_rw_attr(size, u); |
48f19ee8 | 310 | /* Time to wait between two messages: 0-1000 ms */ |
b36f04fa | 311 | gb_dev_loopback_rw_attr(us_wait, d); |
00af6583 | 312 | /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */ |
67d1eece BD |
313 | gb_dev_loopback_rw_attr(iteration_max, u); |
314 | /* The current index of the for (i = 0; i < iteration_max; i++) loop */ | |
f06272b2 | 315 | gb_dev_loopback_ro_attr(iteration_count, false); |
12927835 BD |
316 | /* A flag to indicate synchronous or asynchronous operations */ |
317 | gb_dev_loopback_rw_attr(async, u); | |
318 | /* Timeout of an individual asynchronous request */ | |
319 | gb_dev_loopback_rw_attr(timeout, u); | |
8e3fba55 BD |
320 | /* Maximum number of in-flight operations before back-off */ |
321 | gb_dev_loopback_rw_attr(outstanding_operations_max, u); | |
355a7058 | 322 | |
8e1d6c33 BD |
323 | static struct attribute *loopback_attrs[] = { |
324 | &dev_attr_latency_min.attr, | |
325 | &dev_attr_latency_max.attr, | |
326 | &dev_attr_latency_avg.attr, | |
327 | &dev_attr_requests_per_second_min.attr, | |
328 | &dev_attr_requests_per_second_max.attr, | |
329 | &dev_attr_requests_per_second_avg.attr, | |
330 | &dev_attr_throughput_min.attr, | |
331 | &dev_attr_throughput_max.attr, | |
332 | &dev_attr_throughput_avg.attr, | |
333 | &dev_attr_apbridge_unipro_latency_min.attr, | |
334 | &dev_attr_apbridge_unipro_latency_max.attr, | |
335 | &dev_attr_apbridge_unipro_latency_avg.attr, | |
e54b106d SP |
336 | &dev_attr_gbphy_firmware_latency_min.attr, |
337 | &dev_attr_gbphy_firmware_latency_max.attr, | |
338 | &dev_attr_gbphy_firmware_latency_avg.attr, | |
355a7058 AB |
339 | &dev_attr_type.attr, |
340 | &dev_attr_size.attr, | |
b36f04fa | 341 | &dev_attr_us_wait.attr, |
00af6583 BD |
342 | &dev_attr_iteration_count.attr, |
343 | &dev_attr_iteration_max.attr, | |
12927835 | 344 | &dev_attr_async.attr, |
8e1d6c33 | 345 | &dev_attr_error.attr, |
12927835 BD |
346 | &dev_attr_requests_completed.attr, |
347 | &dev_attr_requests_timedout.attr, | |
348 | &dev_attr_timeout.attr, | |
8e3fba55 | 349 | &dev_attr_outstanding_operations_max.attr, |
12927835 BD |
350 | &dev_attr_timeout_min.attr, |
351 | &dev_attr_timeout_max.attr, | |
355a7058 AB |
352 | NULL, |
353 | }; | |
8e1d6c33 | 354 | ATTRIBUTE_GROUPS(loopback); |
355a7058 | 355 | |
ab81bb9c | 356 | static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error); |
12927835 | 357 | |
bd416103 BD |
358 | static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs) |
359 | { | |
bd416103 | 360 | do_div(elapsed_nsecs, NSEC_PER_USEC); |
5c143123 | 361 | return elapsed_nsecs; |
bd416103 BD |
362 | } |
363 | ||
7c985351 BD |
364 | static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) |
365 | { | |
366 | if (t2 > t1) | |
367 | return t2 - t1; | |
368 | else | |
369 | return NSEC_PER_DAY - t2 + t1; | |
370 | } | |
371 | ||
31408d16 | 372 | static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te) |
4c192665 | 373 | { |
31408d16 | 374 | return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te)); |
4c192665 BD |
375 | } |
376 | ||
fbb8edba BD |
377 | static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, |
378 | void *request, int request_size, | |
379 | void *response, int response_size) | |
384a7a3c | 380 | { |
fbb8edba | 381 | struct gb_operation *operation; |
31408d16 | 382 | ktime_t ts, te; |
fbb8edba | 383 | int ret; |
384a7a3c | 384 | |
31408d16 | 385 | ts = ktime_get(); |
fbb8edba BD |
386 | operation = gb_operation_create(gb->connection, type, request_size, |
387 | response_size, GFP_KERNEL); | |
d9048d8c AB |
388 | if (!operation) |
389 | return -ENOMEM; | |
c3bba87a | 390 | |
fbb8edba BD |
391 | if (request_size) |
392 | memcpy(operation->request->payload, request, request_size); | |
393 | ||
394 | ret = gb_operation_request_send_sync(operation); | |
395 | if (ret) { | |
d9a9ea1b | 396 | dev_err(&gb->connection->bundle->dev, |
fbb8edba | 397 | "synchronous operation failed: %d\n", ret); |
d9048d8c | 398 | goto out_put_operation; |
fbb8edba BD |
399 | } else { |
400 | if (response_size == operation->response->payload_size) { | |
401 | memcpy(response, operation->response->payload, | |
402 | response_size); | |
403 | } else { | |
d9a9ea1b | 404 | dev_err(&gb->connection->bundle->dev, |
fbb8edba BD |
405 | "response size %zu expected %d\n", |
406 | operation->response->payload_size, | |
407 | response_size); | |
d9048d8c AB |
408 | ret = -EINVAL; |
409 | goto out_put_operation; | |
fbb8edba BD |
410 | } |
411 | } | |
6ab1ce4d | 412 | |
31408d16 | 413 | te = ktime_get(); |
2f842304 BD |
414 | |
415 | /* Calculate the total time the message took */ | |
31408d16 | 416 | gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te); |
2f842304 | 417 | |
d9048d8c AB |
418 | out_put_operation: |
419 | gb_operation_put(operation); | |
420 | ||
fbb8edba BD |
421 | return ret; |
422 | } | |
384a7a3c | 423 | |
36f241ff BD |
424 | static void gb_loopback_async_wait_all(struct gb_loopback *gb) |
425 | { | |
426 | wait_event(gb->wq_completion, | |
427 | !atomic_read(&gb->outstanding_operations)); | |
428 | } | |
429 | ||
12927835 BD |
430 | static void gb_loopback_async_operation_callback(struct gb_operation *operation) |
431 | { | |
432 | struct gb_loopback_async_operation *op_async; | |
433 | struct gb_loopback *gb; | |
31408d16 | 434 | ktime_t te; |
262edc35 | 435 | int result; |
12927835 | 436 | |
31408d16 | 437 | te = ktime_get(); |
262edc35 BD |
438 | result = gb_operation_result(operation); |
439 | op_async = gb_operation_get_data(operation); | |
12927835 | 440 | gb = op_async->gb; |
262edc35 | 441 | |
12927835 BD |
442 | mutex_lock(&gb->mutex); |
443 | ||
262edc35 BD |
444 | if (!result && op_async->completion) |
445 | result = op_async->completion(op_async); | |
12927835 | 446 | |
262edc35 | 447 | if (!result) { |
31408d16 | 448 | gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te); |
262edc35 BD |
449 | } else { |
450 | gb->error++; | |
451 | if (result == -ETIMEDOUT) | |
452 | gb->requests_timedout++; | |
12927835 | 453 | } |
12927835 | 454 | |
262edc35 BD |
455 | gb->iteration_count++; |
456 | gb_loopback_calculate_stats(gb, result); | |
12927835 | 457 | |
12927835 BD |
458 | mutex_unlock(&gb->mutex); |
459 | ||
262edc35 | 460 | dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", |
12927835 BD |
461 | operation->id); |
462 | ||
262edc35 BD |
463 | /* Wake up waiters */ |
464 | atomic_dec(&op_async->gb->outstanding_operations); | |
465 | wake_up(&gb->wq_completion); | |
12927835 | 466 | |
262edc35 BD |
467 | /* Release resources */ |
468 | gb_operation_put(operation); | |
469 | kfree(op_async); | |
12927835 BD |
470 | } |
471 | ||
472 | static int gb_loopback_async_operation(struct gb_loopback *gb, int type, | |
473 | void *request, int request_size, | |
474 | int response_size, | |
475 | void *completion) | |
476 | { | |
477 | struct gb_loopback_async_operation *op_async; | |
478 | struct gb_operation *operation; | |
479 | int ret; | |
12927835 BD |
480 | |
481 | op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); | |
482 | if (!op_async) | |
483 | return -ENOMEM; | |
484 | ||
12927835 BD |
485 | operation = gb_operation_create(gb->connection, type, request_size, |
486 | response_size, GFP_KERNEL); | |
487 | if (!operation) { | |
c7aae4e6 BD |
488 | kfree(op_async); |
489 | return -ENOMEM; | |
12927835 BD |
490 | } |
491 | ||
492 | if (request_size) | |
493 | memcpy(operation->request->payload, request, request_size); | |
494 | ||
262edc35 BD |
495 | gb_operation_set_data(operation, op_async); |
496 | ||
12927835 BD |
497 | op_async->gb = gb; |
498 | op_async->operation = operation; | |
499 | op_async->completion = completion; | |
500 | ||
31408d16 | 501 | op_async->ts = ktime_get(); |
262edc35 | 502 | |
36f241ff | 503 | atomic_inc(&gb->outstanding_operations); |
12927835 BD |
504 | ret = gb_operation_request_send(operation, |
505 | gb_loopback_async_operation_callback, | |
262edc35 | 506 | jiffies_to_msecs(gb->jiffy_timeout), |
12927835 | 507 | GFP_KERNEL); |
262edc35 BD |
508 | if (ret) { |
509 | atomic_dec(&gb->outstanding_operations); | |
510 | gb_operation_put(operation); | |
511 | kfree(op_async); | |
512 | } | |
12927835 BD |
513 | return ret; |
514 | } | |
515 | ||
516 | static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len) | |
fbb8edba BD |
517 | { |
518 | struct gb_loopback_transfer_request *request; | |
519 | int retval; | |
520 | ||
521 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
522 | if (!request) | |
523 | return -ENOMEM; | |
524 | ||
525 | request->len = cpu_to_le32(len); | |
526 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK, | |
527 | request, len + sizeof(*request), | |
528 | NULL, 0); | |
384a7a3c BD |
529 | kfree(request); |
530 | return retval; | |
531 | } | |
532 | ||
12927835 | 533 | static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len) |
355a7058 | 534 | { |
355a7058 AB |
535 | struct gb_loopback_transfer_request *request; |
536 | struct gb_loopback_transfer_response *response; | |
537 | int retval; | |
538 | ||
d6a1a3b5 | 539 | gb->apbridge_latency_ts = 0; |
e54b106d | 540 | gb->gbphy_latency_ts = 0; |
d6a1a3b5 | 541 | |
355a7058 AB |
542 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); |
543 | if (!request) | |
544 | return -ENOMEM; | |
545 | response = kmalloc(len + sizeof(*response), GFP_KERNEL); | |
546 | if (!response) { | |
547 | kfree(request); | |
548 | return -ENOMEM; | |
549 | } | |
550 | ||
dc4a1069 VK |
551 | memset(request->data, 0x5A, len); |
552 | ||
355a7058 | 553 | request->len = cpu_to_le32(len); |
fbb8edba BD |
554 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER, |
555 | request, len + sizeof(*request), | |
556 | response, len + sizeof(*response)); | |
355a7058 AB |
557 | if (retval) |
558 | goto gb_error; | |
559 | ||
dc366f8e | 560 | if (memcmp(request->data, response->data, len)) { |
d9a9ea1b GKH |
561 | dev_err(&gb->connection->bundle->dev, |
562 | "Loopback Data doesn't match\n"); | |
355a7058 | 563 | retval = -EREMOTEIO; |
dc366f8e | 564 | } |
1ec5843e | 565 | gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0); |
e54b106d | 566 | gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1); |
355a7058 AB |
567 | |
568 | gb_error: | |
569 | kfree(request); | |
570 | kfree(response); | |
571 | ||
572 | return retval; | |
573 | } | |
574 | ||
12927835 | 575 | static int gb_loopback_sync_ping(struct gb_loopback *gb) |
355a7058 | 576 | { |
fbb8edba BD |
577 | return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING, |
578 | NULL, 0, NULL, 0); | |
355a7058 AB |
579 | } |
580 | ||
12927835 BD |
581 | static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len) |
582 | { | |
583 | struct gb_loopback_transfer_request *request; | |
584 | int retval; | |
585 | ||
586 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
587 | if (!request) | |
588 | return -ENOMEM; | |
589 | ||
590 | request->len = cpu_to_le32(len); | |
591 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK, | |
592 | request, len + sizeof(*request), | |
593 | 0, NULL); | |
594 | kfree(request); | |
595 | return retval; | |
596 | } | |
597 | ||
598 | static int gb_loopback_async_transfer_complete( | |
599 | struct gb_loopback_async_operation *op_async) | |
600 | { | |
601 | struct gb_loopback *gb; | |
602 | struct gb_operation *operation; | |
603 | struct gb_loopback_transfer_request *request; | |
604 | struct gb_loopback_transfer_response *response; | |
605 | size_t len; | |
606 | int retval = 0; | |
607 | ||
608 | gb = op_async->gb; | |
609 | operation = op_async->operation; | |
610 | request = operation->request->payload; | |
611 | response = operation->response->payload; | |
612 | len = le32_to_cpu(request->len); | |
613 | ||
614 | if (memcmp(request->data, response->data, len)) { | |
615 | dev_err(&gb->connection->bundle->dev, | |
616 | "Loopback Data doesn't match operation id %d\n", | |
617 | operation->id); | |
618 | retval = -EREMOTEIO; | |
619 | } else { | |
620 | gb->apbridge_latency_ts = | |
621 | (u32)__le32_to_cpu(response->reserved0); | |
e54b106d | 622 | gb->gbphy_latency_ts = |
12927835 BD |
623 | (u32)__le32_to_cpu(response->reserved1); |
624 | } | |
625 | ||
626 | return retval; | |
627 | } | |
628 | ||
629 | static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len) | |
630 | { | |
631 | struct gb_loopback_transfer_request *request; | |
632 | int retval, response_len; | |
633 | ||
634 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
635 | if (!request) | |
636 | return -ENOMEM; | |
637 | ||
638 | memset(request->data, 0x5A, len); | |
639 | ||
640 | request->len = cpu_to_le32(len); | |
641 | response_len = sizeof(struct gb_loopback_transfer_response); | |
642 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER, | |
643 | request, len + sizeof(*request), | |
644 | len + response_len, | |
645 | gb_loopback_async_transfer_complete); | |
646 | if (retval) | |
647 | goto gb_error; | |
648 | ||
649 | gb_error: | |
650 | kfree(request); | |
651 | return retval; | |
652 | } | |
653 | ||
654 | static int gb_loopback_async_ping(struct gb_loopback *gb) | |
655 | { | |
656 | return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING, | |
657 | NULL, 0, 0, NULL); | |
658 | } | |
659 | ||
e82a11dc | 660 | static int gb_loopback_request_handler(struct gb_operation *operation) |
ac1c2840 AE |
661 | { |
662 | struct gb_connection *connection = operation->connection; | |
663 | struct gb_loopback_transfer_request *request; | |
664 | struct gb_loopback_transfer_response *response; | |
d9a9ea1b | 665 | struct device *dev = &connection->bundle->dev; |
e51eafeb | 666 | size_t len; |
ac1c2840 AE |
667 | |
668 | /* By convention, the AP initiates the version operation */ | |
e82a11dc | 669 | switch (operation->type) { |
ac1c2840 | 670 | case GB_LOOPBACK_TYPE_PING: |
384a7a3c | 671 | case GB_LOOPBACK_TYPE_SINK: |
ac1c2840 AE |
672 | return 0; |
673 | case GB_LOOPBACK_TYPE_TRANSFER: | |
674 | if (operation->request->payload_size < sizeof(*request)) { | |
d9a9ea1b | 675 | dev_err(dev, "transfer request too small (%zu < %zu)\n", |
ac1c2840 AE |
676 | operation->request->payload_size, |
677 | sizeof(*request)); | |
678 | return -EINVAL; /* -EMSGSIZE */ | |
679 | } | |
680 | request = operation->request->payload; | |
681 | len = le32_to_cpu(request->len); | |
67d1eece | 682 | if (len > gb_dev.size_max) { |
d9a9ea1b | 683 | dev_err(dev, "transfer request too large (%zu > %zu)\n", |
67d1eece | 684 | len, gb_dev.size_max); |
c3bba87a BD |
685 | return -EINVAL; |
686 | } | |
687 | ||
81ad6994 BG |
688 | if (!gb_operation_response_alloc(operation, |
689 | len + sizeof(*response), GFP_KERNEL)) { | |
690 | dev_err(dev, "error allocating response\n"); | |
691 | return -ENOMEM; | |
ac1c2840 | 692 | } |
81ad6994 BG |
693 | response = operation->response->payload; |
694 | response->len = cpu_to_le32(len); | |
695 | if (len) | |
696 | memcpy(response->data, request->data, len); | |
697 | ||
ac1c2840 AE |
698 | return 0; |
699 | default: | |
e82a11dc | 700 | dev_err(dev, "unsupported request: %u\n", operation->type); |
ac1c2840 AE |
701 | return -EINVAL; |
702 | } | |
703 | } | |
704 | ||
8e1d6c33 | 705 | static void gb_loopback_reset_stats(struct gb_loopback *gb) |
355a7058 AB |
706 | { |
707 | struct gb_loopback_stats reset = { | |
e051c82b | 708 | .min = U32_MAX, |
355a7058 | 709 | }; |
67d1eece BD |
710 | |
711 | /* Reset per-connection stats */ | |
8e1d6c33 BD |
712 | memcpy(&gb->latency, &reset, |
713 | sizeof(struct gb_loopback_stats)); | |
714 | memcpy(&gb->throughput, &reset, | |
715 | sizeof(struct gb_loopback_stats)); | |
716 | memcpy(&gb->requests_per_second, &reset, | |
717 | sizeof(struct gb_loopback_stats)); | |
718 | memcpy(&gb->apbridge_unipro_latency, &reset, | |
719 | sizeof(struct gb_loopback_stats)); | |
e54b106d | 720 | memcpy(&gb->gbphy_firmware_latency, &reset, |
8e1d6c33 | 721 | sizeof(struct gb_loopback_stats)); |
67d1eece | 722 | |
f42a6891 BD |
723 | /* Should be initialized at least once per transaction set */ |
724 | gb->apbridge_latency_ts = 0; | |
e54b106d | 725 | gb->gbphy_latency_ts = 0; |
31408d16 | 726 | gb->ts = ktime_set(0, 0); |
355a7058 AB |
727 | } |
728 | ||
a6e7e535 | 729 | static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) |
355a7058 | 730 | { |
00af6583 BD |
731 | if (stats->min > val) |
732 | stats->min = val; | |
733 | if (stats->max < val) | |
734 | stats->max = val; | |
735 | stats->sum += val; | |
736 | stats->count++; | |
355a7058 AB |
737 | } |
738 | ||
ab81bb9c AB |
739 | static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats, |
740 | u64 val, u32 count) | |
741 | { | |
742 | stats->sum += val; | |
743 | stats->count += count; | |
744 | ||
745 | do_div(val, count); | |
746 | if (stats->min > val) | |
747 | stats->min = val; | |
748 | if (stats->max < val) | |
749 | stats->max = val; | |
750 | } | |
751 | ||
583cbf50 | 752 | static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 753 | { |
ab81bb9c | 754 | u64 req = gb->requests_completed * USEC_PER_SEC; |
00af6583 | 755 | |
ab81bb9c | 756 | gb_loopback_update_stats_window(&gb->requests_per_second, req, latency); |
355a7058 AB |
757 | } |
758 | ||
00af6583 | 759 | static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 760 | { |
ab81bb9c | 761 | u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2; |
f7908e4d | 762 | |
8e1d6c33 | 763 | switch (gb->type) { |
f7908e4d BD |
764 | case GB_LOOPBACK_TYPE_PING: |
765 | break; | |
766 | case GB_LOOPBACK_TYPE_SINK: | |
767 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
8e1d6c33 | 768 | gb->size; |
f7908e4d BD |
769 | break; |
770 | case GB_LOOPBACK_TYPE_TRANSFER: | |
771 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
772 | sizeof(struct gb_loopback_transfer_response) + | |
8e1d6c33 | 773 | gb->size * 2; |
f7908e4d BD |
774 | break; |
775 | default: | |
776 | return; | |
777 | } | |
00af6583 | 778 | |
ab81bb9c AB |
779 | aggregate_size *= gb->requests_completed; |
780 | aggregate_size *= USEC_PER_SEC; | |
781 | gb_loopback_update_stats_window(&gb->throughput, aggregate_size, | |
782 | latency); | |
355a7058 AB |
783 | } |
784 | ||
ab81bb9c | 785 | static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb) |
355a7058 AB |
786 | { |
787 | u32 lat; | |
355a7058 | 788 | |
00af6583 | 789 | /* Express latency in terms of microseconds */ |
bd416103 | 790 | lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs); |
355a7058 | 791 | |
98676ca8 | 792 | /* Log latency stastic */ |
00af6583 | 793 | gb_loopback_update_stats(&gb->latency, lat); |
67d1eece BD |
794 | |
795 | /* Raw latency log on a per thread basis */ | |
4b0ea00c | 796 | kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat)); |
00af6583 | 797 | |
1ec5843e | 798 | /* Log the firmware supplied latency values */ |
1ec5843e BD |
799 | gb_loopback_update_stats(&gb->apbridge_unipro_latency, |
800 | gb->apbridge_latency_ts); | |
e54b106d SP |
801 | gb_loopback_update_stats(&gb->gbphy_firmware_latency, |
802 | gb->gbphy_latency_ts); | |
355a7058 AB |
803 | } |
804 | ||
ab81bb9c AB |
805 | static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error) |
806 | { | |
807 | u64 nlat; | |
808 | u32 lat; | |
31408d16 | 809 | ktime_t te; |
ab81bb9c AB |
810 | |
811 | if (!error) { | |
812 | gb->requests_completed++; | |
813 | gb_loopback_calculate_latency_stats(gb); | |
814 | } | |
815 | ||
31408d16 AB |
816 | te = ktime_get(); |
817 | nlat = gb_loopback_calc_latency(gb->ts, te); | |
ab81bb9c AB |
818 | if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) { |
819 | lat = gb_loopback_nsec_to_usec_latency(nlat); | |
820 | ||
821 | gb_loopback_throughput_update(gb, lat); | |
822 | gb_loopback_requests_update(gb, lat); | |
823 | ||
824 | if (gb->iteration_count != gb->iteration_max) { | |
825 | gb->ts = te; | |
826 | gb->requests_completed = 0; | |
827 | } | |
828 | } | |
829 | } | |
830 | ||
8e3fba55 BD |
831 | static void gb_loopback_async_wait_to_send(struct gb_loopback *gb) |
832 | { | |
833 | if (!(gb->async && gb->outstanding_operations_max)) | |
834 | return; | |
835 | wait_event_interruptible(gb->wq_completion, | |
836 | (atomic_read(&gb->outstanding_operations) < | |
837 | gb->outstanding_operations_max) || | |
838 | kthread_should_stop()); | |
839 | } | |
840 | ||
355a7058 AB |
841 | static int gb_loopback_fn(void *data) |
842 | { | |
843 | int error = 0; | |
b36f04fa | 844 | int us_wait = 0; |
67d1eece | 845 | int type; |
e854ff58 | 846 | int ret; |
67d1eece | 847 | u32 size; |
01480ba3 | 848 | |
3f12c3ed | 849 | struct gb_loopback *gb = data; |
e854ff58 AH |
850 | struct gb_bundle *bundle = gb->connection->bundle; |
851 | ||
852 | ret = gb_pm_runtime_get_sync(bundle); | |
853 | if (ret) | |
854 | return ret; | |
355a7058 | 855 | |
3dfe8aaa | 856 | while (1) { |
e854ff58 AH |
857 | if (!gb->type) { |
858 | gb_pm_runtime_put_autosuspend(bundle); | |
8e1d6c33 | 859 | wait_event_interruptible(gb->wq, gb->type || |
3dfe8aaa | 860 | kthread_should_stop()); |
e854ff58 AH |
861 | ret = gb_pm_runtime_get_sync(bundle); |
862 | if (ret) | |
863 | return ret; | |
864 | } | |
865 | ||
8e3fba55 BD |
866 | if (kthread_should_stop()) |
867 | break; | |
12927835 | 868 | |
8e3fba55 BD |
869 | /* Limit the maximum number of in-flight async operations */ |
870 | gb_loopback_async_wait_to_send(gb); | |
3dfe8aaa BD |
871 | if (kthread_should_stop()) |
872 | break; | |
85d678c0 | 873 | |
67d1eece | 874 | mutex_lock(&gb->mutex); |
8e1d6c33 BD |
875 | |
876 | /* Optionally terminate */ | |
01480ba3 | 877 | if (gb->send_count == gb->iteration_max) { |
a072a72d BD |
878 | mutex_unlock(&gb->mutex); |
879 | ||
880 | /* Wait for synchronous and asynchronus completion */ | |
881 | gb_loopback_async_wait_all(gb); | |
882 | ||
883 | /* Mark complete unless user-space has poked us */ | |
884 | mutex_lock(&gb->mutex); | |
ab81bb9c AB |
885 | if (gb->iteration_count == gb->iteration_max) { |
886 | gb->type = 0; | |
01480ba3 | 887 | gb->send_count = 0; |
39c2787b AH |
888 | sysfs_notify(&gb->dev->kobj, NULL, |
889 | "iteration_count"); | |
a072a72d BD |
890 | dev_dbg(&bundle->dev, "load test complete\n"); |
891 | } else { | |
892 | dev_dbg(&bundle->dev, | |
893 | "continuing on with new test set\n"); | |
ab81bb9c | 894 | } |
67d1eece | 895 | mutex_unlock(&gb->mutex); |
8e1d6c33 | 896 | continue; |
67d1eece | 897 | } |
8e1d6c33 | 898 | size = gb->size; |
b36f04fa | 899 | us_wait = gb->us_wait; |
8e1d6c33 | 900 | type = gb->type; |
31408d16 AB |
901 | if (ktime_to_ns(gb->ts) == 0) |
902 | gb->ts = ktime_get(); | |
d9fb3754 | 903 | |
67d1eece | 904 | /* Else operations to perform */ |
12927835 | 905 | if (gb->async) { |
a1f10771 | 906 | if (type == GB_LOOPBACK_TYPE_PING) |
12927835 | 907 | error = gb_loopback_async_ping(gb); |
a1f10771 | 908 | else if (type == GB_LOOPBACK_TYPE_TRANSFER) |
12927835 | 909 | error = gb_loopback_async_transfer(gb, size); |
a1f10771 | 910 | else if (type == GB_LOOPBACK_TYPE_SINK) |
12927835 | 911 | error = gb_loopback_async_sink(gb, size); |
12927835 | 912 | |
44b02da3 | 913 | if (error) { |
12927835 | 914 | gb->error++; |
44b02da3 BD |
915 | gb->iteration_count++; |
916 | } | |
12927835 BD |
917 | } else { |
918 | /* We are effectively single threaded here */ | |
919 | if (type == GB_LOOPBACK_TYPE_PING) | |
920 | error = gb_loopback_sync_ping(gb); | |
921 | else if (type == GB_LOOPBACK_TYPE_TRANSFER) | |
922 | error = gb_loopback_sync_transfer(gb, size); | |
923 | else if (type == GB_LOOPBACK_TYPE_SINK) | |
924 | error = gb_loopback_sync_sink(gb, size); | |
925 | ||
926 | if (error) | |
927 | gb->error++; | |
928 | gb->iteration_count++; | |
ab81bb9c | 929 | gb_loopback_calculate_stats(gb, !!error); |
12927835 | 930 | } |
01480ba3 | 931 | gb->send_count++; |
5a70524b | 932 | mutex_unlock(&gb->mutex); |
33b8807a JH |
933 | |
934 | if (us_wait) { | |
935 | if (us_wait < 20000) | |
936 | usleep_range(us_wait, us_wait + 100); | |
937 | else | |
938 | msleep(us_wait / 1000); | |
939 | } | |
355a7058 | 940 | } |
e854ff58 AH |
941 | |
942 | gb_pm_runtime_put_autosuspend(bundle); | |
943 | ||
355a7058 AB |
944 | return 0; |
945 | } | |
946 | ||
4b0ea00c BD |
947 | static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s, |
948 | struct kfifo *kfifo, | |
949 | struct mutex *mutex) | |
aa27bf82 | 950 | { |
aa27bf82 BD |
951 | u32 latency; |
952 | int retval; | |
953 | ||
4b0ea00c | 954 | if (kfifo_len(kfifo) == 0) { |
aa27bf82 BD |
955 | retval = -EAGAIN; |
956 | goto done; | |
957 | } | |
958 | ||
4b0ea00c BD |
959 | mutex_lock(mutex); |
960 | retval = kfifo_out(kfifo, &latency, sizeof(latency)); | |
aa27bf82 BD |
961 | if (retval > 0) { |
962 | seq_printf(s, "%u", latency); | |
963 | retval = 0; | |
964 | } | |
4b0ea00c | 965 | mutex_unlock(mutex); |
aa27bf82 BD |
966 | done: |
967 | return retval; | |
968 | } | |
969 | ||
4b0ea00c BD |
970 | static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused) |
971 | { | |
972 | struct gb_loopback *gb = s->private; | |
973 | ||
974 | return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat, | |
975 | &gb->mutex); | |
976 | } | |
977 | ||
aa27bf82 BD |
978 | static int gb_loopback_latency_open(struct inode *inode, struct file *file) |
979 | { | |
980 | return single_open(file, gb_loopback_dbgfs_latency_show, | |
981 | inode->i_private); | |
982 | } | |
983 | ||
984 | static const struct file_operations gb_loopback_debugfs_latency_ops = { | |
985 | .open = gb_loopback_latency_open, | |
986 | .read = seq_read, | |
987 | .llseek = seq_lseek, | |
988 | .release = single_release, | |
989 | }; | |
990 | ||
5015115d BD |
991 | static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha, |
992 | struct list_head *lhb) | |
993 | { | |
994 | struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry); | |
995 | struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry); | |
996 | struct gb_connection *ca = a->connection; | |
997 | struct gb_connection *cb = b->connection; | |
998 | ||
5015115d BD |
999 | if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id) |
1000 | return -1; | |
1001 | if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id) | |
1002 | return 1; | |
1003 | if (ca->bundle->id < cb->bundle->id) | |
1004 | return -1; | |
1005 | if (cb->bundle->id < ca->bundle->id) | |
1006 | return 1; | |
1007 | if (ca->intf_cport_id < cb->intf_cport_id) | |
1008 | return -1; | |
1009 | else if (cb->intf_cport_id < ca->intf_cport_id) | |
1010 | return 1; | |
1011 | ||
1012 | return 0; | |
1013 | } | |
1014 | ||
1015 | static void gb_loopback_insert_id(struct gb_loopback *gb) | |
1016 | { | |
1017 | struct gb_loopback *gb_list; | |
1018 | u32 new_lbid = 0; | |
1019 | ||
1020 | /* perform an insertion sort */ | |
1021 | list_add_tail(&gb->entry, &gb_dev.list); | |
1022 | list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare); | |
1023 | list_for_each_entry(gb_list, &gb_dev.list, entry) { | |
1024 | gb_list->lbid = 1 << new_lbid; | |
1025 | new_lbid++; | |
1026 | } | |
1027 | } | |
1028 | ||
67d1eece | 1029 | #define DEBUGFS_NAMELEN 32 |
aa27bf82 | 1030 | |
e82a11dc VK |
1031 | static int gb_loopback_probe(struct gb_bundle *bundle, |
1032 | const struct greybus_bundle_id *id) | |
355a7058 | 1033 | { |
e82a11dc VK |
1034 | struct greybus_descriptor_cport *cport_desc; |
1035 | struct gb_connection *connection; | |
355a7058 | 1036 | struct gb_loopback *gb; |
079fa32b | 1037 | struct device *dev; |
355a7058 | 1038 | int retval; |
aa27bf82 | 1039 | char name[DEBUGFS_NAMELEN]; |
2e238d71 | 1040 | unsigned long flags; |
355a7058 | 1041 | |
e82a11dc VK |
1042 | if (bundle->num_cports != 1) |
1043 | return -ENODEV; | |
1044 | ||
1045 | cport_desc = &bundle->cport_desc[0]; | |
1046 | if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK) | |
1047 | return -ENODEV; | |
1048 | ||
355a7058 AB |
1049 | gb = kzalloc(sizeof(*gb), GFP_KERNEL); |
1050 | if (!gb) | |
1051 | return -ENOMEM; | |
1052 | ||
e82a11dc VK |
1053 | connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), |
1054 | gb_loopback_request_handler); | |
1055 | if (IS_ERR(connection)) { | |
1056 | retval = PTR_ERR(connection); | |
1057 | goto out_kzalloc; | |
1058 | } | |
1059 | ||
1060 | gb->connection = connection; | |
1061 | greybus_set_drvdata(bundle, gb); | |
1062 | ||
8e1d6c33 | 1063 | init_waitqueue_head(&gb->wq); |
36f241ff BD |
1064 | init_waitqueue_head(&gb->wq_completion); |
1065 | atomic_set(&gb->outstanding_operations, 0); | |
8e1d6c33 BD |
1066 | gb_loopback_reset_stats(gb); |
1067 | ||
12927835 BD |
1068 | /* Reported values to user-space for min/max timeouts */ |
1069 | gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN); | |
1070 | gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX); | |
1071 | ||
f06272b2 | 1072 | if (!gb_dev.count) { |
f06272b2 BD |
1073 | /* Calculate maximum payload */ |
1074 | gb_dev.size_max = gb_operation_get_payload_size_max(connection); | |
1075 | if (gb_dev.size_max <= | |
1076 | sizeof(struct gb_loopback_transfer_request)) { | |
1077 | retval = -EINVAL; | |
e82a11dc | 1078 | goto out_connection_destroy; |
f06272b2 BD |
1079 | } |
1080 | gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request); | |
1081 | } | |
f06272b2 BD |
1082 | |
1083 | /* Create per-connection sysfs and debugfs data-points */ | |
8d8d36da | 1084 | snprintf(name, sizeof(name), "raw_latency_%s", |
d9a9ea1b | 1085 | dev_name(&connection->bundle->dev)); |
563a8412 | 1086 | gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb, |
aa27bf82 | 1087 | &gb_loopback_debugfs_latency_ops); |
079fa32b AH |
1088 | |
1089 | gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL); | |
1090 | if (gb->id < 0) { | |
1091 | retval = gb->id; | |
e82a11dc | 1092 | goto out_debugfs_remove; |
079fa32b AH |
1093 | } |
1094 | ||
e82a11dc VK |
1095 | retval = gb_connection_enable(connection); |
1096 | if (retval) | |
1097 | goto out_ida_remove; | |
1098 | ||
079fa32b AH |
1099 | dev = device_create_with_groups(&loopback_class, |
1100 | &connection->bundle->dev, | |
1101 | MKDEV(0, 0), gb, loopback_groups, | |
1102 | "gb_loopback%d", gb->id); | |
1103 | if (IS_ERR(dev)) { | |
1104 | retval = PTR_ERR(dev); | |
e82a11dc | 1105 | goto out_connection_disable; |
079fa32b AH |
1106 | } |
1107 | gb->dev = dev; | |
c3bba87a | 1108 | |
cbd204b4 | 1109 | /* Allocate kfifo */ |
4b0ea00c | 1110 | if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32), |
cbd204b4 BD |
1111 | GFP_KERNEL)) { |
1112 | retval = -ENOMEM; | |
079fa32b | 1113 | goto out_conn; |
cbd204b4 | 1114 | } |
cbd204b4 | 1115 | /* Fork worker thread */ |
85d678c0 | 1116 | mutex_init(&gb->mutex); |
355a7058 AB |
1117 | gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); |
1118 | if (IS_ERR(gb->task)) { | |
69f60347 | 1119 | retval = PTR_ERR(gb->task); |
8563a49c | 1120 | goto out_kfifo; |
355a7058 AB |
1121 | } |
1122 | ||
2e238d71 | 1123 | spin_lock_irqsave(&gb_dev.lock, flags); |
5015115d | 1124 | gb_loopback_insert_id(gb); |
aa27bf82 | 1125 | gb_dev.count++; |
2e238d71 BD |
1126 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1127 | ||
1128 | gb_connection_latency_tag_enable(connection); | |
e854ff58 AH |
1129 | |
1130 | gb_pm_runtime_put_autosuspend(bundle); | |
1131 | ||
355a7058 AB |
1132 | return 0; |
1133 | ||
8563a49c | 1134 | out_kfifo: |
4b0ea00c | 1135 | kfifo_free(&gb->kfifo_lat); |
079fa32b AH |
1136 | out_conn: |
1137 | device_unregister(dev); | |
e82a11dc VK |
1138 | out_connection_disable: |
1139 | gb_connection_disable(connection); | |
1140 | out_ida_remove: | |
079fa32b | 1141 | ida_simple_remove(&loopback_ida, gb->id); |
e82a11dc | 1142 | out_debugfs_remove: |
aa27bf82 | 1143 | debugfs_remove(gb->file); |
e82a11dc VK |
1144 | out_connection_destroy: |
1145 | gb_connection_destroy(connection); | |
2e238d71 | 1146 | out_kzalloc: |
355a7058 | 1147 | kfree(gb); |
1fb807cf | 1148 | |
355a7058 AB |
1149 | return retval; |
1150 | } | |
1151 | ||
e82a11dc | 1152 | static void gb_loopback_disconnect(struct gb_bundle *bundle) |
355a7058 | 1153 | { |
e82a11dc | 1154 | struct gb_loopback *gb = greybus_get_drvdata(bundle); |
2e238d71 | 1155 | unsigned long flags; |
e854ff58 AH |
1156 | int ret; |
1157 | ||
1158 | ret = gb_pm_runtime_get_sync(bundle); | |
1159 | if (ret) | |
1160 | gb_pm_runtime_get_noresume(bundle); | |
355a7058 | 1161 | |
e82a11dc VK |
1162 | gb_connection_disable(gb->connection); |
1163 | ||
355a7058 AB |
1164 | if (!IS_ERR_OR_NULL(gb->task)) |
1165 | kthread_stop(gb->task); | |
cbd204b4 | 1166 | |
4b0ea00c | 1167 | kfifo_free(&gb->kfifo_lat); |
e82a11dc | 1168 | gb_connection_latency_tag_disable(gb->connection); |
aa27bf82 | 1169 | debugfs_remove(gb->file); |
e82a11dc VK |
1170 | |
1171 | /* | |
1172 | * FIXME: gb_loopback_async_wait_all() is redundant now, as connection | |
1173 | * is disabled at the beginning and so we can't have any more | |
1174 | * incoming/outgoing requests. | |
1175 | */ | |
36f241ff | 1176 | gb_loopback_async_wait_all(gb); |
2e238d71 BD |
1177 | |
1178 | spin_lock_irqsave(&gb_dev.lock, flags); | |
1179 | gb_dev.count--; | |
ff477d07 | 1180 | list_del(&gb->entry); |
2e238d71 BD |
1181 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1182 | ||
079fa32b AH |
1183 | device_unregister(gb->dev); |
1184 | ida_simple_remove(&loopback_ida, gb->id); | |
1185 | ||
e82a11dc | 1186 | gb_connection_destroy(gb->connection); |
5f3e0d17 | 1187 | kfree(gb); |
355a7058 AB |
1188 | } |
1189 | ||
e82a11dc VK |
1190 | static const struct greybus_bundle_id gb_loopback_id_table[] = { |
1191 | { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) }, | |
1192 | { } | |
1193 | }; | |
1194 | MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table); | |
1195 | ||
1196 | static struct greybus_driver gb_loopback_driver = { | |
1197 | .name = "loopback", | |
1198 | .probe = gb_loopback_probe, | |
1199 | .disconnect = gb_loopback_disconnect, | |
1200 | .id_table = gb_loopback_id_table, | |
355a7058 AB |
1201 | }; |
1202 | ||
cbd204b4 BD |
1203 | static int loopback_init(void) |
1204 | { | |
4b0ea00c BD |
1205 | int retval; |
1206 | ||
67d1eece | 1207 | INIT_LIST_HEAD(&gb_dev.list); |
12927835 | 1208 | INIT_LIST_HEAD(&gb_dev.list_op_async); |
2e238d71 | 1209 | spin_lock_init(&gb_dev.lock); |
aa27bf82 | 1210 | gb_dev.root = debugfs_create_dir("gb_loopback", NULL); |
67d1eece | 1211 | |
079fa32b AH |
1212 | retval = class_register(&loopback_class); |
1213 | if (retval) | |
1214 | goto err; | |
1215 | ||
e82a11dc | 1216 | retval = greybus_register(&gb_loopback_driver); |
079fa32b AH |
1217 | if (retval) |
1218 | goto err_unregister; | |
1219 | ||
1220 | return 0; | |
4b0ea00c | 1221 | |
079fa32b AH |
1222 | err_unregister: |
1223 | class_unregister(&loopback_class); | |
1224 | err: | |
4b0ea00c BD |
1225 | debugfs_remove_recursive(gb_dev.root); |
1226 | return retval; | |
cbd204b4 BD |
1227 | } |
1228 | module_init(loopback_init); | |
1229 | ||
1230 | static void __exit loopback_exit(void) | |
1231 | { | |
aa27bf82 | 1232 | debugfs_remove_recursive(gb_dev.root); |
e82a11dc | 1233 | greybus_deregister(&gb_loopback_driver); |
079fa32b AH |
1234 | class_unregister(&loopback_class); |
1235 | ida_destroy(&loopback_ida); | |
cbd204b4 BD |
1236 | } |
1237 | module_exit(loopback_exit); | |
355a7058 AB |
1238 | |
1239 | MODULE_LICENSE("GPL v2"); |