Commit | Line | Data |
---|---|---|
355a7058 AB |
1 | /* |
2 | * Loopback bridge driver for the Greybus loopback module. | |
3 | * | |
4 | * Copyright 2014 Google Inc. | |
5 | * Copyright 2014 Linaro Ltd. | |
6 | * | |
7 | * Released under the GPLv2 only. | |
8 | */ | |
8923c5b5 JH |
9 | |
10 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
11 | ||
355a7058 AB |
12 | #include <linux/kernel.h> |
13 | #include <linux/module.h> | |
85d678c0 | 14 | #include <linux/mutex.h> |
355a7058 AB |
15 | #include <linux/slab.h> |
16 | #include <linux/kthread.h> | |
17 | #include <linux/delay.h> | |
18 | #include <linux/random.h> | |
5679f783 | 19 | #include <linux/sizes.h> |
cbd204b4 BD |
20 | #include <linux/cdev.h> |
21 | #include <linux/fs.h> | |
22 | #include <linux/kfifo.h> | |
aa27bf82 | 23 | #include <linux/debugfs.h> |
5015115d | 24 | #include <linux/list_sort.h> |
2e238d71 | 25 | #include <linux/spinlock.h> |
12927835 | 26 | #include <linux/workqueue.h> |
36f241ff | 27 | #include <linux/atomic.h> |
e854ff58 | 28 | #include <linux/pm_runtime.h> |
cbd204b4 | 29 | |
1ffc12be JH |
30 | #include <asm/div64.h> |
31 | ||
355a7058 | 32 | #include "greybus.h" |
fd58926e | 33 | #include "connection.h" |
355a7058 | 34 | |
fd489e1a BD |
35 | #define NSEC_PER_DAY 86400000000000ULL |
36 | ||
355a7058 | 37 | struct gb_loopback_stats { |
e051c82b AE |
38 | u32 min; |
39 | u32 max; | |
00af6583 | 40 | u64 sum; |
e13fc28f | 41 | u32 count; |
355a7058 AB |
42 | }; |
43 | ||
aa27bf82 BD |
44 | struct gb_loopback_device { |
45 | struct dentry *root; | |
46 | u32 count; | |
8e1d6c33 | 47 | size_t size_max; |
67d1eece | 48 | |
2e238d71 BD |
49 | /* We need to take a lock in atomic context */ |
50 | spinlock_t lock; | |
67d1eece | 51 | struct list_head list; |
12927835 BD |
52 | struct list_head list_op_async; |
53 | wait_queue_head_t wq; | |
aa27bf82 BD |
54 | }; |
55 | ||
56 | static struct gb_loopback_device gb_dev; | |
57 | ||
12927835 BD |
58 | struct gb_loopback_async_operation { |
59 | struct gb_loopback *gb; | |
60 | struct gb_operation *operation; | |
31408d16 | 61 | ktime_t ts; |
12927835 BD |
62 | struct timer_list timer; |
63 | struct list_head entry; | |
64 | struct work_struct work; | |
65 | struct kref kref; | |
66 | bool pending; | |
67 | int (*completion)(struct gb_loopback_async_operation *op_async); | |
68 | }; | |
69 | ||
355a7058 AB |
70 | struct gb_loopback { |
71 | struct gb_connection *connection; | |
355a7058 | 72 | |
aa27bf82 | 73 | struct dentry *file; |
4b0ea00c | 74 | struct kfifo kfifo_lat; |
85d678c0 | 75 | struct mutex mutex; |
355a7058 | 76 | struct task_struct *task; |
67d1eece | 77 | struct list_head entry; |
079fa32b | 78 | struct device *dev; |
8e1d6c33 | 79 | wait_queue_head_t wq; |
36f241ff BD |
80 | wait_queue_head_t wq_completion; |
81 | atomic_t outstanding_operations; | |
355a7058 | 82 | |
67d1eece | 83 | /* Per connection stats */ |
31408d16 | 84 | ktime_t ts; |
355a7058 AB |
85 | struct gb_loopback_stats latency; |
86 | struct gb_loopback_stats throughput; | |
583cbf50 | 87 | struct gb_loopback_stats requests_per_second; |
1ec5843e | 88 | struct gb_loopback_stats apbridge_unipro_latency; |
e54b106d | 89 | struct gb_loopback_stats gbphy_firmware_latency; |
67d1eece | 90 | |
8e1d6c33 | 91 | int type; |
12927835 | 92 | int async; |
079fa32b | 93 | int id; |
8e1d6c33 BD |
94 | u32 size; |
95 | u32 iteration_max; | |
67d1eece | 96 | u32 iteration_count; |
b36f04fa | 97 | int us_wait; |
355a7058 | 98 | u32 error; |
12927835 BD |
99 | u32 requests_completed; |
100 | u32 requests_timedout; | |
101 | u32 timeout; | |
102 | u32 jiffy_timeout; | |
103 | u32 timeout_min; | |
104 | u32 timeout_max; | |
8e3fba55 | 105 | u32 outstanding_operations_max; |
8e1d6c33 BD |
106 | u32 lbid; |
107 | u64 elapsed_nsecs; | |
e6227ee6 | 108 | u32 apbridge_latency_ts; |
e54b106d | 109 | u32 gbphy_latency_ts; |
01480ba3 AB |
110 | |
111 | u32 send_count; | |
355a7058 AB |
112 | }; |
113 | ||
079fa32b AH |
114 | static struct class loopback_class = { |
115 | .name = "gb_loopback", | |
116 | .owner = THIS_MODULE, | |
117 | }; | |
118 | static DEFINE_IDA(loopback_ida); | |
119 | ||
12927835 BD |
120 | /* Min/max values in jiffies */ |
121 | #define GB_LOOPBACK_TIMEOUT_MIN 1 | |
122 | #define GB_LOOPBACK_TIMEOUT_MAX 10000 | |
123 | ||
cbd204b4 BD |
124 | #define GB_LOOPBACK_FIFO_DEFAULT 8192 |
125 | ||
1d3dfbd1 | 126 | static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT; |
cbd204b4 BD |
127 | module_param(kfifo_depth, uint, 0444); |
128 | ||
cbd204b4 BD |
129 | /* Maximum size of any one send data buffer we support */ |
130 | #define MAX_PACKET_SIZE (PAGE_SIZE * 2) | |
131 | ||
b36f04fa | 132 | #define GB_LOOPBACK_US_WAIT_MAX 1000000 |
355a7058 | 133 | |
355a7058 | 134 | /* interface sysfs attributes */ |
8e1d6c33 BD |
135 | #define gb_loopback_ro_attr(field) \ |
136 | static ssize_t field##_show(struct device *dev, \ | |
355a7058 AB |
137 | struct device_attribute *attr, \ |
138 | char *buf) \ | |
139 | { \ | |
079fa32b | 140 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 141 | return sprintf(buf, "%u\n", gb->field); \ |
355a7058 | 142 | } \ |
8e1d6c33 | 143 | static DEVICE_ATTR_RO(field) |
355a7058 | 144 | |
8e1d6c33 BD |
145 | #define gb_loopback_ro_stats_attr(name, field, type) \ |
146 | static ssize_t name##_##field##_show(struct device *dev, \ | |
355a7058 AB |
147 | struct device_attribute *attr, \ |
148 | char *buf) \ | |
149 | { \ | |
079fa32b | 150 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
1dc8d3d7 AB |
151 | /* Report 0 for min and max if no transfer successed */ \ |
152 | if (!gb->requests_completed) \ | |
153 | return sprintf(buf, "0\n"); \ | |
8e1d6c33 | 154 | return sprintf(buf, "%"#type"\n", gb->name.field); \ |
355a7058 | 155 | } \ |
8e1d6c33 | 156 | static DEVICE_ATTR_RO(name##_##field) |
355a7058 | 157 | |
8e1d6c33 BD |
158 | #define gb_loopback_ro_avg_attr(name) \ |
159 | static ssize_t name##_avg_show(struct device *dev, \ | |
7a135a96 AE |
160 | struct device_attribute *attr, \ |
161 | char *buf) \ | |
162 | { \ | |
f06272b2 | 163 | struct gb_loopback_stats *stats; \ |
f06272b2 | 164 | struct gb_loopback *gb; \ |
fb37f137 AB |
165 | u64 avg, rem; \ |
166 | u32 count; \ | |
079fa32b | 167 | gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 168 | stats = &gb->name; \ |
f06272b2 | 169 | count = stats->count ? stats->count : 1; \ |
58a527af | 170 | avg = stats->sum + count / 2000000; /* round closest */ \ |
f06272b2 | 171 | rem = do_div(avg, count); \ |
89ec14ce AB |
172 | rem *= 1000000; \ |
173 | do_div(rem, count); \ | |
fb37f137 | 174 | return sprintf(buf, "%llu.%06u\n", avg, (u32)rem); \ |
7a135a96 | 175 | } \ |
8e1d6c33 | 176 | static DEVICE_ATTR_RO(name##_avg) |
7a135a96 | 177 | |
8e1d6c33 BD |
178 | #define gb_loopback_stats_attrs(field) \ |
179 | gb_loopback_ro_stats_attr(field, min, u); \ | |
180 | gb_loopback_ro_stats_attr(field, max, u); \ | |
181 | gb_loopback_ro_avg_attr(field) | |
355a7058 AB |
182 | |
183 | #define gb_loopback_attr(field, type) \ | |
184 | static ssize_t field##_show(struct device *dev, \ | |
185 | struct device_attribute *attr, \ | |
186 | char *buf) \ | |
187 | { \ | |
079fa32b | 188 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
355a7058 AB |
189 | return sprintf(buf, "%"#type"\n", gb->field); \ |
190 | } \ | |
191 | static ssize_t field##_store(struct device *dev, \ | |
192 | struct device_attribute *attr, \ | |
193 | const char *buf, \ | |
194 | size_t len) \ | |
195 | { \ | |
196 | int ret; \ | |
079fa32b | 197 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 198 | mutex_lock(&gb->mutex); \ |
355a7058 | 199 | ret = sscanf(buf, "%"#type, &gb->field); \ |
355a7058 | 200 | if (ret != 1) \ |
85d678c0 BD |
201 | len = -EINVAL; \ |
202 | else \ | |
8e1d6c33 BD |
203 | gb_loopback_check_attr(gb, bundle); \ |
204 | mutex_unlock(&gb->mutex); \ | |
355a7058 AB |
205 | return len; \ |
206 | } \ | |
207 | static DEVICE_ATTR_RW(field) | |
208 | ||
f06272b2 BD |
209 | #define gb_dev_loopback_ro_attr(field, conn) \ |
210 | static ssize_t field##_show(struct device *dev, \ | |
67d1eece BD |
211 | struct device_attribute *attr, \ |
212 | char *buf) \ | |
213 | { \ | |
079fa32b | 214 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 215 | return sprintf(buf, "%u\n", gb->field); \ |
67d1eece BD |
216 | } \ |
217 | static DEVICE_ATTR_RO(field) | |
218 | ||
219 | #define gb_dev_loopback_rw_attr(field, type) \ | |
220 | static ssize_t field##_show(struct device *dev, \ | |
221 | struct device_attribute *attr, \ | |
222 | char *buf) \ | |
223 | { \ | |
079fa32b | 224 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 225 | return sprintf(buf, "%"#type"\n", gb->field); \ |
67d1eece BD |
226 | } \ |
227 | static ssize_t field##_store(struct device *dev, \ | |
228 | struct device_attribute *attr, \ | |
229 | const char *buf, \ | |
230 | size_t len) \ | |
231 | { \ | |
232 | int ret; \ | |
079fa32b | 233 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 BD |
234 | mutex_lock(&gb->mutex); \ |
235 | ret = sscanf(buf, "%"#type, &gb->field); \ | |
67d1eece BD |
236 | if (ret != 1) \ |
237 | len = -EINVAL; \ | |
238 | else \ | |
079fa32b | 239 | gb_loopback_check_attr(gb); \ |
8e1d6c33 | 240 | mutex_unlock(&gb->mutex); \ |
67d1eece BD |
241 | return len; \ |
242 | } \ | |
243 | static DEVICE_ATTR_RW(field) | |
244 | ||
8e1d6c33 | 245 | static void gb_loopback_reset_stats(struct gb_loopback *gb); |
079fa32b | 246 | static void gb_loopback_check_attr(struct gb_loopback *gb) |
355a7058 | 247 | { |
b36f04fa BD |
248 | if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX) |
249 | gb->us_wait = GB_LOOPBACK_US_WAIT_MAX; | |
8e1d6c33 BD |
250 | if (gb->size > gb_dev.size_max) |
251 | gb->size = gb_dev.size_max; | |
12927835 BD |
252 | gb->requests_timedout = 0; |
253 | gb->requests_completed = 0; | |
8e1d6c33 | 254 | gb->iteration_count = 0; |
01480ba3 | 255 | gb->send_count = 0; |
8e1d6c33 BD |
256 | gb->error = 0; |
257 | ||
258 | if (kfifo_depth < gb->iteration_max) { | |
079fa32b | 259 | dev_warn(gb->dev, |
8e1d6c33 BD |
260 | "cannot log bytes %u kfifo_depth %u\n", |
261 | gb->iteration_max, kfifo_depth); | |
cb60f496 | 262 | } |
8e1d6c33 | 263 | kfifo_reset_out(&gb->kfifo_lat); |
cb60f496 | 264 | |
8e1d6c33 | 265 | switch (gb->type) { |
a598f438 BD |
266 | case GB_LOOPBACK_TYPE_PING: |
267 | case GB_LOOPBACK_TYPE_TRANSFER: | |
384a7a3c | 268 | case GB_LOOPBACK_TYPE_SINK: |
12927835 BD |
269 | gb->jiffy_timeout = usecs_to_jiffies(gb->timeout); |
270 | if (!gb->jiffy_timeout) | |
271 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN; | |
272 | else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX) | |
273 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX; | |
8e1d6c33 BD |
274 | gb_loopback_reset_stats(gb); |
275 | wake_up(&gb->wq); | |
a598f438 BD |
276 | break; |
277 | default: | |
8e1d6c33 | 278 | gb->type = 0; |
a598f438 BD |
279 | break; |
280 | } | |
355a7058 AB |
281 | } |
282 | ||
283 | /* Time to send and receive one message */ | |
8e1d6c33 | 284 | gb_loopback_stats_attrs(latency); |
583cbf50 | 285 | /* Number of requests sent per second on this cport */ |
8e1d6c33 | 286 | gb_loopback_stats_attrs(requests_per_second); |
355a7058 | 287 | /* Quantity of data sent and received on this cport */ |
8e1d6c33 | 288 | gb_loopback_stats_attrs(throughput); |
1ec5843e | 289 | /* Latency across the UniPro link from APBridge's perspective */ |
8e1d6c33 | 290 | gb_loopback_stats_attrs(apbridge_unipro_latency); |
1ec5843e | 291 | /* Firmware induced overhead in the GPBridge */ |
e54b106d | 292 | gb_loopback_stats_attrs(gbphy_firmware_latency); |
8e1d6c33 | 293 | |
e140c75e | 294 | /* Number of errors encountered during loop */ |
8e1d6c33 | 295 | gb_loopback_ro_attr(error); |
12927835 BD |
296 | /* Number of requests successfully completed async */ |
297 | gb_loopback_ro_attr(requests_completed); | |
298 | /* Number of requests timed out async */ | |
299 | gb_loopback_ro_attr(requests_timedout); | |
300 | /* Timeout minimum in useconds */ | |
301 | gb_loopback_ro_attr(timeout_min); | |
302 | /* Timeout minimum in useconds */ | |
303 | gb_loopback_ro_attr(timeout_max); | |
355a7058 AB |
304 | |
305 | /* | |
799a3f03 | 306 | * Type of loopback message to send based on protocol type definitions |
355a7058 | 307 | * 0 => Don't send message |
799a3f03 | 308 | * 2 => Send ping message continuously (message without payload) |
006335a0 | 309 | * 3 => Send transfer message continuously (message with payload, |
799a3f03 BD |
310 | * payload returned in response) |
311 | * 4 => Send a sink message (message with payload, no payload in response) | |
355a7058 | 312 | */ |
67d1eece | 313 | gb_dev_loopback_rw_attr(type, d); |
355a7058 | 314 | /* Size of transfer message payload: 0-4096 bytes */ |
67d1eece | 315 | gb_dev_loopback_rw_attr(size, u); |
48f19ee8 | 316 | /* Time to wait between two messages: 0-1000 ms */ |
b36f04fa | 317 | gb_dev_loopback_rw_attr(us_wait, d); |
00af6583 | 318 | /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */ |
67d1eece BD |
319 | gb_dev_loopback_rw_attr(iteration_max, u); |
320 | /* The current index of the for (i = 0; i < iteration_max; i++) loop */ | |
f06272b2 | 321 | gb_dev_loopback_ro_attr(iteration_count, false); |
12927835 BD |
322 | /* A flag to indicate synchronous or asynchronous operations */ |
323 | gb_dev_loopback_rw_attr(async, u); | |
324 | /* Timeout of an individual asynchronous request */ | |
325 | gb_dev_loopback_rw_attr(timeout, u); | |
8e3fba55 BD |
326 | /* Maximum number of in-flight operations before back-off */ |
327 | gb_dev_loopback_rw_attr(outstanding_operations_max, u); | |
355a7058 | 328 | |
8e1d6c33 BD |
329 | static struct attribute *loopback_attrs[] = { |
330 | &dev_attr_latency_min.attr, | |
331 | &dev_attr_latency_max.attr, | |
332 | &dev_attr_latency_avg.attr, | |
333 | &dev_attr_requests_per_second_min.attr, | |
334 | &dev_attr_requests_per_second_max.attr, | |
335 | &dev_attr_requests_per_second_avg.attr, | |
336 | &dev_attr_throughput_min.attr, | |
337 | &dev_attr_throughput_max.attr, | |
338 | &dev_attr_throughput_avg.attr, | |
339 | &dev_attr_apbridge_unipro_latency_min.attr, | |
340 | &dev_attr_apbridge_unipro_latency_max.attr, | |
341 | &dev_attr_apbridge_unipro_latency_avg.attr, | |
e54b106d SP |
342 | &dev_attr_gbphy_firmware_latency_min.attr, |
343 | &dev_attr_gbphy_firmware_latency_max.attr, | |
344 | &dev_attr_gbphy_firmware_latency_avg.attr, | |
355a7058 AB |
345 | &dev_attr_type.attr, |
346 | &dev_attr_size.attr, | |
b36f04fa | 347 | &dev_attr_us_wait.attr, |
00af6583 BD |
348 | &dev_attr_iteration_count.attr, |
349 | &dev_attr_iteration_max.attr, | |
12927835 | 350 | &dev_attr_async.attr, |
8e1d6c33 | 351 | &dev_attr_error.attr, |
12927835 BD |
352 | &dev_attr_requests_completed.attr, |
353 | &dev_attr_requests_timedout.attr, | |
354 | &dev_attr_timeout.attr, | |
8e3fba55 | 355 | &dev_attr_outstanding_operations_max.attr, |
12927835 BD |
356 | &dev_attr_timeout_min.attr, |
357 | &dev_attr_timeout_max.attr, | |
355a7058 AB |
358 | NULL, |
359 | }; | |
8e1d6c33 | 360 | ATTRIBUTE_GROUPS(loopback); |
355a7058 | 361 | |
ab81bb9c | 362 | static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error); |
12927835 | 363 | |
bd416103 BD |
364 | static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs) |
365 | { | |
bd416103 | 366 | do_div(elapsed_nsecs, NSEC_PER_USEC); |
5c143123 | 367 | return elapsed_nsecs; |
bd416103 BD |
368 | } |
369 | ||
7c985351 BD |
370 | static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) |
371 | { | |
372 | if (t2 > t1) | |
373 | return t2 - t1; | |
374 | else | |
375 | return NSEC_PER_DAY - t2 + t1; | |
376 | } | |
377 | ||
31408d16 | 378 | static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te) |
4c192665 | 379 | { |
31408d16 | 380 | return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te)); |
4c192665 BD |
381 | } |
382 | ||
fbb8edba BD |
383 | static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, |
384 | void *request, int request_size, | |
385 | void *response, int response_size) | |
384a7a3c | 386 | { |
fbb8edba | 387 | struct gb_operation *operation; |
31408d16 | 388 | ktime_t ts, te; |
fbb8edba | 389 | int ret; |
384a7a3c | 390 | |
31408d16 | 391 | ts = ktime_get(); |
fbb8edba BD |
392 | operation = gb_operation_create(gb->connection, type, request_size, |
393 | response_size, GFP_KERNEL); | |
d9048d8c AB |
394 | if (!operation) |
395 | return -ENOMEM; | |
c3bba87a | 396 | |
fbb8edba BD |
397 | if (request_size) |
398 | memcpy(operation->request->payload, request, request_size); | |
399 | ||
400 | ret = gb_operation_request_send_sync(operation); | |
401 | if (ret) { | |
d9a9ea1b | 402 | dev_err(&gb->connection->bundle->dev, |
fbb8edba | 403 | "synchronous operation failed: %d\n", ret); |
d9048d8c | 404 | goto out_put_operation; |
fbb8edba BD |
405 | } else { |
406 | if (response_size == operation->response->payload_size) { | |
407 | memcpy(response, operation->response->payload, | |
408 | response_size); | |
409 | } else { | |
d9a9ea1b | 410 | dev_err(&gb->connection->bundle->dev, |
fbb8edba BD |
411 | "response size %zu expected %d\n", |
412 | operation->response->payload_size, | |
413 | response_size); | |
d9048d8c AB |
414 | ret = -EINVAL; |
415 | goto out_put_operation; | |
fbb8edba BD |
416 | } |
417 | } | |
6ab1ce4d | 418 | |
31408d16 | 419 | te = ktime_get(); |
2f842304 BD |
420 | |
421 | /* Calculate the total time the message took */ | |
31408d16 | 422 | gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te); |
2f842304 | 423 | |
d9048d8c AB |
424 | out_put_operation: |
425 | gb_operation_put(operation); | |
426 | ||
fbb8edba BD |
427 | return ret; |
428 | } | |
384a7a3c | 429 | |
12927835 BD |
430 | static void __gb_loopback_async_operation_destroy(struct kref *kref) |
431 | { | |
432 | struct gb_loopback_async_operation *op_async; | |
433 | ||
434 | op_async = container_of(kref, struct gb_loopback_async_operation, kref); | |
435 | ||
436 | list_del(&op_async->entry); | |
437 | if (op_async->operation) | |
438 | gb_operation_put(op_async->operation); | |
36f241ff BD |
439 | atomic_dec(&op_async->gb->outstanding_operations); |
440 | wake_up(&op_async->gb->wq_completion); | |
12927835 BD |
441 | kfree(op_async); |
442 | } | |
443 | ||
444 | static void gb_loopback_async_operation_get(struct gb_loopback_async_operation | |
445 | *op_async) | |
446 | { | |
447 | kref_get(&op_async->kref); | |
448 | } | |
449 | ||
450 | static void gb_loopback_async_operation_put(struct gb_loopback_async_operation | |
451 | *op_async) | |
452 | { | |
453 | unsigned long flags; | |
454 | ||
455 | spin_lock_irqsave(&gb_dev.lock, flags); | |
456 | kref_put(&op_async->kref, __gb_loopback_async_operation_destroy); | |
457 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
458 | } | |
459 | ||
460 | static struct gb_loopback_async_operation * | |
461 | gb_loopback_operation_find(u16 id) | |
462 | { | |
463 | struct gb_loopback_async_operation *op_async; | |
464 | bool found = false; | |
465 | unsigned long flags; | |
466 | ||
467 | spin_lock_irqsave(&gb_dev.lock, flags); | |
468 | list_for_each_entry(op_async, &gb_dev.list_op_async, entry) { | |
469 | if (op_async->operation->id == id) { | |
470 | gb_loopback_async_operation_get(op_async); | |
471 | found = true; | |
472 | break; | |
473 | } | |
474 | } | |
475 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
476 | ||
477 | return found ? op_async : NULL; | |
478 | } | |
479 | ||
36f241ff BD |
480 | static void gb_loopback_async_wait_all(struct gb_loopback *gb) |
481 | { | |
482 | wait_event(gb->wq_completion, | |
483 | !atomic_read(&gb->outstanding_operations)); | |
484 | } | |
485 | ||
12927835 BD |
486 | static void gb_loopback_async_operation_callback(struct gb_operation *operation) |
487 | { | |
488 | struct gb_loopback_async_operation *op_async; | |
489 | struct gb_loopback *gb; | |
31408d16 | 490 | ktime_t te; |
12927835 BD |
491 | bool err = false; |
492 | ||
31408d16 | 493 | te = ktime_get(); |
12927835 BD |
494 | op_async = gb_loopback_operation_find(operation->id); |
495 | if (!op_async) | |
496 | return; | |
497 | ||
498 | gb = op_async->gb; | |
499 | mutex_lock(&gb->mutex); | |
500 | ||
501 | if (!op_async->pending || gb_operation_result(operation)) { | |
502 | err = true; | |
503 | } else { | |
504 | if (op_async->completion) | |
505 | if (op_async->completion(op_async)) | |
506 | err = true; | |
507 | } | |
508 | ||
8563a49c | 509 | if (!err) |
31408d16 | 510 | gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te); |
12927835 BD |
511 | |
512 | if (op_async->pending) { | |
41993cd5 JH |
513 | if (err) |
514 | gb->error++; | |
12927835 BD |
515 | gb->iteration_count++; |
516 | op_async->pending = false; | |
517 | del_timer_sync(&op_async->timer); | |
518 | gb_loopback_async_operation_put(op_async); | |
ab81bb9c | 519 | gb_loopback_calculate_stats(gb, err); |
12927835 BD |
520 | } |
521 | mutex_unlock(&gb->mutex); | |
522 | ||
523 | dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", | |
524 | operation->id); | |
525 | ||
526 | gb_loopback_async_operation_put(op_async); | |
527 | } | |
528 | ||
529 | static void gb_loopback_async_operation_work(struct work_struct *work) | |
530 | { | |
531 | struct gb_loopback *gb; | |
532 | struct gb_operation *operation; | |
533 | struct gb_loopback_async_operation *op_async; | |
534 | ||
535 | op_async = container_of(work, struct gb_loopback_async_operation, work); | |
12927835 BD |
536 | gb = op_async->gb; |
537 | operation = op_async->operation; | |
538 | ||
539 | mutex_lock(&gb->mutex); | |
540 | if (op_async->pending) { | |
541 | gb->requests_timedout++; | |
542 | gb->error++; | |
543 | gb->iteration_count++; | |
544 | op_async->pending = false; | |
545 | gb_loopback_async_operation_put(op_async); | |
ab81bb9c | 546 | gb_loopback_calculate_stats(gb, true); |
12927835 BD |
547 | } |
548 | mutex_unlock(&gb->mutex); | |
549 | ||
550 | dev_dbg(&gb->connection->bundle->dev, "timeout operation %d\n", | |
551 | operation->id); | |
552 | ||
553 | gb_operation_cancel(operation, -ETIMEDOUT); | |
554 | gb_loopback_async_operation_put(op_async); | |
555 | } | |
556 | ||
557 | static void gb_loopback_async_operation_timeout(unsigned long data) | |
558 | { | |
559 | struct gb_loopback_async_operation *op_async; | |
560 | u16 id = data; | |
561 | ||
562 | op_async = gb_loopback_operation_find(id); | |
563 | if (!op_async) { | |
564 | pr_err("operation %d not found - time out ?\n", id); | |
565 | return; | |
566 | } | |
567 | schedule_work(&op_async->work); | |
568 | } | |
569 | ||
570 | static int gb_loopback_async_operation(struct gb_loopback *gb, int type, | |
571 | void *request, int request_size, | |
572 | int response_size, | |
573 | void *completion) | |
574 | { | |
575 | struct gb_loopback_async_operation *op_async; | |
576 | struct gb_operation *operation; | |
577 | int ret; | |
578 | unsigned long flags; | |
579 | ||
580 | op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); | |
581 | if (!op_async) | |
582 | return -ENOMEM; | |
583 | ||
584 | INIT_WORK(&op_async->work, gb_loopback_async_operation_work); | |
12927835 BD |
585 | kref_init(&op_async->kref); |
586 | ||
587 | operation = gb_operation_create(gb->connection, type, request_size, | |
588 | response_size, GFP_KERNEL); | |
589 | if (!operation) { | |
c7aae4e6 BD |
590 | kfree(op_async); |
591 | return -ENOMEM; | |
12927835 BD |
592 | } |
593 | ||
594 | if (request_size) | |
595 | memcpy(operation->request->payload, request, request_size); | |
596 | ||
597 | op_async->gb = gb; | |
598 | op_async->operation = operation; | |
599 | op_async->completion = completion; | |
600 | ||
601 | spin_lock_irqsave(&gb_dev.lock, flags); | |
602 | list_add_tail(&op_async->entry, &gb_dev.list_op_async); | |
603 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
604 | ||
31408d16 | 605 | op_async->ts = ktime_get(); |
12927835 | 606 | op_async->pending = true; |
36f241ff | 607 | atomic_inc(&gb->outstanding_operations); |
fece9c87 | 608 | mutex_lock(&gb->mutex); |
12927835 BD |
609 | ret = gb_operation_request_send(operation, |
610 | gb_loopback_async_operation_callback, | |
dbec2729 | 611 | 0, |
12927835 BD |
612 | GFP_KERNEL); |
613 | if (ret) | |
614 | goto error; | |
615 | ||
82af03f7 | 616 | setup_timer(&op_async->timer, gb_loopback_async_operation_timeout, |
617 | (unsigned long)operation->id); | |
12927835 | 618 | op_async->timer.expires = jiffies + gb->jiffy_timeout; |
12927835 BD |
619 | add_timer(&op_async->timer); |
620 | ||
fece9c87 | 621 | goto done; |
12927835 BD |
622 | error: |
623 | gb_loopback_async_operation_put(op_async); | |
fece9c87 BD |
624 | done: |
625 | mutex_unlock(&gb->mutex); | |
12927835 BD |
626 | return ret; |
627 | } | |
628 | ||
629 | static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len) | |
fbb8edba BD |
630 | { |
631 | struct gb_loopback_transfer_request *request; | |
632 | int retval; | |
633 | ||
634 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
635 | if (!request) | |
636 | return -ENOMEM; | |
637 | ||
638 | request->len = cpu_to_le32(len); | |
639 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK, | |
640 | request, len + sizeof(*request), | |
641 | NULL, 0); | |
384a7a3c BD |
642 | kfree(request); |
643 | return retval; | |
644 | } | |
645 | ||
12927835 | 646 | static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len) |
355a7058 | 647 | { |
355a7058 AB |
648 | struct gb_loopback_transfer_request *request; |
649 | struct gb_loopback_transfer_response *response; | |
650 | int retval; | |
651 | ||
d6a1a3b5 | 652 | gb->apbridge_latency_ts = 0; |
e54b106d | 653 | gb->gbphy_latency_ts = 0; |
d6a1a3b5 | 654 | |
355a7058 AB |
655 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); |
656 | if (!request) | |
657 | return -ENOMEM; | |
658 | response = kmalloc(len + sizeof(*response), GFP_KERNEL); | |
659 | if (!response) { | |
660 | kfree(request); | |
661 | return -ENOMEM; | |
662 | } | |
663 | ||
dc4a1069 VK |
664 | memset(request->data, 0x5A, len); |
665 | ||
355a7058 | 666 | request->len = cpu_to_le32(len); |
fbb8edba BD |
667 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER, |
668 | request, len + sizeof(*request), | |
669 | response, len + sizeof(*response)); | |
355a7058 AB |
670 | if (retval) |
671 | goto gb_error; | |
672 | ||
dc366f8e | 673 | if (memcmp(request->data, response->data, len)) { |
d9a9ea1b GKH |
674 | dev_err(&gb->connection->bundle->dev, |
675 | "Loopback Data doesn't match\n"); | |
355a7058 | 676 | retval = -EREMOTEIO; |
dc366f8e | 677 | } |
1ec5843e | 678 | gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0); |
e54b106d | 679 | gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1); |
355a7058 AB |
680 | |
681 | gb_error: | |
682 | kfree(request); | |
683 | kfree(response); | |
684 | ||
685 | return retval; | |
686 | } | |
687 | ||
12927835 | 688 | static int gb_loopback_sync_ping(struct gb_loopback *gb) |
355a7058 | 689 | { |
fbb8edba BD |
690 | return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING, |
691 | NULL, 0, NULL, 0); | |
355a7058 AB |
692 | } |
693 | ||
12927835 BD |
694 | static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len) |
695 | { | |
696 | struct gb_loopback_transfer_request *request; | |
697 | int retval; | |
698 | ||
699 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
700 | if (!request) | |
701 | return -ENOMEM; | |
702 | ||
703 | request->len = cpu_to_le32(len); | |
704 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK, | |
705 | request, len + sizeof(*request), | |
706 | 0, NULL); | |
707 | kfree(request); | |
708 | return retval; | |
709 | } | |
710 | ||
711 | static int gb_loopback_async_transfer_complete( | |
712 | struct gb_loopback_async_operation *op_async) | |
713 | { | |
714 | struct gb_loopback *gb; | |
715 | struct gb_operation *operation; | |
716 | struct gb_loopback_transfer_request *request; | |
717 | struct gb_loopback_transfer_response *response; | |
718 | size_t len; | |
719 | int retval = 0; | |
720 | ||
721 | gb = op_async->gb; | |
722 | operation = op_async->operation; | |
723 | request = operation->request->payload; | |
724 | response = operation->response->payload; | |
725 | len = le32_to_cpu(request->len); | |
726 | ||
727 | if (memcmp(request->data, response->data, len)) { | |
728 | dev_err(&gb->connection->bundle->dev, | |
729 | "Loopback Data doesn't match operation id %d\n", | |
730 | operation->id); | |
731 | retval = -EREMOTEIO; | |
732 | } else { | |
733 | gb->apbridge_latency_ts = | |
734 | (u32)__le32_to_cpu(response->reserved0); | |
e54b106d | 735 | gb->gbphy_latency_ts = |
12927835 BD |
736 | (u32)__le32_to_cpu(response->reserved1); |
737 | } | |
738 | ||
739 | return retval; | |
740 | } | |
741 | ||
742 | static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len) | |
743 | { | |
744 | struct gb_loopback_transfer_request *request; | |
745 | int retval, response_len; | |
746 | ||
747 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
748 | if (!request) | |
749 | return -ENOMEM; | |
750 | ||
751 | memset(request->data, 0x5A, len); | |
752 | ||
753 | request->len = cpu_to_le32(len); | |
754 | response_len = sizeof(struct gb_loopback_transfer_response); | |
755 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER, | |
756 | request, len + sizeof(*request), | |
757 | len + response_len, | |
758 | gb_loopback_async_transfer_complete); | |
759 | if (retval) | |
760 | goto gb_error; | |
761 | ||
762 | gb_error: | |
763 | kfree(request); | |
764 | return retval; | |
765 | } | |
766 | ||
767 | static int gb_loopback_async_ping(struct gb_loopback *gb) | |
768 | { | |
769 | return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING, | |
770 | NULL, 0, 0, NULL); | |
771 | } | |
772 | ||
e82a11dc | 773 | static int gb_loopback_request_handler(struct gb_operation *operation) |
ac1c2840 AE |
774 | { |
775 | struct gb_connection *connection = operation->connection; | |
776 | struct gb_loopback_transfer_request *request; | |
777 | struct gb_loopback_transfer_response *response; | |
d9a9ea1b | 778 | struct device *dev = &connection->bundle->dev; |
e51eafeb | 779 | size_t len; |
ac1c2840 AE |
780 | |
781 | /* By convention, the AP initiates the version operation */ | |
e82a11dc | 782 | switch (operation->type) { |
ac1c2840 | 783 | case GB_LOOPBACK_TYPE_PING: |
384a7a3c | 784 | case GB_LOOPBACK_TYPE_SINK: |
ac1c2840 AE |
785 | return 0; |
786 | case GB_LOOPBACK_TYPE_TRANSFER: | |
787 | if (operation->request->payload_size < sizeof(*request)) { | |
d9a9ea1b | 788 | dev_err(dev, "transfer request too small (%zu < %zu)\n", |
ac1c2840 AE |
789 | operation->request->payload_size, |
790 | sizeof(*request)); | |
791 | return -EINVAL; /* -EMSGSIZE */ | |
792 | } | |
793 | request = operation->request->payload; | |
794 | len = le32_to_cpu(request->len); | |
67d1eece | 795 | if (len > gb_dev.size_max) { |
d9a9ea1b | 796 | dev_err(dev, "transfer request too large (%zu > %zu)\n", |
67d1eece | 797 | len, gb_dev.size_max); |
c3bba87a BD |
798 | return -EINVAL; |
799 | } | |
800 | ||
81ad6994 BG |
801 | if (!gb_operation_response_alloc(operation, |
802 | len + sizeof(*response), GFP_KERNEL)) { | |
803 | dev_err(dev, "error allocating response\n"); | |
804 | return -ENOMEM; | |
ac1c2840 | 805 | } |
81ad6994 BG |
806 | response = operation->response->payload; |
807 | response->len = cpu_to_le32(len); | |
808 | if (len) | |
809 | memcpy(response->data, request->data, len); | |
810 | ||
ac1c2840 AE |
811 | return 0; |
812 | default: | |
e82a11dc | 813 | dev_err(dev, "unsupported request: %u\n", operation->type); |
ac1c2840 AE |
814 | return -EINVAL; |
815 | } | |
816 | } | |
817 | ||
8e1d6c33 | 818 | static void gb_loopback_reset_stats(struct gb_loopback *gb) |
355a7058 AB |
819 | { |
820 | struct gb_loopback_stats reset = { | |
e051c82b | 821 | .min = U32_MAX, |
355a7058 | 822 | }; |
67d1eece BD |
823 | |
824 | /* Reset per-connection stats */ | |
8e1d6c33 BD |
825 | memcpy(&gb->latency, &reset, |
826 | sizeof(struct gb_loopback_stats)); | |
827 | memcpy(&gb->throughput, &reset, | |
828 | sizeof(struct gb_loopback_stats)); | |
829 | memcpy(&gb->requests_per_second, &reset, | |
830 | sizeof(struct gb_loopback_stats)); | |
831 | memcpy(&gb->apbridge_unipro_latency, &reset, | |
832 | sizeof(struct gb_loopback_stats)); | |
e54b106d | 833 | memcpy(&gb->gbphy_firmware_latency, &reset, |
8e1d6c33 | 834 | sizeof(struct gb_loopback_stats)); |
67d1eece | 835 | |
f42a6891 BD |
836 | /* Should be initialized at least once per transaction set */ |
837 | gb->apbridge_latency_ts = 0; | |
e54b106d | 838 | gb->gbphy_latency_ts = 0; |
31408d16 | 839 | gb->ts = ktime_set(0, 0); |
355a7058 AB |
840 | } |
841 | ||
a6e7e535 | 842 | static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) |
355a7058 | 843 | { |
00af6583 BD |
844 | if (stats->min > val) |
845 | stats->min = val; | |
846 | if (stats->max < val) | |
847 | stats->max = val; | |
848 | stats->sum += val; | |
849 | stats->count++; | |
355a7058 AB |
850 | } |
851 | ||
ab81bb9c AB |
852 | static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats, |
853 | u64 val, u32 count) | |
854 | { | |
855 | stats->sum += val; | |
856 | stats->count += count; | |
857 | ||
858 | do_div(val, count); | |
859 | if (stats->min > val) | |
860 | stats->min = val; | |
861 | if (stats->max < val) | |
862 | stats->max = val; | |
863 | } | |
864 | ||
583cbf50 | 865 | static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 866 | { |
ab81bb9c | 867 | u64 req = gb->requests_completed * USEC_PER_SEC; |
00af6583 | 868 | |
ab81bb9c | 869 | gb_loopback_update_stats_window(&gb->requests_per_second, req, latency); |
355a7058 AB |
870 | } |
871 | ||
00af6583 | 872 | static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 873 | { |
ab81bb9c | 874 | u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2; |
f7908e4d | 875 | |
8e1d6c33 | 876 | switch (gb->type) { |
f7908e4d BD |
877 | case GB_LOOPBACK_TYPE_PING: |
878 | break; | |
879 | case GB_LOOPBACK_TYPE_SINK: | |
880 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
8e1d6c33 | 881 | gb->size; |
f7908e4d BD |
882 | break; |
883 | case GB_LOOPBACK_TYPE_TRANSFER: | |
884 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
885 | sizeof(struct gb_loopback_transfer_response) + | |
8e1d6c33 | 886 | gb->size * 2; |
f7908e4d BD |
887 | break; |
888 | default: | |
889 | return; | |
890 | } | |
00af6583 | 891 | |
ab81bb9c AB |
892 | aggregate_size *= gb->requests_completed; |
893 | aggregate_size *= USEC_PER_SEC; | |
894 | gb_loopback_update_stats_window(&gb->throughput, aggregate_size, | |
895 | latency); | |
355a7058 AB |
896 | } |
897 | ||
ab81bb9c | 898 | static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb) |
355a7058 AB |
899 | { |
900 | u32 lat; | |
355a7058 | 901 | |
00af6583 | 902 | /* Express latency in terms of microseconds */ |
bd416103 | 903 | lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs); |
355a7058 | 904 | |
98676ca8 | 905 | /* Log latency stastic */ |
00af6583 | 906 | gb_loopback_update_stats(&gb->latency, lat); |
67d1eece BD |
907 | |
908 | /* Raw latency log on a per thread basis */ | |
4b0ea00c | 909 | kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat)); |
00af6583 | 910 | |
1ec5843e | 911 | /* Log the firmware supplied latency values */ |
1ec5843e BD |
912 | gb_loopback_update_stats(&gb->apbridge_unipro_latency, |
913 | gb->apbridge_latency_ts); | |
e54b106d SP |
914 | gb_loopback_update_stats(&gb->gbphy_firmware_latency, |
915 | gb->gbphy_latency_ts); | |
355a7058 AB |
916 | } |
917 | ||
ab81bb9c AB |
918 | static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error) |
919 | { | |
920 | u64 nlat; | |
921 | u32 lat; | |
31408d16 | 922 | ktime_t te; |
ab81bb9c AB |
923 | |
924 | if (!error) { | |
925 | gb->requests_completed++; | |
926 | gb_loopback_calculate_latency_stats(gb); | |
927 | } | |
928 | ||
31408d16 AB |
929 | te = ktime_get(); |
930 | nlat = gb_loopback_calc_latency(gb->ts, te); | |
ab81bb9c AB |
931 | if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) { |
932 | lat = gb_loopback_nsec_to_usec_latency(nlat); | |
933 | ||
934 | gb_loopback_throughput_update(gb, lat); | |
935 | gb_loopback_requests_update(gb, lat); | |
936 | ||
937 | if (gb->iteration_count != gb->iteration_max) { | |
938 | gb->ts = te; | |
939 | gb->requests_completed = 0; | |
940 | } | |
941 | } | |
942 | } | |
943 | ||
8e3fba55 BD |
944 | static void gb_loopback_async_wait_to_send(struct gb_loopback *gb) |
945 | { | |
946 | if (!(gb->async && gb->outstanding_operations_max)) | |
947 | return; | |
948 | wait_event_interruptible(gb->wq_completion, | |
949 | (atomic_read(&gb->outstanding_operations) < | |
950 | gb->outstanding_operations_max) || | |
951 | kthread_should_stop()); | |
952 | } | |
953 | ||
355a7058 AB |
954 | static int gb_loopback_fn(void *data) |
955 | { | |
956 | int error = 0; | |
b36f04fa | 957 | int us_wait = 0; |
67d1eece | 958 | int type; |
e854ff58 | 959 | int ret; |
67d1eece | 960 | u32 size; |
01480ba3 | 961 | |
3f12c3ed | 962 | struct gb_loopback *gb = data; |
e854ff58 AH |
963 | struct gb_bundle *bundle = gb->connection->bundle; |
964 | ||
965 | ret = gb_pm_runtime_get_sync(bundle); | |
966 | if (ret) | |
967 | return ret; | |
355a7058 | 968 | |
3dfe8aaa | 969 | while (1) { |
e854ff58 AH |
970 | if (!gb->type) { |
971 | gb_pm_runtime_put_autosuspend(bundle); | |
8e1d6c33 | 972 | wait_event_interruptible(gb->wq, gb->type || |
3dfe8aaa | 973 | kthread_should_stop()); |
e854ff58 AH |
974 | ret = gb_pm_runtime_get_sync(bundle); |
975 | if (ret) | |
976 | return ret; | |
977 | } | |
978 | ||
8e3fba55 BD |
979 | if (kthread_should_stop()) |
980 | break; | |
12927835 | 981 | |
8e3fba55 BD |
982 | /* Limit the maximum number of in-flight async operations */ |
983 | gb_loopback_async_wait_to_send(gb); | |
3dfe8aaa BD |
984 | if (kthread_should_stop()) |
985 | break; | |
85d678c0 | 986 | |
67d1eece | 987 | mutex_lock(&gb->mutex); |
8e1d6c33 BD |
988 | |
989 | /* Optionally terminate */ | |
01480ba3 | 990 | if (gb->send_count == gb->iteration_max) { |
a072a72d BD |
991 | mutex_unlock(&gb->mutex); |
992 | ||
993 | /* Wait for synchronous and asynchronus completion */ | |
994 | gb_loopback_async_wait_all(gb); | |
995 | ||
996 | /* Mark complete unless user-space has poked us */ | |
997 | mutex_lock(&gb->mutex); | |
ab81bb9c AB |
998 | if (gb->iteration_count == gb->iteration_max) { |
999 | gb->type = 0; | |
01480ba3 | 1000 | gb->send_count = 0; |
39c2787b AH |
1001 | sysfs_notify(&gb->dev->kobj, NULL, |
1002 | "iteration_count"); | |
a072a72d BD |
1003 | dev_dbg(&bundle->dev, "load test complete\n"); |
1004 | } else { | |
1005 | dev_dbg(&bundle->dev, | |
1006 | "continuing on with new test set\n"); | |
ab81bb9c | 1007 | } |
67d1eece | 1008 | mutex_unlock(&gb->mutex); |
8e1d6c33 | 1009 | continue; |
67d1eece | 1010 | } |
8e1d6c33 | 1011 | size = gb->size; |
b36f04fa | 1012 | us_wait = gb->us_wait; |
8e1d6c33 | 1013 | type = gb->type; |
31408d16 AB |
1014 | if (ktime_to_ns(gb->ts) == 0) |
1015 | gb->ts = ktime_get(); | |
d9fb3754 BD |
1016 | mutex_unlock(&gb->mutex); |
1017 | ||
67d1eece | 1018 | /* Else operations to perform */ |
12927835 | 1019 | if (gb->async) { |
a1f10771 | 1020 | if (type == GB_LOOPBACK_TYPE_PING) |
12927835 | 1021 | error = gb_loopback_async_ping(gb); |
a1f10771 | 1022 | else if (type == GB_LOOPBACK_TYPE_TRANSFER) |
12927835 | 1023 | error = gb_loopback_async_transfer(gb, size); |
a1f10771 | 1024 | else if (type == GB_LOOPBACK_TYPE_SINK) |
12927835 | 1025 | error = gb_loopback_async_sink(gb, size); |
12927835 BD |
1026 | |
1027 | if (error) | |
1028 | gb->error++; | |
1029 | } else { | |
1030 | /* We are effectively single threaded here */ | |
1031 | if (type == GB_LOOPBACK_TYPE_PING) | |
1032 | error = gb_loopback_sync_ping(gb); | |
1033 | else if (type == GB_LOOPBACK_TYPE_TRANSFER) | |
1034 | error = gb_loopback_sync_transfer(gb, size); | |
1035 | else if (type == GB_LOOPBACK_TYPE_SINK) | |
1036 | error = gb_loopback_sync_sink(gb, size); | |
1037 | ||
1038 | if (error) | |
1039 | gb->error++; | |
1040 | gb->iteration_count++; | |
ab81bb9c | 1041 | gb_loopback_calculate_stats(gb, !!error); |
12927835 | 1042 | } |
01480ba3 | 1043 | gb->send_count++; |
33b8807a JH |
1044 | |
1045 | if (us_wait) { | |
1046 | if (us_wait < 20000) | |
1047 | usleep_range(us_wait, us_wait + 100); | |
1048 | else | |
1049 | msleep(us_wait / 1000); | |
1050 | } | |
355a7058 | 1051 | } |
e854ff58 AH |
1052 | |
1053 | gb_pm_runtime_put_autosuspend(bundle); | |
1054 | ||
355a7058 AB |
1055 | return 0; |
1056 | } | |
1057 | ||
4b0ea00c BD |
1058 | static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s, |
1059 | struct kfifo *kfifo, | |
1060 | struct mutex *mutex) | |
aa27bf82 | 1061 | { |
aa27bf82 BD |
1062 | u32 latency; |
1063 | int retval; | |
1064 | ||
4b0ea00c | 1065 | if (kfifo_len(kfifo) == 0) { |
aa27bf82 BD |
1066 | retval = -EAGAIN; |
1067 | goto done; | |
1068 | } | |
1069 | ||
4b0ea00c BD |
1070 | mutex_lock(mutex); |
1071 | retval = kfifo_out(kfifo, &latency, sizeof(latency)); | |
aa27bf82 BD |
1072 | if (retval > 0) { |
1073 | seq_printf(s, "%u", latency); | |
1074 | retval = 0; | |
1075 | } | |
4b0ea00c | 1076 | mutex_unlock(mutex); |
aa27bf82 BD |
1077 | done: |
1078 | return retval; | |
1079 | } | |
1080 | ||
4b0ea00c BD |
1081 | static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused) |
1082 | { | |
1083 | struct gb_loopback *gb = s->private; | |
1084 | ||
1085 | return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat, | |
1086 | &gb->mutex); | |
1087 | } | |
1088 | ||
aa27bf82 BD |
1089 | static int gb_loopback_latency_open(struct inode *inode, struct file *file) |
1090 | { | |
1091 | return single_open(file, gb_loopback_dbgfs_latency_show, | |
1092 | inode->i_private); | |
1093 | } | |
1094 | ||
1095 | static const struct file_operations gb_loopback_debugfs_latency_ops = { | |
1096 | .open = gb_loopback_latency_open, | |
1097 | .read = seq_read, | |
1098 | .llseek = seq_lseek, | |
1099 | .release = single_release, | |
1100 | }; | |
1101 | ||
5015115d BD |
1102 | static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha, |
1103 | struct list_head *lhb) | |
1104 | { | |
1105 | struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry); | |
1106 | struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry); | |
1107 | struct gb_connection *ca = a->connection; | |
1108 | struct gb_connection *cb = b->connection; | |
1109 | ||
5015115d BD |
1110 | if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id) |
1111 | return -1; | |
1112 | if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id) | |
1113 | return 1; | |
1114 | if (ca->bundle->id < cb->bundle->id) | |
1115 | return -1; | |
1116 | if (cb->bundle->id < ca->bundle->id) | |
1117 | return 1; | |
1118 | if (ca->intf_cport_id < cb->intf_cport_id) | |
1119 | return -1; | |
1120 | else if (cb->intf_cport_id < ca->intf_cport_id) | |
1121 | return 1; | |
1122 | ||
1123 | return 0; | |
1124 | } | |
1125 | ||
1126 | static void gb_loopback_insert_id(struct gb_loopback *gb) | |
1127 | { | |
1128 | struct gb_loopback *gb_list; | |
1129 | u32 new_lbid = 0; | |
1130 | ||
1131 | /* perform an insertion sort */ | |
1132 | list_add_tail(&gb->entry, &gb_dev.list); | |
1133 | list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare); | |
1134 | list_for_each_entry(gb_list, &gb_dev.list, entry) { | |
1135 | gb_list->lbid = 1 << new_lbid; | |
1136 | new_lbid++; | |
1137 | } | |
1138 | } | |
1139 | ||
67d1eece | 1140 | #define DEBUGFS_NAMELEN 32 |
aa27bf82 | 1141 | |
e82a11dc VK |
1142 | static int gb_loopback_probe(struct gb_bundle *bundle, |
1143 | const struct greybus_bundle_id *id) | |
355a7058 | 1144 | { |
e82a11dc VK |
1145 | struct greybus_descriptor_cport *cport_desc; |
1146 | struct gb_connection *connection; | |
355a7058 | 1147 | struct gb_loopback *gb; |
079fa32b | 1148 | struct device *dev; |
355a7058 | 1149 | int retval; |
aa27bf82 | 1150 | char name[DEBUGFS_NAMELEN]; |
2e238d71 | 1151 | unsigned long flags; |
355a7058 | 1152 | |
e82a11dc VK |
1153 | if (bundle->num_cports != 1) |
1154 | return -ENODEV; | |
1155 | ||
1156 | cport_desc = &bundle->cport_desc[0]; | |
1157 | if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK) | |
1158 | return -ENODEV; | |
1159 | ||
355a7058 AB |
1160 | gb = kzalloc(sizeof(*gb), GFP_KERNEL); |
1161 | if (!gb) | |
1162 | return -ENOMEM; | |
1163 | ||
e82a11dc VK |
1164 | connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), |
1165 | gb_loopback_request_handler); | |
1166 | if (IS_ERR(connection)) { | |
1167 | retval = PTR_ERR(connection); | |
1168 | goto out_kzalloc; | |
1169 | } | |
1170 | ||
1171 | gb->connection = connection; | |
1172 | greybus_set_drvdata(bundle, gb); | |
1173 | ||
8e1d6c33 | 1174 | init_waitqueue_head(&gb->wq); |
36f241ff BD |
1175 | init_waitqueue_head(&gb->wq_completion); |
1176 | atomic_set(&gb->outstanding_operations, 0); | |
8e1d6c33 BD |
1177 | gb_loopback_reset_stats(gb); |
1178 | ||
12927835 BD |
1179 | /* Reported values to user-space for min/max timeouts */ |
1180 | gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN); | |
1181 | gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX); | |
1182 | ||
f06272b2 | 1183 | if (!gb_dev.count) { |
f06272b2 BD |
1184 | /* Calculate maximum payload */ |
1185 | gb_dev.size_max = gb_operation_get_payload_size_max(connection); | |
1186 | if (gb_dev.size_max <= | |
1187 | sizeof(struct gb_loopback_transfer_request)) { | |
1188 | retval = -EINVAL; | |
e82a11dc | 1189 | goto out_connection_destroy; |
f06272b2 BD |
1190 | } |
1191 | gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request); | |
1192 | } | |
f06272b2 BD |
1193 | |
1194 | /* Create per-connection sysfs and debugfs data-points */ | |
8d8d36da | 1195 | snprintf(name, sizeof(name), "raw_latency_%s", |
d9a9ea1b | 1196 | dev_name(&connection->bundle->dev)); |
563a8412 | 1197 | gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb, |
aa27bf82 | 1198 | &gb_loopback_debugfs_latency_ops); |
079fa32b AH |
1199 | |
1200 | gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL); | |
1201 | if (gb->id < 0) { | |
1202 | retval = gb->id; | |
e82a11dc | 1203 | goto out_debugfs_remove; |
079fa32b AH |
1204 | } |
1205 | ||
e82a11dc VK |
1206 | retval = gb_connection_enable(connection); |
1207 | if (retval) | |
1208 | goto out_ida_remove; | |
1209 | ||
079fa32b AH |
1210 | dev = device_create_with_groups(&loopback_class, |
1211 | &connection->bundle->dev, | |
1212 | MKDEV(0, 0), gb, loopback_groups, | |
1213 | "gb_loopback%d", gb->id); | |
1214 | if (IS_ERR(dev)) { | |
1215 | retval = PTR_ERR(dev); | |
e82a11dc | 1216 | goto out_connection_disable; |
079fa32b AH |
1217 | } |
1218 | gb->dev = dev; | |
c3bba87a | 1219 | |
cbd204b4 | 1220 | /* Allocate kfifo */ |
4b0ea00c | 1221 | if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32), |
cbd204b4 BD |
1222 | GFP_KERNEL)) { |
1223 | retval = -ENOMEM; | |
079fa32b | 1224 | goto out_conn; |
cbd204b4 | 1225 | } |
cbd204b4 | 1226 | /* Fork worker thread */ |
85d678c0 | 1227 | mutex_init(&gb->mutex); |
355a7058 AB |
1228 | gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); |
1229 | if (IS_ERR(gb->task)) { | |
69f60347 | 1230 | retval = PTR_ERR(gb->task); |
8563a49c | 1231 | goto out_kfifo; |
355a7058 AB |
1232 | } |
1233 | ||
2e238d71 | 1234 | spin_lock_irqsave(&gb_dev.lock, flags); |
5015115d | 1235 | gb_loopback_insert_id(gb); |
aa27bf82 | 1236 | gb_dev.count++; |
2e238d71 BD |
1237 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1238 | ||
1239 | gb_connection_latency_tag_enable(connection); | |
e854ff58 AH |
1240 | |
1241 | gb_pm_runtime_put_autosuspend(bundle); | |
1242 | ||
355a7058 AB |
1243 | return 0; |
1244 | ||
8563a49c | 1245 | out_kfifo: |
4b0ea00c | 1246 | kfifo_free(&gb->kfifo_lat); |
079fa32b AH |
1247 | out_conn: |
1248 | device_unregister(dev); | |
e82a11dc VK |
1249 | out_connection_disable: |
1250 | gb_connection_disable(connection); | |
1251 | out_ida_remove: | |
079fa32b | 1252 | ida_simple_remove(&loopback_ida, gb->id); |
e82a11dc | 1253 | out_debugfs_remove: |
aa27bf82 | 1254 | debugfs_remove(gb->file); |
e82a11dc VK |
1255 | out_connection_destroy: |
1256 | gb_connection_destroy(connection); | |
2e238d71 | 1257 | out_kzalloc: |
355a7058 | 1258 | kfree(gb); |
1fb807cf | 1259 | |
355a7058 AB |
1260 | return retval; |
1261 | } | |
1262 | ||
e82a11dc | 1263 | static void gb_loopback_disconnect(struct gb_bundle *bundle) |
355a7058 | 1264 | { |
e82a11dc | 1265 | struct gb_loopback *gb = greybus_get_drvdata(bundle); |
2e238d71 | 1266 | unsigned long flags; |
e854ff58 AH |
1267 | int ret; |
1268 | ||
1269 | ret = gb_pm_runtime_get_sync(bundle); | |
1270 | if (ret) | |
1271 | gb_pm_runtime_get_noresume(bundle); | |
355a7058 | 1272 | |
e82a11dc VK |
1273 | gb_connection_disable(gb->connection); |
1274 | ||
355a7058 AB |
1275 | if (!IS_ERR_OR_NULL(gb->task)) |
1276 | kthread_stop(gb->task); | |
cbd204b4 | 1277 | |
4b0ea00c | 1278 | kfifo_free(&gb->kfifo_lat); |
e82a11dc | 1279 | gb_connection_latency_tag_disable(gb->connection); |
aa27bf82 | 1280 | debugfs_remove(gb->file); |
e82a11dc VK |
1281 | |
1282 | /* | |
1283 | * FIXME: gb_loopback_async_wait_all() is redundant now, as connection | |
1284 | * is disabled at the beginning and so we can't have any more | |
1285 | * incoming/outgoing requests. | |
1286 | */ | |
36f241ff | 1287 | gb_loopback_async_wait_all(gb); |
2e238d71 BD |
1288 | |
1289 | spin_lock_irqsave(&gb_dev.lock, flags); | |
1290 | gb_dev.count--; | |
ff477d07 | 1291 | list_del(&gb->entry); |
2e238d71 BD |
1292 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1293 | ||
079fa32b AH |
1294 | device_unregister(gb->dev); |
1295 | ida_simple_remove(&loopback_ida, gb->id); | |
1296 | ||
e82a11dc | 1297 | gb_connection_destroy(gb->connection); |
5f3e0d17 | 1298 | kfree(gb); |
355a7058 AB |
1299 | } |
1300 | ||
e82a11dc VK |
1301 | static const struct greybus_bundle_id gb_loopback_id_table[] = { |
1302 | { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) }, | |
1303 | { } | |
1304 | }; | |
1305 | MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table); | |
1306 | ||
1307 | static struct greybus_driver gb_loopback_driver = { | |
1308 | .name = "loopback", | |
1309 | .probe = gb_loopback_probe, | |
1310 | .disconnect = gb_loopback_disconnect, | |
1311 | .id_table = gb_loopback_id_table, | |
355a7058 AB |
1312 | }; |
1313 | ||
cbd204b4 BD |
1314 | static int loopback_init(void) |
1315 | { | |
4b0ea00c BD |
1316 | int retval; |
1317 | ||
67d1eece | 1318 | INIT_LIST_HEAD(&gb_dev.list); |
12927835 | 1319 | INIT_LIST_HEAD(&gb_dev.list_op_async); |
2e238d71 | 1320 | spin_lock_init(&gb_dev.lock); |
aa27bf82 | 1321 | gb_dev.root = debugfs_create_dir("gb_loopback", NULL); |
67d1eece | 1322 | |
079fa32b AH |
1323 | retval = class_register(&loopback_class); |
1324 | if (retval) | |
1325 | goto err; | |
1326 | ||
e82a11dc | 1327 | retval = greybus_register(&gb_loopback_driver); |
079fa32b AH |
1328 | if (retval) |
1329 | goto err_unregister; | |
1330 | ||
1331 | return 0; | |
4b0ea00c | 1332 | |
079fa32b AH |
1333 | err_unregister: |
1334 | class_unregister(&loopback_class); | |
1335 | err: | |
4b0ea00c BD |
1336 | debugfs_remove_recursive(gb_dev.root); |
1337 | return retval; | |
cbd204b4 BD |
1338 | } |
1339 | module_init(loopback_init); | |
1340 | ||
1341 | static void __exit loopback_exit(void) | |
1342 | { | |
aa27bf82 | 1343 | debugfs_remove_recursive(gb_dev.root); |
e82a11dc | 1344 | greybus_deregister(&gb_loopback_driver); |
079fa32b AH |
1345 | class_unregister(&loopback_class); |
1346 | ida_destroy(&loopback_ida); | |
cbd204b4 BD |
1347 | } |
1348 | module_exit(loopback_exit); | |
355a7058 AB |
1349 | |
1350 | MODULE_LICENSE("GPL v2"); |