Commit | Line | Data |
---|---|---|
355a7058 AB |
1 | /* |
2 | * Loopback bridge driver for the Greybus loopback module. | |
3 | * | |
4 | * Copyright 2014 Google Inc. | |
5 | * Copyright 2014 Linaro Ltd. | |
6 | * | |
7 | * Released under the GPLv2 only. | |
8 | */ | |
8923c5b5 JH |
9 | |
10 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
11 | ||
355a7058 AB |
12 | #include <linux/kernel.h> |
13 | #include <linux/module.h> | |
85d678c0 | 14 | #include <linux/mutex.h> |
355a7058 AB |
15 | #include <linux/slab.h> |
16 | #include <linux/kthread.h> | |
17 | #include <linux/delay.h> | |
18 | #include <linux/random.h> | |
5679f783 | 19 | #include <linux/sizes.h> |
cbd204b4 BD |
20 | #include <linux/cdev.h> |
21 | #include <linux/fs.h> | |
22 | #include <linux/kfifo.h> | |
aa27bf82 | 23 | #include <linux/debugfs.h> |
5015115d | 24 | #include <linux/list_sort.h> |
2e238d71 | 25 | #include <linux/spinlock.h> |
12927835 | 26 | #include <linux/workqueue.h> |
36f241ff | 27 | #include <linux/atomic.h> |
cbd204b4 | 28 | |
1ffc12be JH |
29 | #include <asm/div64.h> |
30 | ||
355a7058 | 31 | #include "greybus.h" |
fd58926e | 32 | #include "connection.h" |
355a7058 | 33 | |
fd489e1a BD |
34 | #define NSEC_PER_DAY 86400000000000ULL |
35 | ||
355a7058 | 36 | struct gb_loopback_stats { |
e051c82b AE |
37 | u32 min; |
38 | u32 max; | |
00af6583 | 39 | u64 sum; |
e13fc28f | 40 | u32 count; |
355a7058 AB |
41 | }; |
42 | ||
aa27bf82 BD |
43 | struct gb_loopback_device { |
44 | struct dentry *root; | |
45 | u32 count; | |
8e1d6c33 | 46 | size_t size_max; |
67d1eece | 47 | |
2e238d71 BD |
48 | /* We need to take a lock in atomic context */ |
49 | spinlock_t lock; | |
67d1eece | 50 | struct list_head list; |
12927835 BD |
51 | struct list_head list_op_async; |
52 | wait_queue_head_t wq; | |
aa27bf82 BD |
53 | }; |
54 | ||
55 | static struct gb_loopback_device gb_dev; | |
56 | ||
12927835 BD |
57 | struct gb_loopback_async_operation { |
58 | struct gb_loopback *gb; | |
59 | struct gb_operation *operation; | |
60 | struct timeval ts; | |
61 | struct timer_list timer; | |
62 | struct list_head entry; | |
63 | struct work_struct work; | |
64 | struct kref kref; | |
65 | bool pending; | |
66 | int (*completion)(struct gb_loopback_async_operation *op_async); | |
67 | }; | |
68 | ||
355a7058 AB |
69 | struct gb_loopback { |
70 | struct gb_connection *connection; | |
355a7058 | 71 | |
aa27bf82 | 72 | struct dentry *file; |
4b0ea00c BD |
73 | struct kfifo kfifo_lat; |
74 | struct kfifo kfifo_ts; | |
85d678c0 | 75 | struct mutex mutex; |
355a7058 | 76 | struct task_struct *task; |
67d1eece | 77 | struct list_head entry; |
079fa32b | 78 | struct device *dev; |
8e1d6c33 | 79 | wait_queue_head_t wq; |
36f241ff BD |
80 | wait_queue_head_t wq_completion; |
81 | atomic_t outstanding_operations; | |
355a7058 | 82 | |
67d1eece | 83 | /* Per connection stats */ |
355a7058 AB |
84 | struct gb_loopback_stats latency; |
85 | struct gb_loopback_stats throughput; | |
583cbf50 | 86 | struct gb_loopback_stats requests_per_second; |
1ec5843e BD |
87 | struct gb_loopback_stats apbridge_unipro_latency; |
88 | struct gb_loopback_stats gpbridge_firmware_latency; | |
67d1eece | 89 | |
8e1d6c33 | 90 | int type; |
12927835 | 91 | int async; |
079fa32b | 92 | int id; |
8e1d6c33 BD |
93 | u32 size; |
94 | u32 iteration_max; | |
67d1eece | 95 | u32 iteration_count; |
b36f04fa | 96 | int us_wait; |
355a7058 | 97 | u32 error; |
12927835 BD |
98 | u32 requests_completed; |
99 | u32 requests_timedout; | |
100 | u32 timeout; | |
101 | u32 jiffy_timeout; | |
102 | u32 timeout_min; | |
103 | u32 timeout_max; | |
8e3fba55 | 104 | u32 outstanding_operations_max; |
8e1d6c33 BD |
105 | u32 lbid; |
106 | u64 elapsed_nsecs; | |
e6227ee6 BD |
107 | u32 apbridge_latency_ts; |
108 | u32 gpbridge_latency_ts; | |
355a7058 AB |
109 | }; |
110 | ||
079fa32b AH |
111 | static struct class loopback_class = { |
112 | .name = "gb_loopback", | |
113 | .owner = THIS_MODULE, | |
114 | }; | |
115 | static DEFINE_IDA(loopback_ida); | |
116 | ||
12927835 BD |
117 | /* Min/max values in jiffies */ |
118 | #define GB_LOOPBACK_TIMEOUT_MIN 1 | |
119 | #define GB_LOOPBACK_TIMEOUT_MAX 10000 | |
120 | ||
cbd204b4 BD |
121 | #define GB_LOOPBACK_FIFO_DEFAULT 8192 |
122 | ||
cbd204b4 BD |
123 | static unsigned kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT; |
124 | module_param(kfifo_depth, uint, 0444); | |
125 | ||
cbd204b4 BD |
126 | /* Maximum size of any one send data buffer we support */ |
127 | #define MAX_PACKET_SIZE (PAGE_SIZE * 2) | |
128 | ||
b36f04fa | 129 | #define GB_LOOPBACK_US_WAIT_MAX 1000000 |
355a7058 | 130 | |
355a7058 | 131 | /* interface sysfs attributes */ |
8e1d6c33 BD |
132 | #define gb_loopback_ro_attr(field) \ |
133 | static ssize_t field##_show(struct device *dev, \ | |
355a7058 AB |
134 | struct device_attribute *attr, \ |
135 | char *buf) \ | |
136 | { \ | |
079fa32b | 137 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 138 | return sprintf(buf, "%u\n", gb->field); \ |
355a7058 | 139 | } \ |
8e1d6c33 | 140 | static DEVICE_ATTR_RO(field) |
355a7058 | 141 | |
8e1d6c33 BD |
142 | #define gb_loopback_ro_stats_attr(name, field, type) \ |
143 | static ssize_t name##_##field##_show(struct device *dev, \ | |
355a7058 AB |
144 | struct device_attribute *attr, \ |
145 | char *buf) \ | |
146 | { \ | |
079fa32b | 147 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 148 | return sprintf(buf, "%"#type"\n", gb->name.field); \ |
355a7058 | 149 | } \ |
8e1d6c33 | 150 | static DEVICE_ATTR_RO(name##_##field) |
355a7058 | 151 | |
8e1d6c33 BD |
152 | #define gb_loopback_ro_avg_attr(name) \ |
153 | static ssize_t name##_avg_show(struct device *dev, \ | |
7a135a96 AE |
154 | struct device_attribute *attr, \ |
155 | char *buf) \ | |
156 | { \ | |
f06272b2 | 157 | struct gb_loopback_stats *stats; \ |
f06272b2 BD |
158 | struct gb_loopback *gb; \ |
159 | u64 avg; \ | |
160 | u32 count, rem; \ | |
079fa32b | 161 | gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 162 | stats = &gb->name; \ |
f06272b2 BD |
163 | count = stats->count ? stats->count : 1; \ |
164 | avg = stats->sum + count / 2; /* round closest */ \ | |
165 | rem = do_div(avg, count); \ | |
ff71d395 | 166 | return sprintf(buf, "%llu.%06u\n", avg, 1000000 * rem / count); \ |
7a135a96 | 167 | } \ |
8e1d6c33 | 168 | static DEVICE_ATTR_RO(name##_avg) |
7a135a96 | 169 | |
8e1d6c33 BD |
170 | #define gb_loopback_stats_attrs(field) \ |
171 | gb_loopback_ro_stats_attr(field, min, u); \ | |
172 | gb_loopback_ro_stats_attr(field, max, u); \ | |
173 | gb_loopback_ro_avg_attr(field) | |
355a7058 AB |
174 | |
175 | #define gb_loopback_attr(field, type) \ | |
176 | static ssize_t field##_show(struct device *dev, \ | |
177 | struct device_attribute *attr, \ | |
178 | char *buf) \ | |
179 | { \ | |
079fa32b | 180 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
355a7058 AB |
181 | return sprintf(buf, "%"#type"\n", gb->field); \ |
182 | } \ | |
183 | static ssize_t field##_store(struct device *dev, \ | |
184 | struct device_attribute *attr, \ | |
185 | const char *buf, \ | |
186 | size_t len) \ | |
187 | { \ | |
188 | int ret; \ | |
079fa32b | 189 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 190 | mutex_lock(&gb->mutex); \ |
355a7058 | 191 | ret = sscanf(buf, "%"#type, &gb->field); \ |
355a7058 | 192 | if (ret != 1) \ |
85d678c0 BD |
193 | len = -EINVAL; \ |
194 | else \ | |
8e1d6c33 BD |
195 | gb_loopback_check_attr(gb, bundle); \ |
196 | mutex_unlock(&gb->mutex); \ | |
355a7058 AB |
197 | return len; \ |
198 | } \ | |
199 | static DEVICE_ATTR_RW(field) | |
200 | ||
f06272b2 BD |
201 | #define gb_dev_loopback_ro_attr(field, conn) \ |
202 | static ssize_t field##_show(struct device *dev, \ | |
67d1eece BD |
203 | struct device_attribute *attr, \ |
204 | char *buf) \ | |
205 | { \ | |
079fa32b | 206 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 207 | return sprintf(buf, "%u\n", gb->field); \ |
67d1eece BD |
208 | } \ |
209 | static DEVICE_ATTR_RO(field) | |
210 | ||
211 | #define gb_dev_loopback_rw_attr(field, type) \ | |
212 | static ssize_t field##_show(struct device *dev, \ | |
213 | struct device_attribute *attr, \ | |
214 | char *buf) \ | |
215 | { \ | |
079fa32b | 216 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 | 217 | return sprintf(buf, "%"#type"\n", gb->field); \ |
67d1eece BD |
218 | } \ |
219 | static ssize_t field##_store(struct device *dev, \ | |
220 | struct device_attribute *attr, \ | |
221 | const char *buf, \ | |
222 | size_t len) \ | |
223 | { \ | |
224 | int ret; \ | |
079fa32b | 225 | struct gb_loopback *gb = dev_get_drvdata(dev); \ |
8e1d6c33 BD |
226 | mutex_lock(&gb->mutex); \ |
227 | ret = sscanf(buf, "%"#type, &gb->field); \ | |
67d1eece BD |
228 | if (ret != 1) \ |
229 | len = -EINVAL; \ | |
230 | else \ | |
079fa32b | 231 | gb_loopback_check_attr(gb); \ |
8e1d6c33 | 232 | mutex_unlock(&gb->mutex); \ |
67d1eece BD |
233 | return len; \ |
234 | } \ | |
235 | static DEVICE_ATTR_RW(field) | |
236 | ||
8e1d6c33 | 237 | static void gb_loopback_reset_stats(struct gb_loopback *gb); |
079fa32b | 238 | static void gb_loopback_check_attr(struct gb_loopback *gb) |
355a7058 | 239 | { |
b36f04fa BD |
240 | if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX) |
241 | gb->us_wait = GB_LOOPBACK_US_WAIT_MAX; | |
8e1d6c33 BD |
242 | if (gb->size > gb_dev.size_max) |
243 | gb->size = gb_dev.size_max; | |
12927835 BD |
244 | gb->requests_timedout = 0; |
245 | gb->requests_completed = 0; | |
8e1d6c33 BD |
246 | gb->iteration_count = 0; |
247 | gb->error = 0; | |
248 | ||
249 | if (kfifo_depth < gb->iteration_max) { | |
079fa32b | 250 | dev_warn(gb->dev, |
8e1d6c33 BD |
251 | "cannot log bytes %u kfifo_depth %u\n", |
252 | gb->iteration_max, kfifo_depth); | |
cb60f496 | 253 | } |
8e1d6c33 BD |
254 | kfifo_reset_out(&gb->kfifo_lat); |
255 | kfifo_reset_out(&gb->kfifo_ts); | |
cb60f496 | 256 | |
8e1d6c33 | 257 | switch (gb->type) { |
a598f438 BD |
258 | case GB_LOOPBACK_TYPE_PING: |
259 | case GB_LOOPBACK_TYPE_TRANSFER: | |
384a7a3c | 260 | case GB_LOOPBACK_TYPE_SINK: |
12927835 BD |
261 | gb->jiffy_timeout = usecs_to_jiffies(gb->timeout); |
262 | if (!gb->jiffy_timeout) | |
263 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN; | |
264 | else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX) | |
265 | gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX; | |
8e1d6c33 BD |
266 | gb_loopback_reset_stats(gb); |
267 | wake_up(&gb->wq); | |
a598f438 BD |
268 | break; |
269 | default: | |
8e1d6c33 | 270 | gb->type = 0; |
a598f438 BD |
271 | break; |
272 | } | |
355a7058 AB |
273 | } |
274 | ||
275 | /* Time to send and receive one message */ | |
8e1d6c33 | 276 | gb_loopback_stats_attrs(latency); |
583cbf50 | 277 | /* Number of requests sent per second on this cport */ |
8e1d6c33 | 278 | gb_loopback_stats_attrs(requests_per_second); |
355a7058 | 279 | /* Quantity of data sent and received on this cport */ |
8e1d6c33 | 280 | gb_loopback_stats_attrs(throughput); |
1ec5843e | 281 | /* Latency across the UniPro link from APBridge's perspective */ |
8e1d6c33 | 282 | gb_loopback_stats_attrs(apbridge_unipro_latency); |
1ec5843e | 283 | /* Firmware induced overhead in the GPBridge */ |
8e1d6c33 BD |
284 | gb_loopback_stats_attrs(gpbridge_firmware_latency); |
285 | ||
e140c75e | 286 | /* Number of errors encountered during loop */ |
8e1d6c33 | 287 | gb_loopback_ro_attr(error); |
12927835 BD |
288 | /* Number of requests successfully completed async */ |
289 | gb_loopback_ro_attr(requests_completed); | |
290 | /* Number of requests timed out async */ | |
291 | gb_loopback_ro_attr(requests_timedout); | |
292 | /* Timeout minimum in useconds */ | |
293 | gb_loopback_ro_attr(timeout_min); | |
294 | /* Timeout minimum in useconds */ | |
295 | gb_loopback_ro_attr(timeout_max); | |
355a7058 AB |
296 | |
297 | /* | |
799a3f03 | 298 | * Type of loopback message to send based on protocol type definitions |
355a7058 | 299 | * 0 => Don't send message |
799a3f03 | 300 | * 2 => Send ping message continuously (message without payload) |
006335a0 | 301 | * 3 => Send transfer message continuously (message with payload, |
799a3f03 BD |
302 | * payload returned in response) |
303 | * 4 => Send a sink message (message with payload, no payload in response) | |
355a7058 | 304 | */ |
67d1eece | 305 | gb_dev_loopback_rw_attr(type, d); |
355a7058 | 306 | /* Size of transfer message payload: 0-4096 bytes */ |
67d1eece | 307 | gb_dev_loopback_rw_attr(size, u); |
48f19ee8 | 308 | /* Time to wait between two messages: 0-1000 ms */ |
b36f04fa | 309 | gb_dev_loopback_rw_attr(us_wait, d); |
00af6583 | 310 | /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */ |
67d1eece BD |
311 | gb_dev_loopback_rw_attr(iteration_max, u); |
312 | /* The current index of the for (i = 0; i < iteration_max; i++) loop */ | |
f06272b2 | 313 | gb_dev_loopback_ro_attr(iteration_count, false); |
12927835 BD |
314 | /* A flag to indicate synchronous or asynchronous operations */ |
315 | gb_dev_loopback_rw_attr(async, u); | |
316 | /* Timeout of an individual asynchronous request */ | |
317 | gb_dev_loopback_rw_attr(timeout, u); | |
8e3fba55 BD |
318 | /* Maximum number of in-flight operations before back-off */ |
319 | gb_dev_loopback_rw_attr(outstanding_operations_max, u); | |
355a7058 | 320 | |
8e1d6c33 BD |
321 | static struct attribute *loopback_attrs[] = { |
322 | &dev_attr_latency_min.attr, | |
323 | &dev_attr_latency_max.attr, | |
324 | &dev_attr_latency_avg.attr, | |
325 | &dev_attr_requests_per_second_min.attr, | |
326 | &dev_attr_requests_per_second_max.attr, | |
327 | &dev_attr_requests_per_second_avg.attr, | |
328 | &dev_attr_throughput_min.attr, | |
329 | &dev_attr_throughput_max.attr, | |
330 | &dev_attr_throughput_avg.attr, | |
331 | &dev_attr_apbridge_unipro_latency_min.attr, | |
332 | &dev_attr_apbridge_unipro_latency_max.attr, | |
333 | &dev_attr_apbridge_unipro_latency_avg.attr, | |
334 | &dev_attr_gpbridge_firmware_latency_min.attr, | |
335 | &dev_attr_gpbridge_firmware_latency_max.attr, | |
336 | &dev_attr_gpbridge_firmware_latency_avg.attr, | |
355a7058 AB |
337 | &dev_attr_type.attr, |
338 | &dev_attr_size.attr, | |
b36f04fa | 339 | &dev_attr_us_wait.attr, |
00af6583 BD |
340 | &dev_attr_iteration_count.attr, |
341 | &dev_attr_iteration_max.attr, | |
12927835 | 342 | &dev_attr_async.attr, |
8e1d6c33 | 343 | &dev_attr_error.attr, |
12927835 BD |
344 | &dev_attr_requests_completed.attr, |
345 | &dev_attr_requests_timedout.attr, | |
346 | &dev_attr_timeout.attr, | |
8e3fba55 | 347 | &dev_attr_outstanding_operations_max.attr, |
12927835 BD |
348 | &dev_attr_timeout_min.attr, |
349 | &dev_attr_timeout_max.attr, | |
355a7058 AB |
350 | NULL, |
351 | }; | |
8e1d6c33 | 352 | ATTRIBUTE_GROUPS(loopback); |
355a7058 | 353 | |
12927835 BD |
354 | static void gb_loopback_calculate_stats(struct gb_loopback *gb); |
355 | ||
bd416103 BD |
356 | static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs) |
357 | { | |
358 | u32 lat; | |
359 | ||
360 | do_div(elapsed_nsecs, NSEC_PER_USEC); | |
361 | lat = elapsed_nsecs; | |
362 | return lat; | |
363 | } | |
364 | ||
7c985351 BD |
365 | static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) |
366 | { | |
367 | if (t2 > t1) | |
368 | return t2 - t1; | |
369 | else | |
370 | return NSEC_PER_DAY - t2 + t1; | |
371 | } | |
372 | ||
2f842304 | 373 | static u64 gb_loopback_calc_latency(struct timeval *ts, struct timeval *te) |
4c192665 BD |
374 | { |
375 | u64 t1, t2; | |
376 | ||
377 | t1 = timeval_to_ns(ts); | |
378 | t2 = timeval_to_ns(te); | |
7c985351 BD |
379 | |
380 | return __gb_loopback_calc_latency(t1, t2); | |
4c192665 BD |
381 | } |
382 | ||
4b0ea00c BD |
383 | static void gb_loopback_push_latency_ts(struct gb_loopback *gb, |
384 | struct timeval *ts, struct timeval *te) | |
385 | { | |
386 | kfifo_in(&gb->kfifo_ts, (unsigned char *)ts, sizeof(*ts)); | |
387 | kfifo_in(&gb->kfifo_ts, (unsigned char *)te, sizeof(*te)); | |
388 | } | |
389 | ||
fbb8edba BD |
390 | static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, |
391 | void *request, int request_size, | |
392 | void *response, int response_size) | |
384a7a3c | 393 | { |
fbb8edba | 394 | struct gb_operation *operation; |
384a7a3c | 395 | struct timeval ts, te; |
fbb8edba | 396 | int ret; |
384a7a3c BD |
397 | |
398 | do_gettimeofday(&ts); | |
fbb8edba BD |
399 | operation = gb_operation_create(gb->connection, type, request_size, |
400 | response_size, GFP_KERNEL); | |
d9048d8c AB |
401 | if (!operation) |
402 | return -ENOMEM; | |
c3bba87a | 403 | |
fbb8edba BD |
404 | if (request_size) |
405 | memcpy(operation->request->payload, request, request_size); | |
406 | ||
407 | ret = gb_operation_request_send_sync(operation); | |
408 | if (ret) { | |
d9a9ea1b | 409 | dev_err(&gb->connection->bundle->dev, |
fbb8edba | 410 | "synchronous operation failed: %d\n", ret); |
d9048d8c | 411 | goto out_put_operation; |
fbb8edba BD |
412 | } else { |
413 | if (response_size == operation->response->payload_size) { | |
414 | memcpy(response, operation->response->payload, | |
415 | response_size); | |
416 | } else { | |
d9a9ea1b | 417 | dev_err(&gb->connection->bundle->dev, |
fbb8edba BD |
418 | "response size %zu expected %d\n", |
419 | operation->response->payload_size, | |
420 | response_size); | |
d9048d8c AB |
421 | ret = -EINVAL; |
422 | goto out_put_operation; | |
fbb8edba BD |
423 | } |
424 | } | |
6ab1ce4d | 425 | |
384a7a3c | 426 | do_gettimeofday(&te); |
2f842304 BD |
427 | |
428 | /* Calculate the total time the message took */ | |
4b0ea00c | 429 | gb_loopback_push_latency_ts(gb, &ts, &te); |
2f842304 BD |
430 | gb->elapsed_nsecs = gb_loopback_calc_latency(&ts, &te); |
431 | ||
d9048d8c AB |
432 | out_put_operation: |
433 | gb_operation_put(operation); | |
434 | ||
fbb8edba BD |
435 | return ret; |
436 | } | |
384a7a3c | 437 | |
12927835 BD |
438 | static void __gb_loopback_async_operation_destroy(struct kref *kref) |
439 | { | |
440 | struct gb_loopback_async_operation *op_async; | |
441 | ||
442 | op_async = container_of(kref, struct gb_loopback_async_operation, kref); | |
443 | ||
444 | list_del(&op_async->entry); | |
445 | if (op_async->operation) | |
446 | gb_operation_put(op_async->operation); | |
36f241ff BD |
447 | atomic_dec(&op_async->gb->outstanding_operations); |
448 | wake_up(&op_async->gb->wq_completion); | |
12927835 BD |
449 | kfree(op_async); |
450 | } | |
451 | ||
452 | static void gb_loopback_async_operation_get(struct gb_loopback_async_operation | |
453 | *op_async) | |
454 | { | |
455 | kref_get(&op_async->kref); | |
456 | } | |
457 | ||
458 | static void gb_loopback_async_operation_put(struct gb_loopback_async_operation | |
459 | *op_async) | |
460 | { | |
461 | unsigned long flags; | |
462 | ||
463 | spin_lock_irqsave(&gb_dev.lock, flags); | |
464 | kref_put(&op_async->kref, __gb_loopback_async_operation_destroy); | |
465 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
466 | } | |
467 | ||
468 | static struct gb_loopback_async_operation * | |
469 | gb_loopback_operation_find(u16 id) | |
470 | { | |
471 | struct gb_loopback_async_operation *op_async; | |
472 | bool found = false; | |
473 | unsigned long flags; | |
474 | ||
475 | spin_lock_irqsave(&gb_dev.lock, flags); | |
476 | list_for_each_entry(op_async, &gb_dev.list_op_async, entry) { | |
477 | if (op_async->operation->id == id) { | |
478 | gb_loopback_async_operation_get(op_async); | |
479 | found = true; | |
480 | break; | |
481 | } | |
482 | } | |
483 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
484 | ||
485 | return found ? op_async : NULL; | |
486 | } | |
487 | ||
36f241ff BD |
488 | static void gb_loopback_async_wait_all(struct gb_loopback *gb) |
489 | { | |
490 | wait_event(gb->wq_completion, | |
491 | !atomic_read(&gb->outstanding_operations)); | |
492 | } | |
493 | ||
12927835 BD |
494 | static void gb_loopback_async_operation_callback(struct gb_operation *operation) |
495 | { | |
496 | struct gb_loopback_async_operation *op_async; | |
497 | struct gb_loopback *gb; | |
498 | struct timeval te; | |
499 | bool err = false; | |
500 | ||
501 | do_gettimeofday(&te); | |
502 | op_async = gb_loopback_operation_find(operation->id); | |
503 | if (!op_async) | |
504 | return; | |
505 | ||
506 | gb = op_async->gb; | |
507 | mutex_lock(&gb->mutex); | |
508 | ||
509 | if (!op_async->pending || gb_operation_result(operation)) { | |
510 | err = true; | |
511 | } else { | |
512 | if (op_async->completion) | |
513 | if (op_async->completion(op_async)) | |
514 | err = true; | |
515 | } | |
516 | ||
517 | if (err) { | |
518 | gb->error++; | |
519 | } else { | |
520 | gb->requests_completed++; | |
521 | gb_loopback_push_latency_ts(gb, &op_async->ts, &te); | |
522 | gb->elapsed_nsecs = gb_loopback_calc_latency(&op_async->ts, | |
523 | &te); | |
2422d366 | 524 | gb_loopback_calculate_stats(gb); |
12927835 BD |
525 | } |
526 | ||
527 | if (op_async->pending) { | |
528 | gb->iteration_count++; | |
529 | op_async->pending = false; | |
530 | del_timer_sync(&op_async->timer); | |
531 | gb_loopback_async_operation_put(op_async); | |
532 | } | |
533 | mutex_unlock(&gb->mutex); | |
534 | ||
535 | dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", | |
536 | operation->id); | |
537 | ||
538 | gb_loopback_async_operation_put(op_async); | |
539 | } | |
540 | ||
541 | static void gb_loopback_async_operation_work(struct work_struct *work) | |
542 | { | |
543 | struct gb_loopback *gb; | |
544 | struct gb_operation *operation; | |
545 | struct gb_loopback_async_operation *op_async; | |
546 | ||
547 | op_async = container_of(work, struct gb_loopback_async_operation, work); | |
12927835 BD |
548 | gb = op_async->gb; |
549 | operation = op_async->operation; | |
550 | ||
551 | mutex_lock(&gb->mutex); | |
552 | if (op_async->pending) { | |
553 | gb->requests_timedout++; | |
554 | gb->error++; | |
555 | gb->iteration_count++; | |
556 | op_async->pending = false; | |
557 | gb_loopback_async_operation_put(op_async); | |
558 | } | |
559 | mutex_unlock(&gb->mutex); | |
560 | ||
561 | dev_dbg(&gb->connection->bundle->dev, "timeout operation %d\n", | |
562 | operation->id); | |
563 | ||
564 | gb_operation_cancel(operation, -ETIMEDOUT); | |
565 | gb_loopback_async_operation_put(op_async); | |
566 | } | |
567 | ||
568 | static void gb_loopback_async_operation_timeout(unsigned long data) | |
569 | { | |
570 | struct gb_loopback_async_operation *op_async; | |
571 | u16 id = data; | |
572 | ||
573 | op_async = gb_loopback_operation_find(id); | |
574 | if (!op_async) { | |
575 | pr_err("operation %d not found - time out ?\n", id); | |
576 | return; | |
577 | } | |
578 | schedule_work(&op_async->work); | |
579 | } | |
580 | ||
581 | static int gb_loopback_async_operation(struct gb_loopback *gb, int type, | |
582 | void *request, int request_size, | |
583 | int response_size, | |
584 | void *completion) | |
585 | { | |
586 | struct gb_loopback_async_operation *op_async; | |
587 | struct gb_operation *operation; | |
588 | int ret; | |
589 | unsigned long flags; | |
590 | ||
591 | op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); | |
592 | if (!op_async) | |
593 | return -ENOMEM; | |
594 | ||
595 | INIT_WORK(&op_async->work, gb_loopback_async_operation_work); | |
596 | init_timer(&op_async->timer); | |
597 | kref_init(&op_async->kref); | |
598 | ||
599 | operation = gb_operation_create(gb->connection, type, request_size, | |
600 | response_size, GFP_KERNEL); | |
601 | if (!operation) { | |
c7aae4e6 BD |
602 | kfree(op_async); |
603 | return -ENOMEM; | |
12927835 BD |
604 | } |
605 | ||
606 | if (request_size) | |
607 | memcpy(operation->request->payload, request, request_size); | |
608 | ||
609 | op_async->gb = gb; | |
610 | op_async->operation = operation; | |
611 | op_async->completion = completion; | |
612 | ||
613 | spin_lock_irqsave(&gb_dev.lock, flags); | |
614 | list_add_tail(&op_async->entry, &gb_dev.list_op_async); | |
615 | spin_unlock_irqrestore(&gb_dev.lock, flags); | |
616 | ||
617 | do_gettimeofday(&op_async->ts); | |
618 | op_async->pending = true; | |
36f241ff | 619 | atomic_inc(&gb->outstanding_operations); |
12927835 BD |
620 | ret = gb_operation_request_send(operation, |
621 | gb_loopback_async_operation_callback, | |
622 | GFP_KERNEL); | |
623 | if (ret) | |
624 | goto error; | |
625 | ||
626 | op_async->timer.function = gb_loopback_async_operation_timeout; | |
627 | op_async->timer.expires = jiffies + gb->jiffy_timeout; | |
628 | op_async->timer.data = (unsigned long)operation->id; | |
629 | add_timer(&op_async->timer); | |
630 | ||
631 | return ret; | |
632 | error: | |
633 | gb_loopback_async_operation_put(op_async); | |
634 | return ret; | |
635 | } | |
636 | ||
637 | static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len) | |
fbb8edba BD |
638 | { |
639 | struct gb_loopback_transfer_request *request; | |
640 | int retval; | |
641 | ||
642 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
643 | if (!request) | |
644 | return -ENOMEM; | |
645 | ||
646 | request->len = cpu_to_le32(len); | |
647 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK, | |
648 | request, len + sizeof(*request), | |
649 | NULL, 0); | |
384a7a3c BD |
650 | kfree(request); |
651 | return retval; | |
652 | } | |
653 | ||
12927835 | 654 | static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len) |
355a7058 | 655 | { |
355a7058 AB |
656 | struct gb_loopback_transfer_request *request; |
657 | struct gb_loopback_transfer_response *response; | |
658 | int retval; | |
659 | ||
d6a1a3b5 BD |
660 | gb->apbridge_latency_ts = 0; |
661 | gb->gpbridge_latency_ts = 0; | |
662 | ||
355a7058 AB |
663 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); |
664 | if (!request) | |
665 | return -ENOMEM; | |
666 | response = kmalloc(len + sizeof(*response), GFP_KERNEL); | |
667 | if (!response) { | |
668 | kfree(request); | |
669 | return -ENOMEM; | |
670 | } | |
671 | ||
dc4a1069 VK |
672 | memset(request->data, 0x5A, len); |
673 | ||
355a7058 | 674 | request->len = cpu_to_le32(len); |
fbb8edba BD |
675 | retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER, |
676 | request, len + sizeof(*request), | |
677 | response, len + sizeof(*response)); | |
355a7058 AB |
678 | if (retval) |
679 | goto gb_error; | |
680 | ||
dc366f8e | 681 | if (memcmp(request->data, response->data, len)) { |
d9a9ea1b GKH |
682 | dev_err(&gb->connection->bundle->dev, |
683 | "Loopback Data doesn't match\n"); | |
355a7058 | 684 | retval = -EREMOTEIO; |
dc366f8e | 685 | } |
1ec5843e BD |
686 | gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0); |
687 | gb->gpbridge_latency_ts = (u32)__le32_to_cpu(response->reserved1); | |
355a7058 AB |
688 | |
689 | gb_error: | |
690 | kfree(request); | |
691 | kfree(response); | |
692 | ||
693 | return retval; | |
694 | } | |
695 | ||
12927835 | 696 | static int gb_loopback_sync_ping(struct gb_loopback *gb) |
355a7058 | 697 | { |
fbb8edba BD |
698 | return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING, |
699 | NULL, 0, NULL, 0); | |
355a7058 AB |
700 | } |
701 | ||
12927835 BD |
702 | static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len) |
703 | { | |
704 | struct gb_loopback_transfer_request *request; | |
705 | int retval; | |
706 | ||
707 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
708 | if (!request) | |
709 | return -ENOMEM; | |
710 | ||
711 | request->len = cpu_to_le32(len); | |
712 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK, | |
713 | request, len + sizeof(*request), | |
714 | 0, NULL); | |
715 | kfree(request); | |
716 | return retval; | |
717 | } | |
718 | ||
719 | static int gb_loopback_async_transfer_complete( | |
720 | struct gb_loopback_async_operation *op_async) | |
721 | { | |
722 | struct gb_loopback *gb; | |
723 | struct gb_operation *operation; | |
724 | struct gb_loopback_transfer_request *request; | |
725 | struct gb_loopback_transfer_response *response; | |
726 | size_t len; | |
727 | int retval = 0; | |
728 | ||
729 | gb = op_async->gb; | |
730 | operation = op_async->operation; | |
731 | request = operation->request->payload; | |
732 | response = operation->response->payload; | |
733 | len = le32_to_cpu(request->len); | |
734 | ||
735 | if (memcmp(request->data, response->data, len)) { | |
736 | dev_err(&gb->connection->bundle->dev, | |
737 | "Loopback Data doesn't match operation id %d\n", | |
738 | operation->id); | |
739 | retval = -EREMOTEIO; | |
740 | } else { | |
741 | gb->apbridge_latency_ts = | |
742 | (u32)__le32_to_cpu(response->reserved0); | |
743 | gb->gpbridge_latency_ts = | |
744 | (u32)__le32_to_cpu(response->reserved1); | |
745 | } | |
746 | ||
747 | return retval; | |
748 | } | |
749 | ||
750 | static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len) | |
751 | { | |
752 | struct gb_loopback_transfer_request *request; | |
753 | int retval, response_len; | |
754 | ||
755 | request = kmalloc(len + sizeof(*request), GFP_KERNEL); | |
756 | if (!request) | |
757 | return -ENOMEM; | |
758 | ||
759 | memset(request->data, 0x5A, len); | |
760 | ||
761 | request->len = cpu_to_le32(len); | |
762 | response_len = sizeof(struct gb_loopback_transfer_response); | |
763 | retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER, | |
764 | request, len + sizeof(*request), | |
765 | len + response_len, | |
766 | gb_loopback_async_transfer_complete); | |
767 | if (retval) | |
768 | goto gb_error; | |
769 | ||
770 | gb_error: | |
771 | kfree(request); | |
772 | return retval; | |
773 | } | |
774 | ||
775 | static int gb_loopback_async_ping(struct gb_loopback *gb) | |
776 | { | |
777 | return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING, | |
778 | NULL, 0, 0, NULL); | |
779 | } | |
780 | ||
e82a11dc | 781 | static int gb_loopback_request_handler(struct gb_operation *operation) |
ac1c2840 AE |
782 | { |
783 | struct gb_connection *connection = operation->connection; | |
784 | struct gb_loopback_transfer_request *request; | |
785 | struct gb_loopback_transfer_response *response; | |
d9a9ea1b | 786 | struct device *dev = &connection->bundle->dev; |
e51eafeb | 787 | size_t len; |
ac1c2840 AE |
788 | |
789 | /* By convention, the AP initiates the version operation */ | |
e82a11dc | 790 | switch (operation->type) { |
0e2462d1 | 791 | case GB_REQUEST_TYPE_PROTOCOL_VERSION: |
d9a9ea1b | 792 | dev_err(dev, "module-initiated version operation\n"); |
ac1c2840 AE |
793 | return -EINVAL; |
794 | case GB_LOOPBACK_TYPE_PING: | |
384a7a3c | 795 | case GB_LOOPBACK_TYPE_SINK: |
ac1c2840 AE |
796 | return 0; |
797 | case GB_LOOPBACK_TYPE_TRANSFER: | |
798 | if (operation->request->payload_size < sizeof(*request)) { | |
d9a9ea1b | 799 | dev_err(dev, "transfer request too small (%zu < %zu)\n", |
ac1c2840 AE |
800 | operation->request->payload_size, |
801 | sizeof(*request)); | |
802 | return -EINVAL; /* -EMSGSIZE */ | |
803 | } | |
804 | request = operation->request->payload; | |
805 | len = le32_to_cpu(request->len); | |
67d1eece | 806 | if (len > gb_dev.size_max) { |
d9a9ea1b | 807 | dev_err(dev, "transfer request too large (%zu > %zu)\n", |
67d1eece | 808 | len, gb_dev.size_max); |
c3bba87a BD |
809 | return -EINVAL; |
810 | } | |
811 | ||
81ad6994 BG |
812 | if (!gb_operation_response_alloc(operation, |
813 | len + sizeof(*response), GFP_KERNEL)) { | |
814 | dev_err(dev, "error allocating response\n"); | |
815 | return -ENOMEM; | |
ac1c2840 | 816 | } |
81ad6994 BG |
817 | response = operation->response->payload; |
818 | response->len = cpu_to_le32(len); | |
819 | if (len) | |
820 | memcpy(response->data, request->data, len); | |
821 | ||
ac1c2840 AE |
822 | return 0; |
823 | default: | |
e82a11dc | 824 | dev_err(dev, "unsupported request: %u\n", operation->type); |
ac1c2840 AE |
825 | return -EINVAL; |
826 | } | |
827 | } | |
828 | ||
8e1d6c33 | 829 | static void gb_loopback_reset_stats(struct gb_loopback *gb) |
355a7058 AB |
830 | { |
831 | struct gb_loopback_stats reset = { | |
e051c82b | 832 | .min = U32_MAX, |
355a7058 | 833 | }; |
67d1eece BD |
834 | |
835 | /* Reset per-connection stats */ | |
8e1d6c33 BD |
836 | memcpy(&gb->latency, &reset, |
837 | sizeof(struct gb_loopback_stats)); | |
838 | memcpy(&gb->throughput, &reset, | |
839 | sizeof(struct gb_loopback_stats)); | |
840 | memcpy(&gb->requests_per_second, &reset, | |
841 | sizeof(struct gb_loopback_stats)); | |
842 | memcpy(&gb->apbridge_unipro_latency, &reset, | |
843 | sizeof(struct gb_loopback_stats)); | |
844 | memcpy(&gb->gpbridge_firmware_latency, &reset, | |
845 | sizeof(struct gb_loopback_stats)); | |
67d1eece | 846 | |
f42a6891 BD |
847 | /* Should be initialized at least once per transaction set */ |
848 | gb->apbridge_latency_ts = 0; | |
849 | gb->gpbridge_latency_ts = 0; | |
355a7058 AB |
850 | } |
851 | ||
a6e7e535 | 852 | static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) |
355a7058 | 853 | { |
00af6583 BD |
854 | if (stats->min > val) |
855 | stats->min = val; | |
856 | if (stats->max < val) | |
857 | stats->max = val; | |
858 | stats->sum += val; | |
859 | stats->count++; | |
355a7058 AB |
860 | } |
861 | ||
583cbf50 | 862 | static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 863 | { |
2422d366 | 864 | u32 req = USEC_PER_SEC; |
00af6583 | 865 | |
583cbf50 | 866 | do_div(req, latency); |
2422d366 | 867 | gb_loopback_update_stats(&gb->requests_per_second, req); |
355a7058 AB |
868 | } |
869 | ||
00af6583 | 870 | static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency) |
355a7058 | 871 | { |
2422d366 | 872 | u32 throughput; |
f7908e4d BD |
873 | u32 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2; |
874 | ||
8e1d6c33 | 875 | switch (gb->type) { |
f7908e4d BD |
876 | case GB_LOOPBACK_TYPE_PING: |
877 | break; | |
878 | case GB_LOOPBACK_TYPE_SINK: | |
879 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
8e1d6c33 | 880 | gb->size; |
f7908e4d BD |
881 | break; |
882 | case GB_LOOPBACK_TYPE_TRANSFER: | |
883 | aggregate_size += sizeof(struct gb_loopback_transfer_request) + | |
884 | sizeof(struct gb_loopback_transfer_response) + | |
8e1d6c33 | 885 | gb->size * 2; |
f7908e4d BD |
886 | break; |
887 | default: | |
888 | return; | |
889 | } | |
00af6583 | 890 | |
2422d366 GKH |
891 | /* Calculate bytes per second */ |
892 | throughput = USEC_PER_SEC; | |
00af6583 | 893 | do_div(throughput, latency); |
2422d366 GKH |
894 | throughput *= aggregate_size; |
895 | gb_loopback_update_stats(&gb->throughput, throughput); | |
355a7058 AB |
896 | } |
897 | ||
2422d366 | 898 | static void gb_loopback_calculate_stats(struct gb_loopback *gb) |
355a7058 AB |
899 | { |
900 | u32 lat; | |
355a7058 | 901 | |
00af6583 | 902 | /* Express latency in terms of microseconds */ |
bd416103 | 903 | lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs); |
355a7058 | 904 | |
98676ca8 | 905 | /* Log latency stastic */ |
00af6583 | 906 | gb_loopback_update_stats(&gb->latency, lat); |
67d1eece BD |
907 | |
908 | /* Raw latency log on a per thread basis */ | |
4b0ea00c | 909 | kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat)); |
00af6583 | 910 | |
2422d366 GKH |
911 | /* Log throughput and requests using latency as benchmark */ |
912 | gb_loopback_throughput_update(gb, lat); | |
913 | gb_loopback_requests_update(gb, lat); | |
914 | ||
1ec5843e | 915 | /* Log the firmware supplied latency values */ |
1ec5843e BD |
916 | gb_loopback_update_stats(&gb->apbridge_unipro_latency, |
917 | gb->apbridge_latency_ts); | |
1ec5843e BD |
918 | gb_loopback_update_stats(&gb->gpbridge_firmware_latency, |
919 | gb->gpbridge_latency_ts); | |
355a7058 AB |
920 | } |
921 | ||
8e3fba55 BD |
922 | static void gb_loopback_async_wait_to_send(struct gb_loopback *gb) |
923 | { | |
924 | if (!(gb->async && gb->outstanding_operations_max)) | |
925 | return; | |
926 | wait_event_interruptible(gb->wq_completion, | |
927 | (atomic_read(&gb->outstanding_operations) < | |
928 | gb->outstanding_operations_max) || | |
929 | kthread_should_stop()); | |
930 | } | |
931 | ||
355a7058 AB |
932 | static int gb_loopback_fn(void *data) |
933 | { | |
934 | int error = 0; | |
b36f04fa | 935 | int us_wait = 0; |
67d1eece BD |
936 | int type; |
937 | u32 size; | |
12927835 | 938 | u32 send_count = 0; |
3f12c3ed | 939 | struct gb_loopback *gb = data; |
355a7058 | 940 | |
3dfe8aaa | 941 | while (1) { |
8e1d6c33 BD |
942 | if (!gb->type) |
943 | wait_event_interruptible(gb->wq, gb->type || | |
3dfe8aaa | 944 | kthread_should_stop()); |
8e3fba55 BD |
945 | if (kthread_should_stop()) |
946 | break; | |
12927835 | 947 | |
8e3fba55 BD |
948 | /* Limit the maximum number of in-flight async operations */ |
949 | gb_loopback_async_wait_to_send(gb); | |
3dfe8aaa BD |
950 | if (kthread_should_stop()) |
951 | break; | |
85d678c0 | 952 | |
67d1eece | 953 | mutex_lock(&gb->mutex); |
8e1d6c33 BD |
954 | sysfs_notify(&gb->connection->bundle->dev.kobj, |
955 | NULL, "iteration_count"); | |
956 | ||
957 | /* Optionally terminate */ | |
12927835 | 958 | if (send_count == gb->iteration_max) { |
2422d366 GKH |
959 | gb->type = 0; |
960 | send_count = 0; | |
67d1eece | 961 | mutex_unlock(&gb->mutex); |
8e1d6c33 | 962 | continue; |
67d1eece | 963 | } |
8e1d6c33 | 964 | size = gb->size; |
b36f04fa | 965 | us_wait = gb->us_wait; |
8e1d6c33 | 966 | type = gb->type; |
d9fb3754 BD |
967 | mutex_unlock(&gb->mutex); |
968 | ||
67d1eece | 969 | /* Else operations to perform */ |
12927835 BD |
970 | if (gb->async) { |
971 | if (type == GB_LOOPBACK_TYPE_PING) { | |
972 | error = gb_loopback_async_ping(gb); | |
12927835 BD |
973 | } else if (type == GB_LOOPBACK_TYPE_TRANSFER) { |
974 | error = gb_loopback_async_transfer(gb, size); | |
975 | } else if (type == GB_LOOPBACK_TYPE_SINK) { | |
976 | error = gb_loopback_async_sink(gb, size); | |
977 | } | |
978 | ||
979 | if (error) | |
980 | gb->error++; | |
981 | } else { | |
982 | /* We are effectively single threaded here */ | |
983 | if (type == GB_LOOPBACK_TYPE_PING) | |
984 | error = gb_loopback_sync_ping(gb); | |
985 | else if (type == GB_LOOPBACK_TYPE_TRANSFER) | |
986 | error = gb_loopback_sync_transfer(gb, size); | |
987 | else if (type == GB_LOOPBACK_TYPE_SINK) | |
988 | error = gb_loopback_sync_sink(gb, size); | |
989 | ||
990 | if (error) | |
991 | gb->error++; | |
d9048d8c AB |
992 | else |
993 | gb_loopback_calculate_stats(gb); | |
12927835 | 994 | gb->iteration_count++; |
12927835 BD |
995 | } |
996 | send_count++; | |
b36f04fa BD |
997 | if (us_wait) |
998 | udelay(us_wait); | |
355a7058 AB |
999 | } |
1000 | return 0; | |
1001 | } | |
1002 | ||
4b0ea00c BD |
1003 | static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s, |
1004 | struct kfifo *kfifo, | |
1005 | struct mutex *mutex) | |
aa27bf82 | 1006 | { |
aa27bf82 BD |
1007 | u32 latency; |
1008 | int retval; | |
1009 | ||
4b0ea00c | 1010 | if (kfifo_len(kfifo) == 0) { |
aa27bf82 BD |
1011 | retval = -EAGAIN; |
1012 | goto done; | |
1013 | } | |
1014 | ||
4b0ea00c BD |
1015 | mutex_lock(mutex); |
1016 | retval = kfifo_out(kfifo, &latency, sizeof(latency)); | |
aa27bf82 BD |
1017 | if (retval > 0) { |
1018 | seq_printf(s, "%u", latency); | |
1019 | retval = 0; | |
1020 | } | |
4b0ea00c | 1021 | mutex_unlock(mutex); |
aa27bf82 BD |
1022 | done: |
1023 | return retval; | |
1024 | } | |
1025 | ||
4b0ea00c BD |
1026 | static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused) |
1027 | { | |
1028 | struct gb_loopback *gb = s->private; | |
1029 | ||
1030 | return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat, | |
1031 | &gb->mutex); | |
1032 | } | |
1033 | ||
aa27bf82 BD |
1034 | static int gb_loopback_latency_open(struct inode *inode, struct file *file) |
1035 | { | |
1036 | return single_open(file, gb_loopback_dbgfs_latency_show, | |
1037 | inode->i_private); | |
1038 | } | |
1039 | ||
1040 | static const struct file_operations gb_loopback_debugfs_latency_ops = { | |
1041 | .open = gb_loopback_latency_open, | |
1042 | .read = seq_read, | |
1043 | .llseek = seq_lseek, | |
1044 | .release = single_release, | |
1045 | }; | |
1046 | ||
5015115d BD |
1047 | static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha, |
1048 | struct list_head *lhb) | |
1049 | { | |
1050 | struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry); | |
1051 | struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry); | |
1052 | struct gb_connection *ca = a->connection; | |
1053 | struct gb_connection *cb = b->connection; | |
1054 | ||
5015115d BD |
1055 | if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id) |
1056 | return -1; | |
1057 | if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id) | |
1058 | return 1; | |
1059 | if (ca->bundle->id < cb->bundle->id) | |
1060 | return -1; | |
1061 | if (cb->bundle->id < ca->bundle->id) | |
1062 | return 1; | |
1063 | if (ca->intf_cport_id < cb->intf_cport_id) | |
1064 | return -1; | |
1065 | else if (cb->intf_cport_id < ca->intf_cport_id) | |
1066 | return 1; | |
1067 | ||
1068 | return 0; | |
1069 | } | |
1070 | ||
1071 | static void gb_loopback_insert_id(struct gb_loopback *gb) | |
1072 | { | |
1073 | struct gb_loopback *gb_list; | |
1074 | u32 new_lbid = 0; | |
1075 | ||
1076 | /* perform an insertion sort */ | |
1077 | list_add_tail(&gb->entry, &gb_dev.list); | |
1078 | list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare); | |
1079 | list_for_each_entry(gb_list, &gb_dev.list, entry) { | |
1080 | gb_list->lbid = 1 << new_lbid; | |
1081 | new_lbid++; | |
1082 | } | |
1083 | } | |
1084 | ||
67d1eece | 1085 | #define DEBUGFS_NAMELEN 32 |
aa27bf82 | 1086 | |
e82a11dc VK |
1087 | static int gb_loopback_probe(struct gb_bundle *bundle, |
1088 | const struct greybus_bundle_id *id) | |
355a7058 | 1089 | { |
e82a11dc VK |
1090 | struct greybus_descriptor_cport *cport_desc; |
1091 | struct gb_connection *connection; | |
355a7058 | 1092 | struct gb_loopback *gb; |
079fa32b | 1093 | struct device *dev; |
355a7058 | 1094 | int retval; |
aa27bf82 | 1095 | char name[DEBUGFS_NAMELEN]; |
2e238d71 | 1096 | unsigned long flags; |
355a7058 | 1097 | |
e82a11dc VK |
1098 | if (bundle->num_cports != 1) |
1099 | return -ENODEV; | |
1100 | ||
1101 | cport_desc = &bundle->cport_desc[0]; | |
1102 | if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK) | |
1103 | return -ENODEV; | |
1104 | ||
355a7058 AB |
1105 | gb = kzalloc(sizeof(*gb), GFP_KERNEL); |
1106 | if (!gb) | |
1107 | return -ENOMEM; | |
1108 | ||
e82a11dc VK |
1109 | connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), |
1110 | gb_loopback_request_handler); | |
1111 | if (IS_ERR(connection)) { | |
1112 | retval = PTR_ERR(connection); | |
1113 | goto out_kzalloc; | |
1114 | } | |
1115 | ||
1116 | gb->connection = connection; | |
1117 | greybus_set_drvdata(bundle, gb); | |
1118 | ||
8e1d6c33 | 1119 | init_waitqueue_head(&gb->wq); |
36f241ff BD |
1120 | init_waitqueue_head(&gb->wq_completion); |
1121 | atomic_set(&gb->outstanding_operations, 0); | |
8e1d6c33 BD |
1122 | gb_loopback_reset_stats(gb); |
1123 | ||
12927835 BD |
1124 | /* Reported values to user-space for min/max timeouts */ |
1125 | gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN); | |
1126 | gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX); | |
1127 | ||
f06272b2 | 1128 | if (!gb_dev.count) { |
f06272b2 BD |
1129 | /* Calculate maximum payload */ |
1130 | gb_dev.size_max = gb_operation_get_payload_size_max(connection); | |
1131 | if (gb_dev.size_max <= | |
1132 | sizeof(struct gb_loopback_transfer_request)) { | |
1133 | retval = -EINVAL; | |
e82a11dc | 1134 | goto out_connection_destroy; |
f06272b2 BD |
1135 | } |
1136 | gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request); | |
1137 | } | |
f06272b2 BD |
1138 | |
1139 | /* Create per-connection sysfs and debugfs data-points */ | |
8d8d36da | 1140 | snprintf(name, sizeof(name), "raw_latency_%s", |
d9a9ea1b | 1141 | dev_name(&connection->bundle->dev)); |
aa27bf82 BD |
1142 | gb->file = debugfs_create_file(name, S_IFREG | S_IRUGO, gb_dev.root, gb, |
1143 | &gb_loopback_debugfs_latency_ops); | |
079fa32b AH |
1144 | |
1145 | gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL); | |
1146 | if (gb->id < 0) { | |
1147 | retval = gb->id; | |
e82a11dc | 1148 | goto out_debugfs_remove; |
079fa32b AH |
1149 | } |
1150 | ||
e82a11dc VK |
1151 | retval = gb_connection_enable(connection); |
1152 | if (retval) | |
1153 | goto out_ida_remove; | |
1154 | ||
079fa32b AH |
1155 | dev = device_create_with_groups(&loopback_class, |
1156 | &connection->bundle->dev, | |
1157 | MKDEV(0, 0), gb, loopback_groups, | |
1158 | "gb_loopback%d", gb->id); | |
1159 | if (IS_ERR(dev)) { | |
1160 | retval = PTR_ERR(dev); | |
e82a11dc | 1161 | goto out_connection_disable; |
079fa32b AH |
1162 | } |
1163 | gb->dev = dev; | |
c3bba87a | 1164 | |
cbd204b4 | 1165 | /* Allocate kfifo */ |
4b0ea00c | 1166 | if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32), |
cbd204b4 BD |
1167 | GFP_KERNEL)) { |
1168 | retval = -ENOMEM; | |
079fa32b | 1169 | goto out_conn; |
cbd204b4 | 1170 | } |
4b0ea00c BD |
1171 | if (kfifo_alloc(&gb->kfifo_ts, kfifo_depth * sizeof(struct timeval) * 2, |
1172 | GFP_KERNEL)) { | |
1173 | retval = -ENOMEM; | |
1174 | goto out_kfifo0; | |
1175 | } | |
cbd204b4 BD |
1176 | |
1177 | /* Fork worker thread */ | |
85d678c0 | 1178 | mutex_init(&gb->mutex); |
355a7058 AB |
1179 | gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); |
1180 | if (IS_ERR(gb->task)) { | |
69f60347 | 1181 | retval = PTR_ERR(gb->task); |
4b0ea00c | 1182 | goto out_kfifo1; |
355a7058 AB |
1183 | } |
1184 | ||
2e238d71 | 1185 | spin_lock_irqsave(&gb_dev.lock, flags); |
5015115d | 1186 | gb_loopback_insert_id(gb); |
aa27bf82 | 1187 | gb_dev.count++; |
2e238d71 BD |
1188 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1189 | ||
1190 | gb_connection_latency_tag_enable(connection); | |
355a7058 AB |
1191 | return 0; |
1192 | ||
4b0ea00c BD |
1193 | out_kfifo1: |
1194 | kfifo_free(&gb->kfifo_ts); | |
1195 | out_kfifo0: | |
1196 | kfifo_free(&gb->kfifo_lat); | |
079fa32b AH |
1197 | out_conn: |
1198 | device_unregister(dev); | |
e82a11dc VK |
1199 | out_connection_disable: |
1200 | gb_connection_disable(connection); | |
1201 | out_ida_remove: | |
079fa32b | 1202 | ida_simple_remove(&loopback_ida, gb->id); |
e82a11dc | 1203 | out_debugfs_remove: |
aa27bf82 | 1204 | debugfs_remove(gb->file); |
e82a11dc VK |
1205 | out_connection_destroy: |
1206 | gb_connection_destroy(connection); | |
2e238d71 | 1207 | out_kzalloc: |
355a7058 | 1208 | kfree(gb); |
1fb807cf | 1209 | |
355a7058 AB |
1210 | return retval; |
1211 | } | |
1212 | ||
e82a11dc | 1213 | static void gb_loopback_disconnect(struct gb_bundle *bundle) |
355a7058 | 1214 | { |
e82a11dc | 1215 | struct gb_loopback *gb = greybus_get_drvdata(bundle); |
2e238d71 | 1216 | unsigned long flags; |
355a7058 | 1217 | |
e82a11dc VK |
1218 | gb_connection_disable(gb->connection); |
1219 | ||
355a7058 AB |
1220 | if (!IS_ERR_OR_NULL(gb->task)) |
1221 | kthread_stop(gb->task); | |
cbd204b4 | 1222 | |
4b0ea00c BD |
1223 | kfifo_free(&gb->kfifo_lat); |
1224 | kfifo_free(&gb->kfifo_ts); | |
e82a11dc | 1225 | gb_connection_latency_tag_disable(gb->connection); |
aa27bf82 | 1226 | debugfs_remove(gb->file); |
e82a11dc VK |
1227 | |
1228 | /* | |
1229 | * FIXME: gb_loopback_async_wait_all() is redundant now, as connection | |
1230 | * is disabled at the beginning and so we can't have any more | |
1231 | * incoming/outgoing requests. | |
1232 | */ | |
36f241ff | 1233 | gb_loopback_async_wait_all(gb); |
2e238d71 BD |
1234 | |
1235 | spin_lock_irqsave(&gb_dev.lock, flags); | |
1236 | gb_dev.count--; | |
ff477d07 | 1237 | list_del(&gb->entry); |
2e238d71 BD |
1238 | spin_unlock_irqrestore(&gb_dev.lock, flags); |
1239 | ||
079fa32b AH |
1240 | device_unregister(gb->dev); |
1241 | ida_simple_remove(&loopback_ida, gb->id); | |
1242 | ||
e82a11dc | 1243 | gb_connection_destroy(gb->connection); |
5f3e0d17 | 1244 | kfree(gb); |
355a7058 AB |
1245 | } |
1246 | ||
e82a11dc VK |
1247 | static const struct greybus_bundle_id gb_loopback_id_table[] = { |
1248 | { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) }, | |
1249 | { } | |
1250 | }; | |
1251 | MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table); | |
1252 | ||
1253 | static struct greybus_driver gb_loopback_driver = { | |
1254 | .name = "loopback", | |
1255 | .probe = gb_loopback_probe, | |
1256 | .disconnect = gb_loopback_disconnect, | |
1257 | .id_table = gb_loopback_id_table, | |
355a7058 AB |
1258 | }; |
1259 | ||
cbd204b4 BD |
1260 | static int loopback_init(void) |
1261 | { | |
4b0ea00c BD |
1262 | int retval; |
1263 | ||
67d1eece | 1264 | INIT_LIST_HEAD(&gb_dev.list); |
12927835 | 1265 | INIT_LIST_HEAD(&gb_dev.list_op_async); |
2e238d71 | 1266 | spin_lock_init(&gb_dev.lock); |
aa27bf82 | 1267 | gb_dev.root = debugfs_create_dir("gb_loopback", NULL); |
67d1eece | 1268 | |
079fa32b AH |
1269 | retval = class_register(&loopback_class); |
1270 | if (retval) | |
1271 | goto err; | |
1272 | ||
e82a11dc | 1273 | retval = greybus_register(&gb_loopback_driver); |
079fa32b AH |
1274 | if (retval) |
1275 | goto err_unregister; | |
1276 | ||
1277 | return 0; | |
4b0ea00c | 1278 | |
079fa32b AH |
1279 | err_unregister: |
1280 | class_unregister(&loopback_class); | |
1281 | err: | |
4b0ea00c BD |
1282 | debugfs_remove_recursive(gb_dev.root); |
1283 | return retval; | |
cbd204b4 BD |
1284 | } |
1285 | module_init(loopback_init); | |
1286 | ||
1287 | static void __exit loopback_exit(void) | |
1288 | { | |
aa27bf82 | 1289 | debugfs_remove_recursive(gb_dev.root); |
e82a11dc | 1290 | greybus_deregister(&gb_loopback_driver); |
079fa32b AH |
1291 | class_unregister(&loopback_class); |
1292 | ida_destroy(&loopback_ida); | |
cbd204b4 BD |
1293 | } |
1294 | module_exit(loopback_exit); | |
355a7058 AB |
1295 | |
1296 | MODULE_LICENSE("GPL v2"); |