projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
staging: speakup: Remove the explicit NULL comparison
[linux-2.6-block.git] / drivers / staging / greybus / loopback.c
CommitLineData
355a7058
AB		 1/*
2 * Loopback bridge driver for the Greybus loopback module.
3 *
4 * Copyright 2014 Google Inc.
5 * Copyright 2014 Linaro Ltd.
6 *
7 * Released under the GPLv2 only.
8 */
8923c5b5
JH		 9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
355a7058
AB		 12#include <linux/kernel.h>
13#include <linux/module.h>
85d678c0 14#include <linux/mutex.h>
355a7058
AB		 15#include <linux/slab.h>
16#include <linux/kthread.h>
17#include <linux/delay.h>
18#include <linux/random.h>
5679f783 19#include <linux/sizes.h>
cbd204b4
BD		 20#include <linux/cdev.h>
21#include <linux/fs.h>
22#include <linux/kfifo.h>
aa27bf82 23#include <linux/debugfs.h>
5015115d 24#include <linux/list_sort.h>
2e238d71 25#include <linux/spinlock.h>
12927835 26#include <linux/workqueue.h>
36f241ff 27#include <linux/atomic.h>
e854ff58 28#include <linux/pm_runtime.h>
cbd204b4 29
1ffc12be
JH		 30#include <asm/div64.h>
31
355a7058 32#include "greybus.h"
fd58926e 33#include "connection.h"
355a7058 34
fd489e1a
BD		 35#define NSEC_PER_DAY 86400000000000ULL
36
355a7058 37struct gb_loopback_stats {
e051c82b
AE		 38 u32 min;
39 u32 max;
00af6583 40 u64 sum;
e13fc28f 41 u32 count;
355a7058
AB		 42};
43
aa27bf82
BD		 44struct gb_loopback_device {
45 struct dentry *root;
46 u32 count;
8e1d6c33 47 size_t size_max;
67d1eece 48
2e238d71
BD		 49 /* We need to take a lock in atomic context */
50 spinlock_t lock;
67d1eece 51 struct list_head list;
12927835
BD		 52 struct list_head list_op_async;
53 wait_queue_head_t wq;
aa27bf82
BD		 54};
55
56static struct gb_loopback_device gb_dev;
57
12927835
BD		 58struct gb_loopback_async_operation {
59 struct gb_loopback *gb;
60 struct gb_operation *operation;
61 struct timeval ts;
62 struct timer_list timer;
63 struct list_head entry;
64 struct work_struct work;
65 struct kref kref;
66 bool pending;
67 int (*completion)(struct gb_loopback_async_operation *op_async);
68};
69
355a7058
AB		 70struct gb_loopback {
71 struct gb_connection *connection;
355a7058 72
aa27bf82 73 struct dentry *file;
4b0ea00c
BD		 74 struct kfifo kfifo_lat;
75 struct kfifo kfifo_ts;
85d678c0 76 struct mutex mutex;
355a7058 77 struct task_struct *task;
67d1eece 78 struct list_head entry;
079fa32b 79 struct device *dev;
8e1d6c33 80 wait_queue_head_t wq;
36f241ff
BD		 81 wait_queue_head_t wq_completion;
82 atomic_t outstanding_operations;
355a7058 83
67d1eece 84 /* Per connection stats */
ab81bb9c 85 struct timeval ts;
355a7058
AB		 86 struct gb_loopback_stats latency;
87 struct gb_loopback_stats throughput;
583cbf50 88 struct gb_loopback_stats requests_per_second;
1ec5843e 89 struct gb_loopback_stats apbridge_unipro_latency;
e54b106d 90 struct gb_loopback_stats gbphy_firmware_latency;
67d1eece 91
8e1d6c33 92 int type;
12927835 93 int async;
079fa32b 94 int id;
8e1d6c33
BD		 95 u32 size;
96 u32 iteration_max;
67d1eece 97 u32 iteration_count;
b36f04fa 98 int us_wait;
355a7058 99 u32 error;
12927835
BD		 100 u32 requests_completed;
101 u32 requests_timedout;
102 u32 timeout;
103 u32 jiffy_timeout;
104 u32 timeout_min;
105 u32 timeout_max;
8e3fba55 106 u32 outstanding_operations_max;
8e1d6c33
BD		 107 u32 lbid;
108 u64 elapsed_nsecs;
e6227ee6 109 u32 apbridge_latency_ts;
e54b106d 110 u32 gbphy_latency_ts;
01480ba3
AB		 111
112 u32 send_count;
355a7058
AB		 113};
114
079fa32b
AH		 115static struct class loopback_class = {
116 .name = "gb_loopback",
117 .owner = THIS_MODULE,
118};
119static DEFINE_IDA(loopback_ida);
120
12927835
BD		 121/* Min/max values in jiffies */
122#define GB_LOOPBACK_TIMEOUT_MIN 1
123#define GB_LOOPBACK_TIMEOUT_MAX 10000
124
cbd204b4
BD		 125#define GB_LOOPBACK_FIFO_DEFAULT 8192
126
1d3dfbd1 127static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT;
cbd204b4
BD		 128module_param(kfifo_depth, uint, 0444);
129
cbd204b4
BD		 130/* Maximum size of any one send data buffer we support */
131#define MAX_PACKET_SIZE (PAGE_SIZE * 2)
132
b36f04fa 133#define GB_LOOPBACK_US_WAIT_MAX 1000000
355a7058 134
355a7058 135/* interface sysfs attributes */
8e1d6c33
BD		 136#define gb_loopback_ro_attr(field) \
137static ssize_t field##_show(struct device *dev, \
355a7058
AB		 138 struct device_attribute *attr, \
139 char *buf) \
140{ \
079fa32b 141 struct gb_loopback *gb = dev_get_drvdata(dev); \
8e1d6c33 142 return sprintf(buf, "%u\n", gb->field); \
355a7058 143} \
8e1d6c33 144static DEVICE_ATTR_RO(field)
355a7058 145
8e1d6c33
BD		 146#define gb_loopback_ro_stats_attr(name, field, type) \
147static ssize_t name##_##field##_show(struct device *dev, \
355a7058
AB		 148 struct device_attribute *attr, \
149 char *buf) \
150{ \
079fa32b 151 struct gb_loopback *gb = dev_get_drvdata(dev); \
1dc8d3d7
AB		 152 /* Report 0 for min and max if no transfer successed */ \
153 if (!gb->requests_completed) \
154 return sprintf(buf, "0\n"); \
8e1d6c33 155 return sprintf(buf, "%"#type"\n", gb->name.field); \
355a7058 156} \
8e1d6c33 157static DEVICE_ATTR_RO(name##_##field)
355a7058 158
8e1d6c33
BD		 159#define gb_loopback_ro_avg_attr(name) \
160static ssize_t name##_avg_show(struct device *dev, \
7a135a96
AE		 161 struct device_attribute *attr, \
162 char *buf) \
163{ \
f06272b2 164 struct gb_loopback_stats *stats; \
f06272b2 165 struct gb_loopback *gb; \
fb37f137
AB		 166 u64 avg, rem; \
167 u32 count; \
079fa32b 168 gb = dev_get_drvdata(dev); \
8e1d6c33 169 stats = &gb->name; \
f06272b2 170 count = stats->count ? stats->count : 1; \
58a527af 171 avg = stats->sum + count / 2000000; /* round closest */ \
f06272b2 172 rem = do_div(avg, count); \
89ec14ce
AB		 173 rem *= 1000000; \
174 do_div(rem, count); \
fb37f137 175 return sprintf(buf, "%llu.%06u\n", avg, (u32)rem); \
7a135a96 176} \
8e1d6c33 177static DEVICE_ATTR_RO(name##_avg)
7a135a96 178
8e1d6c33
BD		 179#define gb_loopback_stats_attrs(field) \
180 gb_loopback_ro_stats_attr(field, min, u); \
181 gb_loopback_ro_stats_attr(field, max, u); \
182 gb_loopback_ro_avg_attr(field)
355a7058
AB		 183
184#define gb_loopback_attr(field, type) \
185static ssize_t field##_show(struct device *dev, \
186 struct device_attribute *attr, \
187 char *buf) \
188{ \
079fa32b 189 struct gb_loopback *gb = dev_get_drvdata(dev); \
355a7058
AB		 190 return sprintf(buf, "%"#type"\n", gb->field); \
191} \
192static ssize_t field##_store(struct device *dev, \
193 struct device_attribute *attr, \
194 const char *buf, \
195 size_t len) \
196{ \
197 int ret; \
079fa32b 198 struct gb_loopback *gb = dev_get_drvdata(dev); \
8e1d6c33 199 mutex_lock(&gb->mutex); \
355a7058 200 ret = sscanf(buf, "%"#type, &gb->field); \
355a7058 201 if (ret != 1) \
85d678c0
BD		 202 len = -EINVAL; \
203 else \
8e1d6c33
BD		 204 gb_loopback_check_attr(gb, bundle); \
205 mutex_unlock(&gb->mutex); \
355a7058
AB		 206 return len; \
207} \
208static DEVICE_ATTR_RW(field)
209
f06272b2
BD		 210#define gb_dev_loopback_ro_attr(field, conn) \
211static ssize_t field##_show(struct device *dev, \
67d1eece
BD		 212 struct device_attribute *attr, \
213 char *buf) \
214{ \
079fa32b 215 struct gb_loopback *gb = dev_get_drvdata(dev); \
8e1d6c33 216 return sprintf(buf, "%u\n", gb->field); \
67d1eece
BD		 217} \
218static DEVICE_ATTR_RO(field)
219
220#define gb_dev_loopback_rw_attr(field, type) \
221static ssize_t field##_show(struct device *dev, \
222 struct device_attribute *attr, \
223 char *buf) \
224{ \
079fa32b 225 struct gb_loopback *gb = dev_get_drvdata(dev); \
8e1d6c33 226 return sprintf(buf, "%"#type"\n", gb->field); \
67d1eece
BD		 227} \
228static ssize_t field##_store(struct device *dev, \
229 struct device_attribute *attr, \
230 const char *buf, \
231 size_t len) \
232{ \
233 int ret; \
079fa32b 234 struct gb_loopback *gb = dev_get_drvdata(dev); \
8e1d6c33
BD		 235 mutex_lock(&gb->mutex); \
236 ret = sscanf(buf, "%"#type, &gb->field); \
67d1eece
BD		 237 if (ret != 1) \
238 len = -EINVAL; \
239 else \
079fa32b 240 gb_loopback_check_attr(gb); \
8e1d6c33 241 mutex_unlock(&gb->mutex); \
67d1eece
BD		 242 return len; \
243} \
244static DEVICE_ATTR_RW(field)
245
8e1d6c33 246static void gb_loopback_reset_stats(struct gb_loopback *gb);
079fa32b 247static void gb_loopback_check_attr(struct gb_loopback *gb)
355a7058 248{
b36f04fa
BD		 249 if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX)
250 gb->us_wait = GB_LOOPBACK_US_WAIT_MAX;
8e1d6c33
BD		 251 if (gb->size > gb_dev.size_max)
252 gb->size = gb_dev.size_max;
12927835
BD		 253 gb->requests_timedout = 0;
254 gb->requests_completed = 0;
8e1d6c33 255 gb->iteration_count = 0;
01480ba3 256 gb->send_count = 0;
8e1d6c33
BD		 257 gb->error = 0;
258
259 if (kfifo_depth < gb->iteration_max) {
079fa32b 260 dev_warn(gb->dev,
8e1d6c33
BD		 261 "cannot log bytes %u kfifo_depth %u\n",
262 gb->iteration_max, kfifo_depth);
cb60f496 263 }
8e1d6c33
BD		 264 kfifo_reset_out(&gb->kfifo_lat);
265 kfifo_reset_out(&gb->kfifo_ts);
cb60f496 266
8e1d6c33 267 switch (gb->type) {
a598f438
BD		 268 case GB_LOOPBACK_TYPE_PING:
269 case GB_LOOPBACK_TYPE_TRANSFER:
384a7a3c 270 case GB_LOOPBACK_TYPE_SINK:
12927835
BD		 271 gb->jiffy_timeout = usecs_to_jiffies(gb->timeout);
272 if (!gb->jiffy_timeout)
273 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN;
274 else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX)
275 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX;
8e1d6c33
BD		 276 gb_loopback_reset_stats(gb);
277 wake_up(&gb->wq);
a598f438
BD		 278 break;
279 default:
8e1d6c33 280 gb->type = 0;
a598f438
BD		 281 break;
282 }
355a7058
AB		 283}
284
285/* Time to send and receive one message */
8e1d6c33 286gb_loopback_stats_attrs(latency);
583cbf50 287/* Number of requests sent per second on this cport */
8e1d6c33 288gb_loopback_stats_attrs(requests_per_second);
355a7058 289/* Quantity of data sent and received on this cport */
8e1d6c33 290gb_loopback_stats_attrs(throughput);
1ec5843e 291/* Latency across the UniPro link from APBridge's perspective */
8e1d6c33 292gb_loopback_stats_attrs(apbridge_unipro_latency);
1ec5843e 293/* Firmware induced overhead in the GPBridge */
e54b106d 294gb_loopback_stats_attrs(gbphy_firmware_latency);
8e1d6c33 295
e140c75e 296/* Number of errors encountered during loop */
8e1d6c33 297gb_loopback_ro_attr(error);
12927835
BD		 298/* Number of requests successfully completed async */
299gb_loopback_ro_attr(requests_completed);
300/* Number of requests timed out async */
301gb_loopback_ro_attr(requests_timedout);
302/* Timeout minimum in useconds */
303gb_loopback_ro_attr(timeout_min);
304/* Timeout minimum in useconds */
305gb_loopback_ro_attr(timeout_max);
355a7058
AB		 306
307/*
799a3f03 308 * Type of loopback message to send based on protocol type definitions
355a7058 309 * 0 => Don't send message
799a3f03 310 * 2 => Send ping message continuously (message without payload)
006335a0 311 * 3 => Send transfer message continuously (message with payload,
799a3f03
BD		 312 * payload returned in response)
313 * 4 => Send a sink message (message with payload, no payload in response)
355a7058 314 */
67d1eece 315gb_dev_loopback_rw_attr(type, d);
355a7058 316/* Size of transfer message payload: 0-4096 bytes */
67d1eece 317gb_dev_loopback_rw_attr(size, u);
48f19ee8 318/* Time to wait between two messages: 0-1000 ms */
b36f04fa 319gb_dev_loopback_rw_attr(us_wait, d);
00af6583 320/* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */
67d1eece
BD		 321gb_dev_loopback_rw_attr(iteration_max, u);
322/* The current index of the for (i = 0; i < iteration_max; i++) loop */
f06272b2 323gb_dev_loopback_ro_attr(iteration_count, false);
12927835
BD		 324/* A flag to indicate synchronous or asynchronous operations */
325gb_dev_loopback_rw_attr(async, u);
326/* Timeout of an individual asynchronous request */
327gb_dev_loopback_rw_attr(timeout, u);
8e3fba55
BD		 328/* Maximum number of in-flight operations before back-off */
329gb_dev_loopback_rw_attr(outstanding_operations_max, u);
355a7058 330
8e1d6c33
BD		 331static struct attribute *loopback_attrs[] = {
332 &dev_attr_latency_min.attr,
333 &dev_attr_latency_max.attr,
334 &dev_attr_latency_avg.attr,
335 &dev_attr_requests_per_second_min.attr,
336 &dev_attr_requests_per_second_max.attr,
337 &dev_attr_requests_per_second_avg.attr,
338 &dev_attr_throughput_min.attr,
339 &dev_attr_throughput_max.attr,
340 &dev_attr_throughput_avg.attr,
341 &dev_attr_apbridge_unipro_latency_min.attr,
342 &dev_attr_apbridge_unipro_latency_max.attr,
343 &dev_attr_apbridge_unipro_latency_avg.attr,
e54b106d
SP		 344 &dev_attr_gbphy_firmware_latency_min.attr,
345 &dev_attr_gbphy_firmware_latency_max.attr,
346 &dev_attr_gbphy_firmware_latency_avg.attr,
355a7058
AB		 347 &dev_attr_type.attr,
348 &dev_attr_size.attr,
b36f04fa 349 &dev_attr_us_wait.attr,
00af6583
BD		 350 &dev_attr_iteration_count.attr,
351 &dev_attr_iteration_max.attr,
12927835 352 &dev_attr_async.attr,
8e1d6c33 353 &dev_attr_error.attr,
12927835
BD		 354 &dev_attr_requests_completed.attr,
355 &dev_attr_requests_timedout.attr,
356 &dev_attr_timeout.attr,
8e3fba55 357 &dev_attr_outstanding_operations_max.attr,
12927835
BD		 358 &dev_attr_timeout_min.attr,
359 &dev_attr_timeout_max.attr,
355a7058
AB		 360 NULL,
361};
8e1d6c33 362ATTRIBUTE_GROUPS(loopback);
355a7058 363
ab81bb9c 364static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error);
12927835 365
bd416103
BD		 366static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs)
367{
368 u32 lat;
369
370 do_div(elapsed_nsecs, NSEC_PER_USEC);
371 lat = elapsed_nsecs;
372 return lat;
373}
374
7c985351
BD		 375static u64 __gb_loopback_calc_latency(u64 t1, u64 t2)
376{
377 if (t2 > t1)
378 return t2 - t1;
379 else
380 return NSEC_PER_DAY - t2 + t1;
381}
382
2f842304 383static u64 gb_loopback_calc_latency(struct timeval *ts, struct timeval *te)
4c192665
BD		 384{
385 u64 t1, t2;
386
387 t1 = timeval_to_ns(ts);
388 t2 = timeval_to_ns(te);
7c985351
BD		 389
390 return __gb_loopback_calc_latency(t1, t2);
4c192665
BD		 391}
392
4b0ea00c
BD		 393static void gb_loopback_push_latency_ts(struct gb_loopback *gb,
394 struct timeval *ts, struct timeval *te)
395{
396 kfifo_in(&gb->kfifo_ts, (unsigned char *)ts, sizeof(*ts));
397 kfifo_in(&gb->kfifo_ts, (unsigned char *)te, sizeof(*te));
398}
399
fbb8edba
BD		 400static int gb_loopback_operation_sync(struct gb_loopback *gb, int type,
401 void *request, int request_size,
402 void *response, int response_size)
384a7a3c 403{
fbb8edba 404 struct gb_operation *operation;
384a7a3c 405 struct timeval ts, te;
fbb8edba 406 int ret;
384a7a3c
BD		 407
408 do_gettimeofday(&ts);
fbb8edba
BD		 409 operation = gb_operation_create(gb->connection, type, request_size,
410 response_size, GFP_KERNEL);
d9048d8c
AB		 411 if (!operation)
412 return -ENOMEM;
c3bba87a 413
fbb8edba
BD		 414 if (request_size)
415 memcpy(operation->request->payload, request, request_size);
416
417 ret = gb_operation_request_send_sync(operation);
418 if (ret) {
d9a9ea1b 419 dev_err(&gb->connection->bundle->dev,
fbb8edba 420 "synchronous operation failed: %d\n", ret);
d9048d8c 421 goto out_put_operation;
fbb8edba
BD		 422 } else {
423 if (response_size == operation->response->payload_size) {
424 memcpy(response, operation->response->payload,
425 response_size);
426 } else {
d9a9ea1b 427 dev_err(&gb->connection->bundle->dev,
fbb8edba
BD		 428 "response size %zu expected %d\n",
429 operation->response->payload_size,
430 response_size);
d9048d8c
AB		 431 ret = -EINVAL;
432 goto out_put_operation;
fbb8edba
BD		 433 }
434 }
6ab1ce4d 435
384a7a3c 436 do_gettimeofday(&te);
2f842304
BD		 437
438 /* Calculate the total time the message took */
4b0ea00c 439 gb_loopback_push_latency_ts(gb, &ts, &te);
2f842304
BD		 440 gb->elapsed_nsecs = gb_loopback_calc_latency(&ts, &te);
441
d9048d8c
AB		 442out_put_operation:
443 gb_operation_put(operation);
444
fbb8edba
BD		 445 return ret;
446}
384a7a3c 447
12927835
BD		 448static void __gb_loopback_async_operation_destroy(struct kref *kref)
449{
450 struct gb_loopback_async_operation *op_async;
451
452 op_async = container_of(kref, struct gb_loopback_async_operation, kref);
453
454 list_del(&op_async->entry);
455 if (op_async->operation)
456 gb_operation_put(op_async->operation);
36f241ff
BD		 457 atomic_dec(&op_async->gb->outstanding_operations);
458 wake_up(&op_async->gb->wq_completion);
12927835
BD		 459 kfree(op_async);
460}
461
462static void gb_loopback_async_operation_get(struct gb_loopback_async_operation
463 *op_async)
464{
465 kref_get(&op_async->kref);
466}
467
468static void gb_loopback_async_operation_put(struct gb_loopback_async_operation
469 *op_async)
470{
471 unsigned long flags;
472
473 spin_lock_irqsave(&gb_dev.lock, flags);
474 kref_put(&op_async->kref, __gb_loopback_async_operation_destroy);
475 spin_unlock_irqrestore(&gb_dev.lock, flags);
476}
477
478static struct gb_loopback_async_operation *
479 gb_loopback_operation_find(u16 id)
480{
481 struct gb_loopback_async_operation *op_async;
482 bool found = false;
483 unsigned long flags;
484
485 spin_lock_irqsave(&gb_dev.lock, flags);
486 list_for_each_entry(op_async, &gb_dev.list_op_async, entry) {
487 if (op_async->operation->id == id) {
488 gb_loopback_async_operation_get(op_async);
489 found = true;
490 break;
491 }
492 }
493 spin_unlock_irqrestore(&gb_dev.lock, flags);
494
495 return found ? op_async : NULL;
496}
497
36f241ff
BD		 498static void gb_loopback_async_wait_all(struct gb_loopback *gb)
499{
500 wait_event(gb->wq_completion,
501 !atomic_read(&gb->outstanding_operations));
502}
503
12927835
BD		 504static void gb_loopback_async_operation_callback(struct gb_operation *operation)
505{
506 struct gb_loopback_async_operation *op_async;
507 struct gb_loopback *gb;
508 struct timeval te;
509 bool err = false;
510
511 do_gettimeofday(&te);
512 op_async = gb_loopback_operation_find(operation->id);
513 if (!op_async)
514 return;
515
516 gb = op_async->gb;
517 mutex_lock(&gb->mutex);
518
519 if (!op_async->pending || gb_operation_result(operation)) {
520 err = true;
521 } else {
522 if (op_async->completion)
523 if (op_async->completion(op_async))
524 err = true;
525 }
526
41993cd5 527 if (!err) {
12927835
BD		 528 gb_loopback_push_latency_ts(gb, &op_async->ts, &te);
529 gb->elapsed_nsecs = gb_loopback_calc_latency(&op_async->ts,
530 &te);
12927835
BD		 531 }
532
533 if (op_async->pending) {
41993cd5
JH		 534 if (err)
535 gb->error++;
12927835
BD		 536 gb->iteration_count++;
537 op_async->pending = false;
538 del_timer_sync(&op_async->timer);
539 gb_loopback_async_operation_put(op_async);
ab81bb9c 540 gb_loopback_calculate_stats(gb, err);
12927835
BD		 541 }
542 mutex_unlock(&gb->mutex);
543
544 dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n",
545 operation->id);
546
547 gb_loopback_async_operation_put(op_async);
548}
549
550static void gb_loopback_async_operation_work(struct work_struct *work)
551{
552 struct gb_loopback *gb;
553 struct gb_operation *operation;
554 struct gb_loopback_async_operation *op_async;
555
556 op_async = container_of(work, struct gb_loopback_async_operation, work);
12927835
BD		 557 gb = op_async->gb;
558 operation = op_async->operation;
559
560 mutex_lock(&gb->mutex);
561 if (op_async->pending) {
562 gb->requests_timedout++;
563 gb->error++;
564 gb->iteration_count++;
565 op_async->pending = false;
566 gb_loopback_async_operation_put(op_async);
ab81bb9c 567 gb_loopback_calculate_stats(gb, true);
12927835
BD		 568 }
569 mutex_unlock(&gb->mutex);
570
571 dev_dbg(&gb->connection->bundle->dev, "timeout operation %d\n",
572 operation->id);
573
574 gb_operation_cancel(operation, -ETIMEDOUT);
575 gb_loopback_async_operation_put(op_async);
576}
577
578static void gb_loopback_async_operation_timeout(unsigned long data)
579{
580 struct gb_loopback_async_operation *op_async;
581 u16 id = data;
582
583 op_async = gb_loopback_operation_find(id);
584 if (!op_async) {
585 pr_err("operation %d not found - time out ?\n", id);
586 return;
587 }
588 schedule_work(&op_async->work);
589}
590
591static int gb_loopback_async_operation(struct gb_loopback *gb, int type,
592 void *request, int request_size,
593 int response_size,
594 void *completion)
595{
596 struct gb_loopback_async_operation *op_async;
597 struct gb_operation *operation;
598 int ret;
599 unsigned long flags;
600
601 op_async = kzalloc(sizeof(*op_async), GFP_KERNEL);
602 if (!op_async)
603 return -ENOMEM;
604
605 INIT_WORK(&op_async->work, gb_loopback_async_operation_work);
12927835
BD		 606 kref_init(&op_async->kref);
607
608 operation = gb_operation_create(gb->connection, type, request_size,
609 response_size, GFP_KERNEL);
610 if (!operation) {
c7aae4e6
BD		 611 kfree(op_async);
612 return -ENOMEM;
12927835
BD		 613 }
614
615 if (request_size)
616 memcpy(operation->request->payload, request, request_size);
617
618 op_async->gb = gb;
619 op_async->operation = operation;
620 op_async->completion = completion;
621
622 spin_lock_irqsave(&gb_dev.lock, flags);
623 list_add_tail(&op_async->entry, &gb_dev.list_op_async);
624 spin_unlock_irqrestore(&gb_dev.lock, flags);
625
626 do_gettimeofday(&op_async->ts);
627 op_async->pending = true;
36f241ff 628 atomic_inc(&gb->outstanding_operations);
fece9c87 629 mutex_lock(&gb->mutex);
12927835
BD		 630 ret = gb_operation_request_send(operation,
631 gb_loopback_async_operation_callback,
dbec2729 632 0,
12927835
BD		 633 GFP_KERNEL);
634 if (ret)
635 goto error;
636
82af03f7 637 setup_timer(&op_async->timer, gb_loopback_async_operation_timeout,
638 (unsigned long)operation->id);
12927835 639 op_async->timer.expires = jiffies + gb->jiffy_timeout;
12927835
BD		 640 add_timer(&op_async->timer);
641
fece9c87 642 goto done;
12927835
BD		 643error:
644 gb_loopback_async_operation_put(op_async);
fece9c87
BD		 645done:
646 mutex_unlock(&gb->mutex);
12927835
BD		 647 return ret;
648}
649
650static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len)
fbb8edba
BD		 651{
652 struct gb_loopback_transfer_request *request;
653 int retval;
654
655 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
656 if (!request)
657 return -ENOMEM;
658
659 request->len = cpu_to_le32(len);
660 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK,
661 request, len + sizeof(*request),
662 NULL, 0);
384a7a3c
BD		 663 kfree(request);
664 return retval;
665}
666
12927835 667static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len)
355a7058 668{
355a7058
AB		 669 struct gb_loopback_transfer_request *request;
670 struct gb_loopback_transfer_response *response;
671 int retval;
672
d6a1a3b5 673 gb->apbridge_latency_ts = 0;
e54b106d 674 gb->gbphy_latency_ts = 0;
d6a1a3b5 675
355a7058
AB		 676 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
677 if (!request)
678 return -ENOMEM;
679 response = kmalloc(len + sizeof(*response), GFP_KERNEL);
680 if (!response) {
681 kfree(request);
682 return -ENOMEM;
683 }
684
dc4a1069
VK		 685 memset(request->data, 0x5A, len);
686
355a7058 687 request->len = cpu_to_le32(len);
fbb8edba
BD		 688 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER,
689 request, len + sizeof(*request),
690 response, len + sizeof(*response));
355a7058
AB		 691 if (retval)
692 goto gb_error;
693
dc366f8e 694 if (memcmp(request->data, response->data, len)) {
d9a9ea1b
GKH		 695 dev_err(&gb->connection->bundle->dev,
696 "Loopback Data doesn't match\n");
355a7058 697 retval = -EREMOTEIO;
dc366f8e 698 }
1ec5843e 699 gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0);
e54b106d 700 gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1);
355a7058
AB		 701
702gb_error:
703 kfree(request);
704 kfree(response);
705
706 return retval;
707}
708
12927835 709static int gb_loopback_sync_ping(struct gb_loopback *gb)
355a7058 710{
fbb8edba
BD		 711 return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING,
712 NULL, 0, NULL, 0);
355a7058
AB		 713}
714
12927835
BD		 715static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len)
716{
717 struct gb_loopback_transfer_request *request;
718 int retval;
719
720 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
721 if (!request)
722 return -ENOMEM;
723
724 request->len = cpu_to_le32(len);
725 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK,
726 request, len + sizeof(*request),
727 0, NULL);
728 kfree(request);
729 return retval;
730}
731
732static int gb_loopback_async_transfer_complete(
733 struct gb_loopback_async_operation *op_async)
734{
735 struct gb_loopback *gb;
736 struct gb_operation *operation;
737 struct gb_loopback_transfer_request *request;
738 struct gb_loopback_transfer_response *response;
739 size_t len;
740 int retval = 0;
741
742 gb = op_async->gb;
743 operation = op_async->operation;
744 request = operation->request->payload;
745 response = operation->response->payload;
746 len = le32_to_cpu(request->len);
747
748 if (memcmp(request->data, response->data, len)) {
749 dev_err(&gb->connection->bundle->dev,
750 "Loopback Data doesn't match operation id %d\n",
751 operation->id);
752 retval = -EREMOTEIO;
753 } else {
754 gb->apbridge_latency_ts =
755 (u32)__le32_to_cpu(response->reserved0);
e54b106d 756 gb->gbphy_latency_ts =
12927835
BD		 757 (u32)__le32_to_cpu(response->reserved1);
758 }
759
760 return retval;
761}
762
763static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len)
764{
765 struct gb_loopback_transfer_request *request;
766 int retval, response_len;
767
768 request = kmalloc(len + sizeof(*request), GFP_KERNEL);
769 if (!request)
770 return -ENOMEM;
771
772 memset(request->data, 0x5A, len);
773
774 request->len = cpu_to_le32(len);
775 response_len = sizeof(struct gb_loopback_transfer_response);
776 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER,
777 request, len + sizeof(*request),
778 len + response_len,
779 gb_loopback_async_transfer_complete);
780 if (retval)
781 goto gb_error;
782
783gb_error:
784 kfree(request);
785 return retval;
786}
787
788static int gb_loopback_async_ping(struct gb_loopback *gb)
789{
790 return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING,
791 NULL, 0, 0, NULL);
792}
793
e82a11dc 794static int gb_loopback_request_handler(struct gb_operation *operation)
ac1c2840
AE		 795{
796 struct gb_connection *connection = operation->connection;
797 struct gb_loopback_transfer_request *request;
798 struct gb_loopback_transfer_response *response;
d9a9ea1b 799 struct device *dev = &connection->bundle->dev;
e51eafeb 800 size_t len;
ac1c2840
AE		 801
802 /* By convention, the AP initiates the version operation */
e82a11dc 803 switch (operation->type) {
ac1c2840 804 case GB_LOOPBACK_TYPE_PING:
384a7a3c 805 case GB_LOOPBACK_TYPE_SINK:
ac1c2840
AE		 806 return 0;
807 case GB_LOOPBACK_TYPE_TRANSFER:
808 if (operation->request->payload_size < sizeof(*request)) {
d9a9ea1b 809 dev_err(dev, "transfer request too small (%zu < %zu)\n",
ac1c2840
AE		 810 operation->request->payload_size,
811 sizeof(*request));
812 return -EINVAL; /* -EMSGSIZE */
813 }
814 request = operation->request->payload;
815 len = le32_to_cpu(request->len);
67d1eece 816 if (len > gb_dev.size_max) {
d9a9ea1b 817 dev_err(dev, "transfer request too large (%zu > %zu)\n",
67d1eece 818 len, gb_dev.size_max);
c3bba87a
BD		 819 return -EINVAL;
820 }
821
81ad6994
BG		 822 if (!gb_operation_response_alloc(operation,
823 len + sizeof(*response), GFP_KERNEL)) {
824 dev_err(dev, "error allocating response\n");
825 return -ENOMEM;
ac1c2840 826 }
81ad6994
BG		 827 response = operation->response->payload;
828 response->len = cpu_to_le32(len);
829 if (len)
830 memcpy(response->data, request->data, len);
831
ac1c2840
AE		 832 return 0;
833 default:
e82a11dc 834 dev_err(dev, "unsupported request: %u\n", operation->type);
ac1c2840
AE		 835 return -EINVAL;
836 }
837}
838
8e1d6c33 839static void gb_loopback_reset_stats(struct gb_loopback *gb)
355a7058
AB		 840{
841 struct gb_loopback_stats reset = {
e051c82b 842 .min = U32_MAX,
355a7058 843 };
67d1eece
BD		 844
845 /* Reset per-connection stats */
8e1d6c33
BD		 846 memcpy(&gb->latency, &reset,
847 sizeof(struct gb_loopback_stats));
848 memcpy(&gb->throughput, &reset,
849 sizeof(struct gb_loopback_stats));
850 memcpy(&gb->requests_per_second, &reset,
851 sizeof(struct gb_loopback_stats));
852 memcpy(&gb->apbridge_unipro_latency, &reset,
853 sizeof(struct gb_loopback_stats));
e54b106d 854 memcpy(&gb->gbphy_firmware_latency, &reset,
8e1d6c33 855 sizeof(struct gb_loopback_stats));
67d1eece 856
f42a6891
BD		 857 /* Should be initialized at least once per transaction set */
858 gb->apbridge_latency_ts = 0;
e54b106d 859 gb->gbphy_latency_ts = 0;
ab81bb9c 860 memset(&gb->ts, 0, sizeof(struct timeval));
355a7058
AB		 861}
862
a6e7e535 863static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val)
355a7058 864{
00af6583
BD		 865 if (stats->min > val)
866 stats->min = val;
867 if (stats->max < val)
868 stats->max = val;
869 stats->sum += val;
870 stats->count++;
355a7058
AB		 871}
872
ab81bb9c
AB		 873static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats,
874 u64 val, u32 count)
875{
876 stats->sum += val;
877 stats->count += count;
878
879 do_div(val, count);
880 if (stats->min > val)
881 stats->min = val;
882 if (stats->max < val)
883 stats->max = val;
884}
885
583cbf50 886static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency)
355a7058 887{
ab81bb9c 888 u64 req = gb->requests_completed * USEC_PER_SEC;
00af6583 889
ab81bb9c 890 gb_loopback_update_stats_window(&gb->requests_per_second, req, latency);
355a7058
AB		 891}
892
00af6583 893static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency)
355a7058 894{
ab81bb9c 895 u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2;
f7908e4d 896
8e1d6c33 897 switch (gb->type) {
f7908e4d
BD		 898 case GB_LOOPBACK_TYPE_PING:
899 break;
900 case GB_LOOPBACK_TYPE_SINK:
901 aggregate_size += sizeof(struct gb_loopback_transfer_request) +
8e1d6c33 902 gb->size;
f7908e4d
BD		 903 break;
904 case GB_LOOPBACK_TYPE_TRANSFER:
905 aggregate_size += sizeof(struct gb_loopback_transfer_request) +
906 sizeof(struct gb_loopback_transfer_response) +
8e1d6c33 907 gb->size * 2;
f7908e4d
BD		 908 break;
909 default:
910 return;
911 }
00af6583 912
ab81bb9c
AB		 913 aggregate_size *= gb->requests_completed;
914 aggregate_size *= USEC_PER_SEC;
915 gb_loopback_update_stats_window(&gb->throughput, aggregate_size,
916 latency);
355a7058
AB		 917}
918
ab81bb9c 919static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb)
355a7058
AB		 920{
921 u32 lat;
355a7058 922
00af6583 923 /* Express latency in terms of microseconds */
bd416103 924 lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs);
355a7058 925
98676ca8 926 /* Log latency stastic */
00af6583 927 gb_loopback_update_stats(&gb->latency, lat);
67d1eece
BD		 928
929 /* Raw latency log on a per thread basis */
4b0ea00c 930 kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat));
00af6583 931
1ec5843e 932 /* Log the firmware supplied latency values */
1ec5843e
BD		 933 gb_loopback_update_stats(&gb->apbridge_unipro_latency,
934 gb->apbridge_latency_ts);
e54b106d
SP		 935 gb_loopback_update_stats(&gb->gbphy_firmware_latency,
936 gb->gbphy_latency_ts);
355a7058
AB		 937}
938
ab81bb9c
AB		 939static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error)
940{
941 u64 nlat;
942 u32 lat;
943 struct timeval te;
944
945 if (!error) {
946 gb->requests_completed++;
947 gb_loopback_calculate_latency_stats(gb);
948 }
949
950 do_gettimeofday(&te);
951 nlat = gb_loopback_calc_latency(&gb->ts, &te);
952 if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) {
953 lat = gb_loopback_nsec_to_usec_latency(nlat);
954
955 gb_loopback_throughput_update(gb, lat);
956 gb_loopback_requests_update(gb, lat);
957
958 if (gb->iteration_count != gb->iteration_max) {
959 gb->ts = te;
960 gb->requests_completed = 0;
961 }
962 }
963}
964
8e3fba55
BD		 965static void gb_loopback_async_wait_to_send(struct gb_loopback *gb)
966{
967 if (!(gb->async && gb->outstanding_operations_max))
968 return;
969 wait_event_interruptible(gb->wq_completion,
970 (atomic_read(&gb->outstanding_operations) <
971 gb->outstanding_operations_max) ||
972 kthread_should_stop());
973}
974
355a7058
AB		 975static int gb_loopback_fn(void *data)
976{
977 int error = 0;
b36f04fa 978 int us_wait = 0;
67d1eece 979 int type;
e854ff58 980 int ret;
67d1eece 981 u32 size;
01480ba3 982
3f12c3ed 983 struct gb_loopback *gb = data;
e854ff58
AH		 984 struct gb_bundle *bundle = gb->connection->bundle;
985
986 ret = gb_pm_runtime_get_sync(bundle);
987 if (ret)
988 return ret;
355a7058 989
3dfe8aaa 990 while (1) {
e854ff58
AH		 991 if (!gb->type) {
992 gb_pm_runtime_put_autosuspend(bundle);
8e1d6c33 993 wait_event_interruptible(gb->wq, gb->type ||
3dfe8aaa 994 kthread_should_stop());
e854ff58
AH		 995 ret = gb_pm_runtime_get_sync(bundle);
996 if (ret)
997 return ret;
998 }
999
8e3fba55
BD		 1000 if (kthread_should_stop())
1001 break;
12927835 1002
8e3fba55
BD		 1003 /* Limit the maximum number of in-flight async operations */
1004 gb_loopback_async_wait_to_send(gb);
3dfe8aaa
BD		 1005 if (kthread_should_stop())
1006 break;
85d678c0 1007
67d1eece 1008 mutex_lock(&gb->mutex);
8e1d6c33
BD		 1009
1010 /* Optionally terminate */
01480ba3 1011 if (gb->send_count == gb->iteration_max) {
a072a72d
BD		 1012 mutex_unlock(&gb->mutex);
1013
1014 /* Wait for synchronous and asynchronus completion */
1015 gb_loopback_async_wait_all(gb);
1016
1017 /* Mark complete unless user-space has poked us */
1018 mutex_lock(&gb->mutex);
ab81bb9c
AB		 1019 if (gb->iteration_count == gb->iteration_max) {
1020 gb->type = 0;
01480ba3 1021 gb->send_count = 0;
39c2787b
AH		 1022 sysfs_notify(&gb->dev->kobj, NULL,
1023 "iteration_count");
a072a72d
BD		 1024 dev_dbg(&bundle->dev, "load test complete\n");
1025 } else {
1026 dev_dbg(&bundle->dev,
1027 "continuing on with new test set\n");
ab81bb9c 1028 }
67d1eece 1029 mutex_unlock(&gb->mutex);
8e1d6c33 1030 continue;
67d1eece 1031 }
8e1d6c33 1032 size = gb->size;
b36f04fa 1033 us_wait = gb->us_wait;
8e1d6c33 1034 type = gb->type;
ab81bb9c
AB		 1035 if (gb->ts.tv_usec == 0 && gb->ts.tv_sec == 0)
1036 do_gettimeofday(&gb->ts);
d9fb3754
BD		 1037 mutex_unlock(&gb->mutex);
1038
67d1eece 1039 /* Else operations to perform */
12927835 1040 if (gb->async) {
a1f10771 1041 if (type == GB_LOOPBACK_TYPE_PING)
12927835 1042 error = gb_loopback_async_ping(gb);
a1f10771 1043 else if (type == GB_LOOPBACK_TYPE_TRANSFER)
12927835 1044 error = gb_loopback_async_transfer(gb, size);
a1f10771 1045 else if (type == GB_LOOPBACK_TYPE_SINK)
12927835 1046 error = gb_loopback_async_sink(gb, size);
12927835
BD		 1047
1048 if (error)
1049 gb->error++;
1050 } else {
1051 /* We are effectively single threaded here */
1052 if (type == GB_LOOPBACK_TYPE_PING)
1053 error = gb_loopback_sync_ping(gb);
1054 else if (type == GB_LOOPBACK_TYPE_TRANSFER)
1055 error = gb_loopback_sync_transfer(gb, size);
1056 else if (type == GB_LOOPBACK_TYPE_SINK)
1057 error = gb_loopback_sync_sink(gb, size);
1058
1059 if (error)
1060 gb->error++;
1061 gb->iteration_count++;
ab81bb9c 1062 gb_loopback_calculate_stats(gb, !!error);
12927835 1063 }
01480ba3 1064 gb->send_count++;
33b8807a
JH		 1065
1066 if (us_wait) {
1067 if (us_wait < 20000)
1068 usleep_range(us_wait, us_wait + 100);
1069 else
1070 msleep(us_wait / 1000);
1071 }
355a7058 1072 }
e854ff58
AH		 1073
1074 gb_pm_runtime_put_autosuspend(bundle);
1075
355a7058
AB		 1076 return 0;
1077}
1078
4b0ea00c
BD		 1079static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s,
1080 struct kfifo *kfifo,
1081 struct mutex *mutex)
aa27bf82 1082{
aa27bf82
BD		 1083 u32 latency;
1084 int retval;
1085
4b0ea00c 1086 if (kfifo_len(kfifo) == 0) {
aa27bf82
BD		 1087 retval = -EAGAIN;
1088 goto done;
1089 }
1090
4b0ea00c
BD		 1091 mutex_lock(mutex);
1092 retval = kfifo_out(kfifo, &latency, sizeof(latency));
aa27bf82
BD		 1093 if (retval > 0) {
1094 seq_printf(s, "%u", latency);
1095 retval = 0;
1096 }
4b0ea00c 1097 mutex_unlock(mutex);
aa27bf82
BD		 1098done:
1099 return retval;
1100}
1101
4b0ea00c
BD		 1102static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused)
1103{
1104 struct gb_loopback *gb = s->private;
1105
1106 return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat,
1107 &gb->mutex);
1108}
1109
aa27bf82
BD		 1110static int gb_loopback_latency_open(struct inode *inode, struct file *file)
1111{
1112 return single_open(file, gb_loopback_dbgfs_latency_show,
1113 inode->i_private);
1114}
1115
1116static const struct file_operations gb_loopback_debugfs_latency_ops = {
1117 .open = gb_loopback_latency_open,
1118 .read = seq_read,
1119 .llseek = seq_lseek,
1120 .release = single_release,
1121};
1122
5015115d
BD		 1123static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha,
1124 struct list_head *lhb)
1125{
1126 struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry);
1127 struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry);
1128 struct gb_connection *ca = a->connection;
1129 struct gb_connection *cb = b->connection;
1130
5015115d
BD		 1131 if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id)
1132 return -1;
1133 if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id)
1134 return 1;
1135 if (ca->bundle->id < cb->bundle->id)
1136 return -1;
1137 if (cb->bundle->id < ca->bundle->id)
1138 return 1;
1139 if (ca->intf_cport_id < cb->intf_cport_id)
1140 return -1;
1141 else if (cb->intf_cport_id < ca->intf_cport_id)
1142 return 1;
1143
1144 return 0;
1145}
1146
1147static void gb_loopback_insert_id(struct gb_loopback *gb)
1148{
1149 struct gb_loopback *gb_list;
1150 u32 new_lbid = 0;
1151
1152 /* perform an insertion sort */
1153 list_add_tail(&gb->entry, &gb_dev.list);
1154 list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare);
1155 list_for_each_entry(gb_list, &gb_dev.list, entry) {
1156 gb_list->lbid = 1 << new_lbid;
1157 new_lbid++;
1158 }
1159}
1160
67d1eece 1161#define DEBUGFS_NAMELEN 32
aa27bf82 1162
e82a11dc
VK		 1163static int gb_loopback_probe(struct gb_bundle *bundle,
1164 const struct greybus_bundle_id *id)
355a7058 1165{
e82a11dc
VK		 1166 struct greybus_descriptor_cport *cport_desc;
1167 struct gb_connection *connection;
355a7058 1168 struct gb_loopback *gb;
079fa32b 1169 struct device *dev;
355a7058 1170 int retval;
aa27bf82 1171 char name[DEBUGFS_NAMELEN];
2e238d71 1172 unsigned long flags;
355a7058 1173
e82a11dc
VK		 1174 if (bundle->num_cports != 1)
1175 return -ENODEV;
1176
1177 cport_desc = &bundle->cport_desc[0];
1178 if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK)
1179 return -ENODEV;
1180
355a7058
AB		 1181 gb = kzalloc(sizeof(*gb), GFP_KERNEL);
1182 if (!gb)
1183 return -ENOMEM;
1184
e82a11dc
VK		 1185 connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id),
1186 gb_loopback_request_handler);
1187 if (IS_ERR(connection)) {
1188 retval = PTR_ERR(connection);
1189 goto out_kzalloc;
1190 }
1191
1192 gb->connection = connection;
1193 greybus_set_drvdata(bundle, gb);
1194
8e1d6c33 1195 init_waitqueue_head(&gb->wq);
36f241ff
BD		 1196 init_waitqueue_head(&gb->wq_completion);
1197 atomic_set(&gb->outstanding_operations, 0);
8e1d6c33
BD		 1198 gb_loopback_reset_stats(gb);
1199
12927835
BD		 1200 /* Reported values to user-space for min/max timeouts */
1201 gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN);
1202 gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX);
1203
f06272b2 1204 if (!gb_dev.count) {
f06272b2
BD		 1205 /* Calculate maximum payload */
1206 gb_dev.size_max = gb_operation_get_payload_size_max(connection);
1207 if (gb_dev.size_max <=
1208 sizeof(struct gb_loopback_transfer_request)) {
1209 retval = -EINVAL;
e82a11dc 1210 goto out_connection_destroy;
f06272b2
BD		 1211 }
1212 gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request);
1213 }
f06272b2
BD		 1214
1215 /* Create per-connection sysfs and debugfs data-points */
8d8d36da 1216 snprintf(name, sizeof(name), "raw_latency_%s",
d9a9ea1b 1217 dev_name(&connection->bundle->dev));
563a8412 1218 gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb,
aa27bf82 1219 &gb_loopback_debugfs_latency_ops);
079fa32b
AH		 1220
1221 gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL);
1222 if (gb->id < 0) {
1223 retval = gb->id;
e82a11dc 1224 goto out_debugfs_remove;
079fa32b
AH		 1225 }
1226
e82a11dc
VK		 1227 retval = gb_connection_enable(connection);
1228 if (retval)
1229 goto out_ida_remove;
1230
079fa32b
AH		 1231 dev = device_create_with_groups(&loopback_class,
1232 &connection->bundle->dev,
1233 MKDEV(0, 0), gb, loopback_groups,
1234 "gb_loopback%d", gb->id);
1235 if (IS_ERR(dev)) {
1236 retval = PTR_ERR(dev);
e82a11dc 1237 goto out_connection_disable;
079fa32b
AH		 1238 }
1239 gb->dev = dev;
c3bba87a 1240
cbd204b4 1241 /* Allocate kfifo */
4b0ea00c 1242 if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32),
cbd204b4
BD		 1243 GFP_KERNEL)) {
1244 retval = -ENOMEM;
079fa32b 1245 goto out_conn;
cbd204b4 1246 }
4b0ea00c
BD		 1247 if (kfifo_alloc(&gb->kfifo_ts, kfifo_depth * sizeof(struct timeval) * 2,
1248 GFP_KERNEL)) {
1249 retval = -ENOMEM;
1250 goto out_kfifo0;
1251 }
cbd204b4
BD		 1252
1253 /* Fork worker thread */
85d678c0 1254 mutex_init(&gb->mutex);
355a7058
AB		 1255 gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback");
1256 if (IS_ERR(gb->task)) {
69f60347 1257 retval = PTR_ERR(gb->task);
4b0ea00c 1258 goto out_kfifo1;
355a7058
AB		 1259 }
1260
2e238d71 1261 spin_lock_irqsave(&gb_dev.lock, flags);
5015115d 1262 gb_loopback_insert_id(gb);
aa27bf82 1263 gb_dev.count++;
2e238d71
BD		 1264 spin_unlock_irqrestore(&gb_dev.lock, flags);
1265
1266 gb_connection_latency_tag_enable(connection);
e854ff58
AH		 1267
1268 gb_pm_runtime_put_autosuspend(bundle);
1269
355a7058
AB		 1270 return 0;
1271
4b0ea00c
BD		 1272out_kfifo1:
1273 kfifo_free(&gb->kfifo_ts);
1274out_kfifo0:
1275 kfifo_free(&gb->kfifo_lat);
079fa32b
AH		 1276out_conn:
1277 device_unregister(dev);
e82a11dc
VK		 1278out_connection_disable:
1279 gb_connection_disable(connection);
1280out_ida_remove:
079fa32b 1281 ida_simple_remove(&loopback_ida, gb->id);
e82a11dc 1282out_debugfs_remove:
aa27bf82 1283 debugfs_remove(gb->file);
e82a11dc
VK		 1284out_connection_destroy:
1285 gb_connection_destroy(connection);
2e238d71 1286out_kzalloc:
355a7058 1287 kfree(gb);
1fb807cf 1288
355a7058
AB		 1289 return retval;
1290}
1291
e82a11dc 1292static void gb_loopback_disconnect(struct gb_bundle *bundle)
355a7058 1293{
e82a11dc 1294 struct gb_loopback *gb = greybus_get_drvdata(bundle);
2e238d71 1295 unsigned long flags;
e854ff58
AH		 1296 int ret;
1297
1298 ret = gb_pm_runtime_get_sync(bundle);
1299 if (ret)
1300 gb_pm_runtime_get_noresume(bundle);
355a7058 1301
e82a11dc
VK		 1302 gb_connection_disable(gb->connection);
1303
355a7058
AB		 1304 if (!IS_ERR_OR_NULL(gb->task))
1305 kthread_stop(gb->task);
cbd204b4 1306
4b0ea00c
BD		 1307 kfifo_free(&gb->kfifo_lat);
1308 kfifo_free(&gb->kfifo_ts);
e82a11dc 1309 gb_connection_latency_tag_disable(gb->connection);
aa27bf82 1310 debugfs_remove(gb->file);
e82a11dc
VK		 1311
1312 /*
1313 * FIXME: gb_loopback_async_wait_all() is redundant now, as connection
1314 * is disabled at the beginning and so we can't have any more
1315 * incoming/outgoing requests.
1316 */
36f241ff 1317 gb_loopback_async_wait_all(gb);
2e238d71
BD		 1318
1319 spin_lock_irqsave(&gb_dev.lock, flags);
1320 gb_dev.count--;
ff477d07 1321 list_del(&gb->entry);
2e238d71
BD		 1322 spin_unlock_irqrestore(&gb_dev.lock, flags);
1323
079fa32b
AH		 1324 device_unregister(gb->dev);
1325 ida_simple_remove(&loopback_ida, gb->id);
1326
e82a11dc 1327 gb_connection_destroy(gb->connection);
5f3e0d17 1328 kfree(gb);
355a7058
AB		 1329}
1330
e82a11dc
VK		 1331static const struct greybus_bundle_id gb_loopback_id_table[] = {
1332 { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) },
1333 { }
1334};
1335MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table);
1336
1337static struct greybus_driver gb_loopback_driver = {
1338 .name = "loopback",
1339 .probe = gb_loopback_probe,
1340 .disconnect = gb_loopback_disconnect,
1341 .id_table = gb_loopback_id_table,
355a7058
AB		 1342};
1343
cbd204b4
BD		 1344static int loopback_init(void)
1345{
4b0ea00c
BD		 1346 int retval;
1347
67d1eece 1348 INIT_LIST_HEAD(&gb_dev.list);
12927835 1349 INIT_LIST_HEAD(&gb_dev.list_op_async);
2e238d71 1350 spin_lock_init(&gb_dev.lock);
aa27bf82 1351 gb_dev.root = debugfs_create_dir("gb_loopback", NULL);
67d1eece 1352
079fa32b
AH		 1353 retval = class_register(&loopback_class);
1354 if (retval)
1355 goto err;
1356
e82a11dc 1357 retval = greybus_register(&gb_loopback_driver);
079fa32b
AH		 1358 if (retval)
1359 goto err_unregister;
1360
1361 return 0;
4b0ea00c 1362
079fa32b
AH		 1363err_unregister:
1364 class_unregister(&loopback_class);
1365err:
4b0ea00c
BD		 1366 debugfs_remove_recursive(gb_dev.root);
1367 return retval;
cbd204b4
BD		 1368}
1369module_init(loopback_init);
1370
1371static void __exit loopback_exit(void)
1372{
aa27bf82 1373 debugfs_remove_recursive(gb_dev.root);
e82a11dc 1374 greybus_deregister(&gb_loopback_driver);
079fa32b
AH		 1375 class_unregister(&loopback_class);
1376 ida_destroy(&loopback_ida);
cbd204b4
BD		 1377}
1378module_exit(loopback_exit);
355a7058
AB		 1379
1380MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)