2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2015 Intel Corporation. All rights reserved.
8 * Copyright(c) 2017 T-Platforms. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
16 * Copyright(c) 2015 Intel Corporation. All rights reserved.
17 * Copyright(c) 2017 T-Platforms. All Rights Reserved.
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * PCIe NTB Perf Linux driver
49 * How to use this tool, by example.
51 * Assuming $DBG_DIR is something like:
52 * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
53 * Suppose aside from local device there is at least one remote device
54 * connected to NTB with index 0.
55 *-----------------------------------------------------------------------------
56 * Eg: install driver with specified chunk/total orders and dma-enabled flag
58 * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
59 *-----------------------------------------------------------------------------
60 * Eg: check NTB ports (index) and MW mapping information
62 * root@self# cat $DBG_DIR/info
63 *-----------------------------------------------------------------------------
64 * Eg: start performance test with peer (index 0) and get the test metrics
66 * root@self# echo 0 > $DBG_DIR/run
67 * root@self# cat $DBG_DIR/run
70 #include <linux/init.h>
71 #include <linux/kernel.h>
72 #include <linux/module.h>
73 #include <linux/sched.h>
74 #include <linux/wait.h>
75 #include <linux/dma-mapping.h>
76 #include <linux/dmaengine.h>
77 #include <linux/pci.h>
78 #include <linux/ktime.h>
79 #include <linux/slab.h>
80 #include <linux/delay.h>
81 #include <linux/sizes.h>
82 #include <linux/workqueue.h>
83 #include <linux/debugfs.h>
84 #include <linux/random.h>
85 #include <linux/ntb.h>
87 #define DRIVER_NAME "ntb_perf"
88 #define DRIVER_VERSION "2.0"
90 MODULE_LICENSE("Dual BSD/GPL");
91 MODULE_VERSION(DRIVER_VERSION);
92 MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
93 MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
95 #define MAX_THREADS_CNT 32
96 #define DEF_THREADS_CNT 1
97 #define MAX_CHUNK_SIZE SZ_1M
98 #define MAX_CHUNK_ORDER 20 /* no larger than 1M */
100 #define DMA_TRIES 100
101 #define DMA_MDELAY 10
103 #define MSG_TRIES 500
104 #define MSG_UDELAY_LOW 1000
105 #define MSG_UDELAY_HIGH 2000
107 #define PERF_BUF_LEN 1024
109 static unsigned long max_mw_size;
110 module_param(max_mw_size, ulong, 0644);
111 MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
113 static unsigned char chunk_order = 19; /* 512K */
114 module_param(chunk_order, byte, 0644);
115 MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
117 static unsigned char total_order = 30; /* 1G */
118 module_param(total_order, byte, 0644);
119 MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
121 static bool use_dma; /* default to 0 */
122 module_param(use_dma, bool, 0644);
123 MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
125 /*==============================================================================
126 * Perf driver data definition
127 *==============================================================================
131 PERF_CMD_INVAL = -1,/* invalid spad command */
132 PERF_CMD_SSIZE = 0, /* send out buffer size */
133 PERF_CMD_RSIZE = 1, /* recv in buffer size */
134 PERF_CMD_SXLAT = 2, /* send in buffer xlat */
135 PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
136 PERF_CMD_CLEAR = 4, /* clear allocated memory */
137 PERF_STS_DONE = 5, /* init is done */
138 PERF_STS_LNKUP = 6, /* link up state flag */
144 struct perf_ctx *perf;
148 /* Outbound MW params */
150 resource_size_t outbuf_size;
151 void __iomem *outbuf;
153 /* Inbound MW params */
154 dma_addr_t inbuf_xlat;
155 resource_size_t inbuf_size;
158 /* NTB connection setup service */
159 struct work_struct service;
162 #define to_peer_service(__work) \
163 container_of(__work, struct perf_peer, service)
166 struct perf_ctx *perf;
169 /* DMA-based test sync parameters */
171 wait_queue_head_t dma_wait;
172 struct dma_chan *dma_chan;
174 /* Data source and measured statistics */
179 struct work_struct work;
181 #define to_thread_work(__work) \
182 container_of(__work, struct perf_thread, work)
187 /* Global device index and peers descriptors */
190 struct perf_peer *peers;
192 /* Performance measuring work-threads interface */
193 unsigned long busy_flag;
194 wait_queue_head_t twait;
197 struct perf_peer *test_peer;
198 struct perf_thread threads[MAX_THREADS_CNT];
200 /* Scratchpad/Message IO operations */
201 int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
202 int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
205 struct dentry *dbgfs_dir;
209 * Scratchpads-base commands interface
211 #define PERF_SPAD_CNT(_pcnt) \
213 #define PERF_SPAD_CMD(_gidx) \
215 #define PERF_SPAD_LDATA(_gidx) \
217 #define PERF_SPAD_HDATA(_gidx) \
219 #define PERF_SPAD_NOTIFY(_gidx) \
223 * Messages-base commands interface
225 #define PERF_MSG_CNT 3
226 #define PERF_MSG_CMD 0
227 #define PERF_MSG_LDATA 1
228 #define PERF_MSG_HDATA 2
230 /*==============================================================================
231 * Static data declarations
232 *==============================================================================
235 static struct dentry *perf_dbgfs_topdir;
237 static struct workqueue_struct *perf_wq __read_mostly;
239 /*==============================================================================
240 * NTB cross-link commands execution service
241 *==============================================================================
244 static void perf_terminate_test(struct perf_ctx *perf);
246 static inline bool perf_link_is_up(struct perf_peer *peer)
250 link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
251 return !!(link & BIT_ULL_MASK(peer->pidx));
254 static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
257 struct perf_ctx *perf = peer->perf;
261 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
264 * Perform predefined number of attempts before give up.
265 * We are sending the data to the port specific scratchpad, so
266 * to prevent a multi-port access race-condition. Additionally
267 * there is no need in local locking since only thread-safe
268 * service work is using this method.
270 for (try = 0; try < MSG_TRIES; try++) {
271 if (!perf_link_is_up(peer))
274 sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
275 PERF_SPAD_CMD(perf->gidx));
276 if (sts != PERF_CMD_INVAL) {
277 usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
281 ntb_peer_spad_write(perf->ntb, peer->pidx,
282 PERF_SPAD_LDATA(perf->gidx),
283 lower_32_bits(data));
284 ntb_peer_spad_write(perf->ntb, peer->pidx,
285 PERF_SPAD_HDATA(perf->gidx),
286 upper_32_bits(data));
287 ntb_peer_spad_write(perf->ntb, peer->pidx,
288 PERF_SPAD_CMD(perf->gidx),
290 ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
292 dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
293 PERF_SPAD_NOTIFY(peer->gidx));
298 return try < MSG_TRIES ? 0 : -EAGAIN;
301 static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
302 enum perf_cmd *cmd, u64 *data)
304 struct perf_peer *peer;
307 ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
310 * We start scanning all over, since cleared DB may have been set
311 * by any peer. Yes, it makes peer with smaller index being
312 * serviced with greater priority, but it's convenient for spad
313 * and message code unification and simplicity.
315 for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
316 peer = &perf->peers[*pidx];
318 if (!perf_link_is_up(peer))
321 val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
322 if (val == PERF_CMD_INVAL)
327 val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
330 val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
331 *data |= (u64)val << 32;
333 /* Next command can be retrieved from now */
334 ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
337 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
345 static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
348 struct perf_ctx *perf = peer->perf;
352 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
355 * Perform predefined number of attempts before give up. Message
356 * registers are free of race-condition problem when accessed
357 * from different ports, so we don't need splitting registers
358 * by global device index. We also won't have local locking,
359 * since the method is used from service work only.
361 outbits = ntb_msg_outbits(perf->ntb);
362 for (try = 0; try < MSG_TRIES; try++) {
363 if (!perf_link_is_up(peer))
366 ret = ntb_msg_clear_sts(perf->ntb, outbits);
370 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
371 lower_32_bits(data));
373 if (ntb_msg_read_sts(perf->ntb) & outbits) {
374 usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
378 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
379 upper_32_bits(data));
381 /* This call shall trigger peer message event */
382 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
387 return try < MSG_TRIES ? 0 : -EAGAIN;
390 static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
391 enum perf_cmd *cmd, u64 *data)
396 inbits = ntb_msg_inbits(perf->ntb);
398 if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
401 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
404 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
407 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
408 *data |= (u64)val << 32;
410 /* Next command can be retrieved from now */
411 ntb_msg_clear_sts(perf->ntb, inbits);
413 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
418 static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
420 struct perf_ctx *perf = peer->perf;
422 if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
423 return perf->cmd_send(peer, cmd, data);
425 dev_err(&perf->ntb->dev, "Send invalid command\n");
429 static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
439 dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
443 /* No need of memory barrier, since bit ops have invernal lock */
444 set_bit(cmd, &peer->sts);
446 dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
448 (void)queue_work(system_highpri_wq, &peer->service);
453 static int perf_cmd_recv(struct perf_ctx *perf)
455 struct perf_peer *peer;
459 while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
460 peer = &perf->peers[pidx];
464 peer->inbuf_size = data;
465 return perf_cmd_exec(peer, PERF_CMD_RSIZE);
467 peer->outbuf_xlat = data;
468 return perf_cmd_exec(peer, PERF_CMD_RXLAT);
470 dev_err(&perf->ntb->dev, "Recv invalid command\n");
475 /* Return 0 if no data left to process, otherwise an error */
476 return ret == -ENODATA ? 0 : ret;
479 static void perf_link_event(void *ctx)
481 struct perf_ctx *perf = ctx;
482 struct perf_peer *peer;
486 for (pidx = 0; pidx < perf->pcnt; pidx++) {
487 peer = &perf->peers[pidx];
489 lnk_up = perf_link_is_up(peer);
492 !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
493 perf_cmd_exec(peer, PERF_CMD_SSIZE);
494 } else if (!lnk_up &&
495 test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
496 perf_cmd_exec(peer, PERF_CMD_CLEAR);
501 static void perf_db_event(void *ctx, int vec)
503 struct perf_ctx *perf = ctx;
505 dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
506 ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
508 /* Just receive all available commands */
509 (void)perf_cmd_recv(perf);
512 static void perf_msg_event(void *ctx)
514 struct perf_ctx *perf = ctx;
516 dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
517 ntb_msg_read_sts(perf->ntb));
519 /* Messages are only sent one-by-one */
520 (void)perf_cmd_recv(perf);
523 static const struct ntb_ctx_ops perf_ops = {
524 .link_event = perf_link_event,
525 .db_event = perf_db_event,
526 .msg_event = perf_msg_event
529 static void perf_free_outbuf(struct perf_peer *peer)
531 (void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
534 static int perf_setup_outbuf(struct perf_peer *peer)
536 struct perf_ctx *perf = peer->perf;
539 /* Outbuf size can be unaligned due to custom max_mw_size */
540 ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
541 peer->outbuf_xlat, peer->outbuf_size);
543 dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
547 /* Initialization is finally done */
548 set_bit(PERF_STS_DONE, &peer->sts);
553 static void perf_free_inbuf(struct perf_peer *peer)
558 (void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
559 dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,
560 peer->inbuf, peer->inbuf_xlat);
564 static int perf_setup_inbuf(struct perf_peer *peer)
566 resource_size_t xlat_align, size_align, size_max;
567 struct perf_ctx *perf = peer->perf;
570 /* Get inbound MW parameters */
571 ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
572 &xlat_align, &size_align, &size_max);
574 dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
578 if (peer->inbuf_size > size_max) {
579 dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
580 &peer->inbuf_size, &size_max);
584 peer->inbuf_size = round_up(peer->inbuf_size, size_align);
586 perf_free_inbuf(peer);
588 peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
589 &peer->inbuf_xlat, GFP_KERNEL);
591 dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
595 if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
596 dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
600 ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
601 peer->inbuf_xlat, peer->inbuf_size);
603 dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
608 * We submit inbuf xlat transmission cmd for execution here to follow
609 * the code architecture, even though this method is called from service
610 * work itself so the command will be executed right after it returns.
612 (void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
617 perf_free_inbuf(peer);
622 static void perf_service_work(struct work_struct *work)
624 struct perf_peer *peer = to_peer_service(work);
626 if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
627 perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
629 if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
630 perf_setup_inbuf(peer);
632 if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
633 perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
635 if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
636 perf_setup_outbuf(peer);
638 if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
639 clear_bit(PERF_STS_DONE, &peer->sts);
640 if (test_bit(0, &peer->perf->busy_flag) &&
641 peer == peer->perf->test_peer) {
642 dev_warn(&peer->perf->ntb->dev,
643 "Freeing while test on-fly\n");
644 perf_terminate_test(peer->perf);
646 perf_free_outbuf(peer);
647 perf_free_inbuf(peer);
651 static int perf_init_service(struct perf_ctx *perf)
655 if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {
656 dev_err(&perf->ntb->dev, "Not enough memory windows\n");
660 if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
661 perf->cmd_send = perf_msg_cmd_send;
662 perf->cmd_recv = perf_msg_cmd_recv;
664 dev_dbg(&perf->ntb->dev, "Message service initialized\n");
669 dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
671 mask = GENMASK_ULL(perf->pcnt, 0);
672 if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
673 (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
674 perf->cmd_send = perf_spad_cmd_send;
675 perf->cmd_recv = perf_spad_cmd_recv;
677 dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
682 dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
684 dev_err(&perf->ntb->dev, "Command services unsupported\n");
689 static int perf_enable_service(struct perf_ctx *perf)
694 mask = ntb_db_valid_mask(perf->ntb);
695 (void)ntb_db_set_mask(perf->ntb, mask);
697 ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
701 if (perf->cmd_send == perf_msg_cmd_send) {
704 inbits = ntb_msg_inbits(perf->ntb);
705 outbits = ntb_msg_outbits(perf->ntb);
706 (void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
708 incmd_bit = BIT_ULL(__ffs64(inbits));
709 ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
711 dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
713 scnt = ntb_spad_count(perf->ntb);
714 for (sidx = 0; sidx < scnt; sidx++)
715 ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
716 incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
717 ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
719 dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
722 ntb_clear_ctx(perf->ntb);
726 ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
727 /* Might be not necessary */
728 ntb_link_event(perf->ntb);
733 static void perf_disable_service(struct perf_ctx *perf)
737 ntb_link_disable(perf->ntb);
739 if (perf->cmd_send == perf_msg_cmd_send) {
742 inbits = ntb_msg_inbits(perf->ntb);
743 (void)ntb_msg_set_mask(perf->ntb, inbits);
745 (void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
748 ntb_clear_ctx(perf->ntb);
750 for (pidx = 0; pidx < perf->pcnt; pidx++)
751 perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
753 for (pidx = 0; pidx < perf->pcnt; pidx++)
754 flush_work(&perf->peers[pidx].service);
757 /*==============================================================================
758 * Performance measuring work-thread
759 *==============================================================================
762 static void perf_dma_copy_callback(void *data)
764 struct perf_thread *pthr = data;
766 atomic_dec(&pthr->dma_sync);
767 wake_up(&pthr->dma_wait);
770 static int perf_copy_chunk(struct perf_thread *pthr,
771 void __iomem *dst, void *src, size_t len)
773 struct dma_async_tx_descriptor *tx;
774 struct dmaengine_unmap_data *unmap;
775 struct device *dma_dev;
776 int try = 0, ret = 0;
779 memcpy_toio(dst, src, len);
780 goto ret_check_tsync;
783 dma_dev = pthr->dma_chan->device->dev;
785 if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
786 offset_in_page(dst), len))
789 unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
794 unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
795 offset_in_page(src), len, DMA_TO_DEVICE);
796 if (dma_mapping_error(dma_dev, unmap->addr[0])) {
798 goto err_free_resource;
802 unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
803 offset_in_page(dst), len, DMA_FROM_DEVICE);
804 if (dma_mapping_error(dma_dev, unmap->addr[1])) {
806 goto err_free_resource;
811 tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
812 unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
815 } while (!tx && (try++ < DMA_TRIES));
819 goto err_free_resource;
822 tx->callback = perf_dma_copy_callback;
823 tx->callback_param = pthr;
824 dma_set_unmap(tx, unmap);
826 ret = dma_submit_error(dmaengine_submit(tx));
828 dmaengine_unmap_put(unmap);
829 goto err_free_resource;
832 dmaengine_unmap_put(unmap);
834 atomic_inc(&pthr->dma_sync);
835 dma_async_issue_pending(pthr->dma_chan);
838 return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
841 dmaengine_unmap_put(unmap);
846 static bool perf_dma_filter(struct dma_chan *chan, void *data)
848 struct perf_ctx *perf = data;
851 node = dev_to_node(&perf->ntb->dev);
853 return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
856 static int perf_init_test(struct perf_thread *pthr)
858 struct perf_ctx *perf = pthr->perf;
859 dma_cap_mask_t dma_mask;
861 pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
862 dev_to_node(&perf->ntb->dev));
866 get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
871 dma_cap_zero(dma_mask);
872 dma_cap_set(DMA_MEMCPY, dma_mask);
873 pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
874 if (!pthr->dma_chan) {
875 dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
877 atomic_dec(&perf->tsync);
878 wake_up(&perf->twait);
883 atomic_set(&pthr->dma_sync, 0);
888 static int perf_run_test(struct perf_thread *pthr)
890 struct perf_peer *peer = pthr->perf->test_peer;
891 struct perf_ctx *perf = pthr->perf;
892 void __iomem *flt_dst, *bnd_dst;
893 u64 total_size, chunk_size;
897 total_size = 1ULL << total_order;
898 chunk_size = 1ULL << chunk_order;
899 chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
902 bnd_dst = peer->outbuf + peer->outbuf_size;
903 flt_dst = peer->outbuf;
905 pthr->duration = ktime_get();
907 /* Copied field is cleared on test launch stage */
908 while (pthr->copied < total_size) {
909 ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
911 dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
916 pthr->copied += chunk_size;
918 flt_dst += chunk_size;
919 flt_src += chunk_size;
920 if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
921 flt_dst = peer->outbuf;
925 /* Give up CPU to give a chance for other threads to use it */
932 static int perf_sync_test(struct perf_thread *pthr)
934 struct perf_ctx *perf = pthr->perf;
939 wait_event(pthr->dma_wait,
940 (atomic_read(&pthr->dma_sync) == 0 ||
941 atomic_read(&perf->tsync) < 0));
943 if (atomic_read(&perf->tsync) < 0)
947 pthr->duration = ktime_sub(ktime_get(), pthr->duration);
949 dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
950 pthr->tidx, pthr->copied);
952 dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
953 pthr->tidx, ktime_to_us(pthr->duration));
955 dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
956 div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
961 static void perf_clear_test(struct perf_thread *pthr)
963 struct perf_ctx *perf = pthr->perf;
969 * If test finished without errors, termination isn't needed.
970 * We call it anyway just to be sure of the transfers completion.
972 (void)dmaengine_terminate_sync(pthr->dma_chan);
974 dma_release_channel(pthr->dma_chan);
977 atomic_dec(&perf->tsync);
978 wake_up(&perf->twait);
982 static void perf_thread_work(struct work_struct *work)
984 struct perf_thread *pthr = to_thread_work(work);
988 * Perform stages in compliance with use_dma flag value.
989 * Test status is changed only if error happened, otherwise
990 * status -ENODATA is kept while test is on-fly. Results
991 * synchronization is performed only if test fininshed
992 * without an error or interruption.
994 ret = perf_init_test(pthr);
1000 ret = perf_run_test(pthr);
1003 goto err_clear_test;
1006 pthr->status = perf_sync_test(pthr);
1009 perf_clear_test(pthr);
1012 static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
1014 if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
1017 if (test_and_set_bit_lock(0, &perf->busy_flag))
1022 clear_bit_unlock(0, &perf->busy_flag);
1027 static void perf_terminate_test(struct perf_ctx *perf)
1031 atomic_set(&perf->tsync, -1);
1032 wake_up(&perf->twait);
1034 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1035 wake_up(&perf->threads[tidx].dma_wait);
1036 cancel_work_sync(&perf->threads[tidx].work);
1040 static int perf_submit_test(struct perf_peer *peer)
1042 struct perf_ctx *perf = peer->perf;
1043 struct perf_thread *pthr;
1046 if (!test_bit(PERF_STS_DONE, &peer->sts))
1049 if (test_and_set_bit_lock(0, &perf->busy_flag))
1052 perf->test_peer = peer;
1053 atomic_set(&perf->tsync, perf->tcnt);
1055 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1056 pthr = &perf->threads[tidx];
1058 pthr->status = -ENODATA;
1060 pthr->duration = ktime_set(0, 0);
1061 if (tidx < perf->tcnt)
1062 (void)queue_work(perf_wq, &pthr->work);
1065 ret = wait_event_interruptible(perf->twait,
1066 atomic_read(&perf->tsync) <= 0);
1067 if (ret == -ERESTARTSYS) {
1068 perf_terminate_test(perf);
1072 clear_bit_unlock(0, &perf->busy_flag);
1077 static int perf_read_stats(struct perf_ctx *perf, char *buf,
1078 size_t size, ssize_t *pos)
1080 struct perf_thread *pthr;
1083 if (test_and_set_bit_lock(0, &perf->busy_flag))
1086 (*pos) += scnprintf(buf + *pos, size - *pos,
1087 " Peer %d test statistics:\n", perf->test_peer->pidx);
1089 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1090 pthr = &perf->threads[tidx];
1092 if (pthr->status == -ENODATA)
1096 (*pos) += scnprintf(buf + *pos, size - *pos,
1097 "%d: error status %d\n", tidx, pthr->status);
1101 (*pos) += scnprintf(buf + *pos, size - *pos,
1102 "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
1103 tidx, pthr->copied, ktime_to_us(pthr->duration),
1104 div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
1107 clear_bit_unlock(0, &perf->busy_flag);
1112 static void perf_init_threads(struct perf_ctx *perf)
1114 struct perf_thread *pthr;
1117 perf->tcnt = DEF_THREADS_CNT;
1118 perf->test_peer = &perf->peers[0];
1119 init_waitqueue_head(&perf->twait);
1121 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1122 pthr = &perf->threads[tidx];
1126 pthr->status = -ENODATA;
1127 init_waitqueue_head(&pthr->dma_wait);
1128 INIT_WORK(&pthr->work, perf_thread_work);
1132 static void perf_clear_threads(struct perf_ctx *perf)
1134 perf_terminate_test(perf);
1137 /*==============================================================================
1139 *==============================================================================
1142 static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
1143 size_t size, loff_t *offp)
1145 struct perf_ctx *perf = filep->private_data;
1146 struct perf_peer *peer;
1152 buf_size = min_t(size_t, size, 0x1000U);
1154 buf = kmalloc(buf_size, GFP_KERNEL);
1158 pos += scnprintf(buf + pos, buf_size - pos,
1159 " Performance measuring tool info:\n\n");
1161 pos += scnprintf(buf + pos, buf_size - pos,
1162 "Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
1164 pos += scnprintf(buf + pos, buf_size - pos, "Test status: ");
1165 if (test_bit(0, &perf->busy_flag)) {
1166 pos += scnprintf(buf + pos, buf_size - pos,
1167 "on-fly with port %d (%d)\n",
1168 ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
1169 perf->test_peer->pidx);
1171 pos += scnprintf(buf + pos, buf_size - pos, "idle\n");
1174 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1175 peer = &perf->peers[pidx];
1177 pos += scnprintf(buf + pos, buf_size - pos,
1178 "Port %d (%d), Global index %d:\n",
1179 ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
1182 pos += scnprintf(buf + pos, buf_size - pos,
1183 "\tLink status: %s\n",
1184 test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");
1186 pos += scnprintf(buf + pos, buf_size - pos,
1187 "\tOut buffer addr 0x%pK\n", peer->outbuf);
1189 pos += scnprintf(buf + pos, buf_size - pos,
1190 "\tOut buffer size %pa\n", &peer->outbuf_size);
1192 pos += scnprintf(buf + pos, buf_size - pos,
1193 "\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);
1196 pos += scnprintf(buf + pos, buf_size - pos,
1197 "\tIn buffer addr: unallocated\n");
1201 pos += scnprintf(buf + pos, buf_size - pos,
1202 "\tIn buffer addr 0x%pK\n", peer->inbuf);
1204 pos += scnprintf(buf + pos, buf_size - pos,
1205 "\tIn buffer size %pa\n", &peer->inbuf_size);
1207 pos += scnprintf(buf + pos, buf_size - pos,
1208 "\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);
1211 ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
1217 static const struct file_operations perf_dbgfs_info = {
1218 .open = simple_open,
1219 .read = perf_dbgfs_read_info
1222 static ssize_t perf_dbgfs_read_run(struct file *filep, char __user *ubuf,
1223 size_t size, loff_t *offp)
1225 struct perf_ctx *perf = filep->private_data;
1226 ssize_t ret, pos = 0;
1229 buf = kmalloc(PERF_BUF_LEN, GFP_KERNEL);
1233 ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
1237 ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
1244 static ssize_t perf_dbgfs_write_run(struct file *filep, const char __user *ubuf,
1245 size_t size, loff_t *offp)
1247 struct perf_ctx *perf = filep->private_data;
1248 struct perf_peer *peer;
1251 ret = kstrtoint_from_user(ubuf, size, 0, &pidx);
1255 if (pidx < 0 || pidx >= perf->pcnt)
1258 peer = &perf->peers[pidx];
1260 ret = perf_submit_test(peer);
1267 static const struct file_operations perf_dbgfs_run = {
1268 .open = simple_open,
1269 .read = perf_dbgfs_read_run,
1270 .write = perf_dbgfs_write_run
1273 static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf,
1274 size_t size, loff_t *offp)
1276 struct perf_ctx *perf = filep->private_data;
1280 pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
1282 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1285 static ssize_t perf_dbgfs_write_tcnt(struct file *filep,
1286 const char __user *ubuf,
1287 size_t size, loff_t *offp)
1289 struct perf_ctx *perf = filep->private_data;
1293 ret = kstrtou8_from_user(ubuf, size, 0, &val);
1297 ret = perf_set_tcnt(perf, val);
1304 static const struct file_operations perf_dbgfs_tcnt = {
1305 .open = simple_open,
1306 .read = perf_dbgfs_read_tcnt,
1307 .write = perf_dbgfs_write_tcnt
1310 static void perf_setup_dbgfs(struct perf_ctx *perf)
1312 struct pci_dev *pdev = perf->ntb->pdev;
1314 perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
1315 if (!perf->dbgfs_dir) {
1316 dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
1320 debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
1323 debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
1326 debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
1329 /* They are made read-only for test exec safety and integrity */
1330 debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
1332 debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
1334 debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
1337 static void perf_clear_dbgfs(struct perf_ctx *perf)
1339 debugfs_remove_recursive(perf->dbgfs_dir);
1342 /*==============================================================================
1343 * Basic driver initialization
1344 *==============================================================================
1347 static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)
1349 struct perf_ctx *perf;
1351 perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
1353 return ERR_PTR(-ENOMEM);
1355 perf->pcnt = ntb_peer_port_count(ntb);
1356 perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
1359 return ERR_PTR(-ENOMEM);
1366 static int perf_setup_peer_mw(struct perf_peer *peer)
1368 struct perf_ctx *perf = peer->perf;
1369 phys_addr_t phys_addr;
1372 /* Get outbound MW parameters and map it */
1373 ret = ntb_peer_mw_get_addr(perf->ntb, peer->gidx, &phys_addr,
1374 &peer->outbuf_size);
1378 peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
1383 if (max_mw_size && peer->outbuf_size > max_mw_size) {
1384 peer->outbuf_size = max_mw_size;
1385 dev_warn(&peer->perf->ntb->dev,
1386 "Peer %d outbuf reduced to %pa\n", peer->pidx,
1387 &peer->outbuf_size);
1393 static int perf_init_peers(struct perf_ctx *perf)
1395 struct perf_peer *peer;
1396 int pidx, lport, ret;
1398 lport = ntb_port_number(perf->ntb);
1400 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1401 peer = &perf->peers[pidx];
1405 if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
1406 if (perf->gidx == -1)
1408 peer->gidx = pidx + 1;
1412 INIT_WORK(&peer->service, perf_service_work);
1414 if (perf->gidx == -1)
1417 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1418 ret = perf_setup_peer_mw(&perf->peers[pidx]);
1423 dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
1428 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
1430 struct perf_ctx *perf;
1433 perf = perf_create_data(ntb);
1435 return PTR_ERR(perf);
1437 ret = perf_init_peers(perf);
1441 perf_init_threads(perf);
1443 ret = perf_init_service(perf);
1447 ret = perf_enable_service(perf);
1451 perf_setup_dbgfs(perf);
1456 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
1458 struct perf_ctx *perf = ntb->ctx;
1460 perf_clear_dbgfs(perf);
1462 perf_disable_service(perf);
1464 perf_clear_threads(perf);
1467 static struct ntb_client perf_client = {
1469 .probe = perf_probe,
1470 .remove = perf_remove
1474 static int __init perf_init(void)
1478 if (chunk_order > MAX_CHUNK_ORDER) {
1479 chunk_order = MAX_CHUNK_ORDER;
1480 pr_info("Chunk order reduced to %hhu\n", chunk_order);
1483 if (total_order < chunk_order) {
1484 total_order = chunk_order;
1485 pr_info("Total data order reduced to %hhu\n", total_order);
1488 perf_wq = alloc_workqueue("perf_wq", WQ_UNBOUND | WQ_SYSFS, 0);
1492 if (debugfs_initialized())
1493 perf_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1495 ret = ntb_register_client(&perf_client);
1497 debugfs_remove_recursive(perf_dbgfs_topdir);
1498 destroy_workqueue(perf_wq);
1503 module_init(perf_init);
1505 static void __exit perf_exit(void)
1507 ntb_unregister_client(&perf_client);
1508 debugfs_remove_recursive(perf_dbgfs_topdir);
1509 destroy_workqueue(perf_wq);
1511 module_exit(perf_exit);