2 * CARMA DATA-FPGA Access Driver
4 * Copyright (c) 2009-2011 Ira W. Snyder <iws@ovro.caltech.edu>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; either version 2 of the License, or (at your
9 * option) any later version.
13 * FPGA Memory Dump Format
15 * FPGA #0 control registers (32 x 32-bit words)
16 * FPGA #1 control registers (32 x 32-bit words)
17 * FPGA #2 control registers (32 x 32-bit words)
18 * FPGA #3 control registers (32 x 32-bit words)
19 * SYSFPGA control registers (32 x 32-bit words)
20 * FPGA #0 correlation array (NUM_CORL0 correlation blocks)
21 * FPGA #1 correlation array (NUM_CORL1 correlation blocks)
22 * FPGA #2 correlation array (NUM_CORL2 correlation blocks)
23 * FPGA #3 correlation array (NUM_CORL3 correlation blocks)
25 * Each correlation array consists of:
27 * Correlation Data (2 x NUM_LAGSn x 32-bit words)
28 * Pipeline Metadata (2 x NUM_METAn x 32-bit words)
29 * Quantization Counters (2 x NUM_QCNTn x 32-bit words)
31 * The NUM_CORLn, NUM_LAGSn, NUM_METAn, and NUM_QCNTn values come from
32 * the FPGA configuration registers. They do not change once the FPGA's
33 * have been programmed, they only change on re-programming.
39 * This driver is used to capture correlation spectra off of the four data
40 * processing FPGAs. The FPGAs are often reprogrammed at runtime, therefore
41 * this driver supports dynamic enable/disable of capture while the device
44 * The nominal capture rate is 64Hz (every 15.625ms). To facilitate this fast
45 * capture rate, all buffers are pre-allocated to avoid any potentially long
46 * running memory allocations while capturing.
48 * There are two lists and one pointer which are used to keep track of the
49 * different states of data buffers.
52 * This list holds all empty data buffers which are ready to receive data.
55 * This pointer holds the currently inflight data buffer. This buffer is having
56 * data copied into it by the DMA engine.
59 * This list holds data buffers which have been filled, and are waiting to be
62 * All buffers start life on the free list, then move successively to the
63 * inflight pointer, and then to the used list. After they have been read by
64 * userspace, they are moved back to the free list. The cycle repeats as long
67 * It should be noted that all buffers are mapped and ready for DMA when they
68 * are on any of the three lists. They are only unmapped when they are in the
69 * process of being read by userspace.
73 * Notes on the IRQ masking scheme:
75 * The IRQ masking scheme here is different than most other hardware. The only
76 * way for the DATA-FPGAs to detect if the kernel has taken too long to copy
77 * the data is if the status registers are not cleared before the next
78 * correlation data dump is ready.
80 * The interrupt line is connected to the status registers, such that when they
81 * are cleared, the interrupt is de-asserted. Therein lies our problem. We need
82 * to schedule a long-running DMA operation and return from the interrupt
83 * handler quickly, but we cannot clear the status registers.
85 * To handle this, the system controller FPGA has the capability to connect the
86 * interrupt line to a user-controlled GPIO pin. This pin is driven high
87 * (unasserted) and left that way. To mask the interrupt, we change the
88 * interrupt source to the GPIO pin. Tada, we hid the interrupt. :)
91 #include <linux/of_address.h>
92 #include <linux/of_irq.h>
93 #include <linux/of_platform.h>
94 #include <linux/dma-mapping.h>
95 #include <linux/miscdevice.h>
96 #include <linux/interrupt.h>
97 #include <linux/dmaengine.h>
98 #include <linux/seq_file.h>
99 #include <linux/highmem.h>
100 #include <linux/debugfs.h>
101 #include <linux/vmalloc.h>
102 #include <linux/kernel.h>
103 #include <linux/module.h>
104 #include <linux/poll.h>
105 #include <linux/slab.h>
106 #include <linux/kref.h>
107 #include <linux/io.h>
109 /* system controller registers */
110 #define SYS_IRQ_SOURCE_CTL 0x24
111 #define SYS_IRQ_OUTPUT_EN 0x28
112 #define SYS_IRQ_OUTPUT_DATA 0x2C
113 #define SYS_IRQ_INPUT_DATA 0x30
114 #define SYS_FPGA_CONFIG_STATUS 0x44
116 /* GPIO IRQ line assignment */
117 #define IRQ_CORL_DONE 0x10
120 #define MMAP_REG_VERSION 0x00
121 #define MMAP_REG_CORL_CONF1 0x08
122 #define MMAP_REG_CORL_CONF2 0x0C
123 #define MMAP_REG_STATUS 0x48
125 #define SYS_FPGA_BLOCK 0xF0000000
127 #define DATA_FPGA_START 0x400000
128 #define DATA_FPGA_SIZE 0x80000
130 static const char drv_name[] = "carma-fpga";
134 #define MIN_DATA_BUFS 8
135 #define MAX_DATA_BUFS 64
138 unsigned int num_lag_ram;
139 unsigned int blk_size;
143 struct list_head entry;
145 struct scatterlist *sglist;
152 /* character device */
153 struct miscdevice miscdev;
157 /* reference count */
160 /* FPGA registers and information */
161 struct fpga_info info[NUM_FPGA];
165 /* FPGA Physical Address/Size Information */
166 resource_size_t phys_addr;
170 struct sg_table corl_table;
171 unsigned int corl_nents;
172 struct dma_chan *chan;
174 /* Protection for all members below */
177 /* Device enable/disable flag */
180 /* Correlation data buffers */
181 wait_queue_head_t wait;
182 struct list_head free;
183 struct list_head used;
184 struct data_buf *inflight;
186 /* Information about data buffers */
187 unsigned int num_dropped;
188 unsigned int num_buffers;
190 struct dentry *dbg_entry;
194 struct fpga_device *priv;
195 struct data_buf *buf;
199 static void fpga_device_release(struct kref *ref)
201 struct fpga_device *priv = container_of(ref, struct fpga_device, ref);
203 /* the last reader has exited, cleanup the last bits */
204 mutex_destroy(&priv->mutex);
209 * Data Buffer Allocation Helpers
212 static int carma_dma_init(struct data_buf *buf, int nr_pages)
217 buf->vaddr = vmalloc_32(nr_pages << PAGE_SHIFT);
218 if (NULL == buf->vaddr) {
219 pr_debug("vmalloc_32(%d pages) failed\n", nr_pages);
223 pr_debug("vmalloc is at addr 0x%08lx, size=%d\n",
224 (unsigned long)buf->vaddr,
225 nr_pages << PAGE_SHIFT);
227 memset(buf->vaddr, 0, nr_pages << PAGE_SHIFT);
228 buf->nr_pages = nr_pages;
230 buf->sglist = vzalloc(buf->nr_pages * sizeof(*buf->sglist));
231 if (NULL == buf->sglist)
234 sg_init_table(buf->sglist, buf->nr_pages);
235 for (i = 0; i < buf->nr_pages; i++) {
236 pg = vmalloc_to_page(buf->vaddr + i * PAGE_SIZE);
238 goto vmalloc_to_page_err;
239 sg_set_page(&buf->sglist[i], pg, PAGE_SIZE, 0);
252 static int carma_dma_map(struct device *dev, struct data_buf *buf)
254 buf->sglen = dma_map_sg(dev, buf->sglist,
255 buf->nr_pages, DMA_FROM_DEVICE);
257 if (0 == buf->sglen) {
258 pr_warn("%s: dma_map_sg failed\n", __func__);
264 static int carma_dma_unmap(struct device *dev, struct data_buf *buf)
269 dma_unmap_sg(dev, buf->sglist, buf->sglen, DMA_FROM_DEVICE);
275 * data_free_buffer() - free a single data buffer and all allocated memory
276 * @buf: the buffer to free
278 * This will free all of the pages allocated to the given data buffer, and
279 * then free the structure itself
281 static void data_free_buffer(struct data_buf *buf)
283 /* It is ok to free a NULL buffer */
287 /* free all memory */
294 * data_alloc_buffer() - allocate and fill a data buffer with pages
295 * @bytes: the number of bytes required
297 * This allocates all space needed for a data buffer. It must be mapped before
298 * use in a DMA transaction using carma_dma_map().
300 * Returns NULL on failure
302 static struct data_buf *data_alloc_buffer(const size_t bytes)
304 unsigned int nr_pages;
305 struct data_buf *buf;
308 /* calculate the number of pages necessary */
309 nr_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
311 /* allocate the buffer structure */
312 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
316 /* initialize internal fields */
317 INIT_LIST_HEAD(&buf->entry);
320 /* allocate the buffer */
321 ret = carma_dma_init(buf, nr_pages);
334 * data_free_buffers() - free all allocated buffers
335 * @priv: the driver's private data structure
337 * Free all buffers allocated by the driver (except those currently in the
338 * process of being read by userspace).
340 * LOCKING: must hold dev->mutex
343 static void data_free_buffers(struct fpga_device *priv)
345 struct data_buf *buf, *tmp;
347 /* the device should be stopped, no DMA in progress */
348 BUG_ON(priv->inflight != NULL);
350 list_for_each_entry_safe(buf, tmp, &priv->free, entry) {
351 list_del_init(&buf->entry);
352 carma_dma_unmap(priv->dev, buf);
353 data_free_buffer(buf);
356 list_for_each_entry_safe(buf, tmp, &priv->used, entry) {
357 list_del_init(&buf->entry);
358 carma_dma_unmap(priv->dev, buf);
359 data_free_buffer(buf);
362 priv->num_buffers = 0;
367 * data_alloc_buffers() - allocate 1 seconds worth of data buffers
368 * @priv: the driver's private data structure
370 * Allocate enough buffers for a whole second worth of data
372 * This routine will attempt to degrade nicely by succeeding even if a full
373 * second worth of data buffers could not be allocated, as long as a minimum
374 * number were allocated. In this case, it will print a message to the kernel
377 * The device must not be modifying any lists when this is called.
380 * LOCKING: must hold dev->mutex
382 * Returns 0 on success, -ERRNO otherwise
384 static int data_alloc_buffers(struct fpga_device *priv)
386 struct data_buf *buf;
389 for (i = 0; i < MAX_DATA_BUFS; i++) {
391 /* allocate a buffer */
392 buf = data_alloc_buffer(priv->bufsize);
397 ret = carma_dma_map(priv->dev, buf);
399 data_free_buffer(buf);
403 /* add it to the list of free buffers */
404 list_add_tail(&buf->entry, &priv->free);
408 /* Make sure we allocated the minimum required number of buffers */
409 if (priv->num_buffers < MIN_DATA_BUFS) {
410 dev_err(priv->dev, "Unable to allocate enough data buffers\n");
411 data_free_buffers(priv);
415 /* Warn if we are running in a degraded state, but do not fail */
416 if (priv->num_buffers < MAX_DATA_BUFS) {
418 "Unable to allocate %d buffers, using %d buffers instead\n",
426 * DMA Operations Helpers
430 * fpga_start_addr() - get the physical address a DATA-FPGA
431 * @priv: the driver's private data structure
432 * @fpga: the DATA-FPGA number (zero based)
434 static dma_addr_t fpga_start_addr(struct fpga_device *priv, unsigned int fpga)
436 return priv->phys_addr + 0x400000 + (0x80000 * fpga);
440 * fpga_block_addr() - get the physical address of a correlation data block
441 * @priv: the driver's private data structure
442 * @fpga: the DATA-FPGA number (zero based)
443 * @blknum: the correlation block number (zero based)
445 static dma_addr_t fpga_block_addr(struct fpga_device *priv, unsigned int fpga,
448 return fpga_start_addr(priv, fpga) + (0x10000 * (1 + blknum));
451 #define REG_BLOCK_SIZE (32 * 4)
454 * data_setup_corl_table() - create the scatterlist for correlation dumps
455 * @priv: the driver's private data structure
457 * Create the scatterlist for transferring a correlation dump from the
458 * DATA FPGAs. This structure will be reused for each buffer than needs
459 * to be filled with correlation data.
461 * Returns 0 on success, -ERRNO otherwise
463 static int data_setup_corl_table(struct fpga_device *priv)
465 struct sg_table *table = &priv->corl_table;
466 struct scatterlist *sg;
467 struct fpga_info *info;
470 /* Calculate the number of entries needed */
471 priv->corl_nents = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
472 for (i = 0; i < NUM_FPGA; i++)
473 priv->corl_nents += priv->info[i].num_lag_ram;
475 /* Allocate the scatterlist table */
476 ret = sg_alloc_table(table, priv->corl_nents, GFP_KERNEL);
478 dev_err(priv->dev, "unable to allocate DMA table\n");
482 /* Add the DATA FPGA registers to the scatterlist */
484 for (i = 0; i < NUM_FPGA; i++) {
485 sg_dma_address(sg) = fpga_start_addr(priv, i);
486 sg_dma_len(sg) = REG_BLOCK_SIZE;
490 /* Add the SYS-FPGA registers to the scatterlist */
491 sg_dma_address(sg) = SYS_FPGA_BLOCK;
492 sg_dma_len(sg) = REG_BLOCK_SIZE;
495 /* Add the FPGA correlation data blocks to the scatterlist */
496 for (i = 0; i < NUM_FPGA; i++) {
497 info = &priv->info[i];
498 for (j = 0; j < info->num_lag_ram; j++) {
499 sg_dma_address(sg) = fpga_block_addr(priv, i, j);
500 sg_dma_len(sg) = info->blk_size;
506 * All physical addresses and lengths are present in the structure
507 * now. It can be reused for every FPGA DATA interrupt
513 * FPGA Register Access Helpers
516 static void fpga_write_reg(struct fpga_device *priv, unsigned int fpga,
517 unsigned int reg, u32 val)
519 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
520 iowrite32be(val, priv->regs + fpga_start + reg);
523 static u32 fpga_read_reg(struct fpga_device *priv, unsigned int fpga,
526 const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE);
527 return ioread32be(priv->regs + fpga_start + reg);
531 * data_calculate_bufsize() - calculate the data buffer size required
532 * @priv: the driver's private data structure
534 * Calculate the total buffer size needed to hold a single block
535 * of correlation data
539 * Returns 0 on success, -ERRNO otherwise
541 static int data_calculate_bufsize(struct fpga_device *priv)
543 u32 num_corl, num_lags, num_meta, num_qcnt, num_pack;
544 u32 conf1, conf2, version;
545 u32 num_lag_ram, blk_size;
548 /* Each buffer starts with the 5 FPGA register areas */
549 priv->bufsize = (1 + NUM_FPGA) * REG_BLOCK_SIZE;
551 /* Read and store the configuration data for each FPGA */
552 for (i = 0; i < NUM_FPGA; i++) {
553 version = fpga_read_reg(priv, i, MMAP_REG_VERSION);
554 conf1 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF1);
555 conf2 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF2);
557 /* minor version 2 and later */
558 if ((version & 0x000000FF) >= 2) {
559 num_corl = (conf1 & 0x000000F0) >> 4;
560 num_pack = (conf1 & 0x00000F00) >> 8;
561 num_lags = (conf1 & 0x00FFF000) >> 12;
562 num_meta = (conf1 & 0x7F000000) >> 24;
563 num_qcnt = (conf2 & 0x00000FFF) >> 0;
565 num_corl = (conf1 & 0x000000F0) >> 4;
566 num_pack = 1; /* implied */
567 num_lags = (conf1 & 0x000FFF00) >> 8;
568 num_meta = (conf1 & 0x7FF00000) >> 20;
569 num_qcnt = (conf2 & 0x00000FFF) >> 0;
572 num_lag_ram = (num_corl + num_pack - 1) / num_pack;
573 blk_size = ((num_pack * num_lags) + num_meta + num_qcnt) * 8;
575 priv->info[i].num_lag_ram = num_lag_ram;
576 priv->info[i].blk_size = blk_size;
577 priv->bufsize += num_lag_ram * blk_size;
579 dev_dbg(priv->dev, "FPGA %d NUM_CORL: %d\n", i, num_corl);
580 dev_dbg(priv->dev, "FPGA %d NUM_PACK: %d\n", i, num_pack);
581 dev_dbg(priv->dev, "FPGA %d NUM_LAGS: %d\n", i, num_lags);
582 dev_dbg(priv->dev, "FPGA %d NUM_META: %d\n", i, num_meta);
583 dev_dbg(priv->dev, "FPGA %d NUM_QCNT: %d\n", i, num_qcnt);
584 dev_dbg(priv->dev, "FPGA %d BLK_SIZE: %d\n", i, blk_size);
587 dev_dbg(priv->dev, "TOTAL BUFFER SIZE: %zu bytes\n", priv->bufsize);
596 * data_disable_interrupts() - stop the device from generating interrupts
597 * @priv: the driver's private data structure
599 * Hide interrupts by switching to GPIO interrupt source
601 * LOCKING: must hold dev->lock
603 static void data_disable_interrupts(struct fpga_device *priv)
605 /* hide the interrupt by switching the IRQ driver to GPIO */
606 iowrite32be(0x2F, priv->regs + SYS_IRQ_SOURCE_CTL);
610 * data_enable_interrupts() - allow the device to generate interrupts
611 * @priv: the driver's private data structure
613 * Unhide interrupts by switching to the FPGA interrupt source. At the
614 * same time, clear the DATA-FPGA status registers.
616 * LOCKING: must hold dev->lock
618 static void data_enable_interrupts(struct fpga_device *priv)
620 /* clear the actual FPGA corl_done interrupt */
621 fpga_write_reg(priv, 0, MMAP_REG_STATUS, 0x0);
622 fpga_write_reg(priv, 1, MMAP_REG_STATUS, 0x0);
623 fpga_write_reg(priv, 2, MMAP_REG_STATUS, 0x0);
624 fpga_write_reg(priv, 3, MMAP_REG_STATUS, 0x0);
626 /* flush the writes */
627 fpga_read_reg(priv, 0, MMAP_REG_STATUS);
628 fpga_read_reg(priv, 1, MMAP_REG_STATUS);
629 fpga_read_reg(priv, 2, MMAP_REG_STATUS);
630 fpga_read_reg(priv, 3, MMAP_REG_STATUS);
632 /* switch back to the external interrupt source */
633 iowrite32be(0x3F, priv->regs + SYS_IRQ_SOURCE_CTL);
637 * data_dma_cb() - DMAEngine callback for DMA completion
638 * @data: the driver's private data structure
640 * Complete a DMA transfer from the DATA-FPGA's
642 * This is called via the DMA callback mechanism, and will handle moving the
643 * completed DMA transaction to the used list, and then wake any processes
644 * waiting for new data
646 * CONTEXT: any, softirq expected
648 static void data_dma_cb(void *data)
650 struct fpga_device *priv = data;
653 spin_lock_irqsave(&priv->lock, flags);
655 /* If there is no inflight buffer, we've got a bug */
656 BUG_ON(priv->inflight == NULL);
658 /* Move the inflight buffer onto the used list */
659 list_move_tail(&priv->inflight->entry, &priv->used);
660 priv->inflight = NULL;
663 * If data dumping is still enabled, then clear the FPGA
664 * status registers and re-enable FPGA interrupts
667 data_enable_interrupts(priv);
669 spin_unlock_irqrestore(&priv->lock, flags);
672 * We've changed both the inflight and used lists, so we need
673 * to wake up any processes that are blocking for those events
675 wake_up(&priv->wait);
679 * data_submit_dma() - prepare and submit the required DMA to fill a buffer
680 * @priv: the driver's private data structure
681 * @buf: the data buffer
683 * Prepare and submit the necessary DMA transactions to fill a correlation
686 * LOCKING: must hold dev->lock
687 * CONTEXT: hardirq only
689 * Returns 0 on success, -ERRNO otherwise
691 static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf)
693 struct scatterlist *dst_sg, *src_sg;
694 unsigned int dst_nents, src_nents;
695 struct dma_chan *chan = priv->chan;
696 struct dma_async_tx_descriptor *tx;
699 unsigned long dma_flags = 0;
701 dst_sg = buf->sglist;
702 dst_nents = buf->sglen;
704 src_sg = priv->corl_table.sgl;
705 src_nents = priv->corl_nents;
708 * All buffers passed to this function should be ready and mapped
709 * for DMA already. Therefore, we don't need to do anything except
710 * submit it to the Freescale DMA Engine for processing
713 /* setup the scatterlist to scatterlist transfer */
714 tx = chan->device->device_prep_dma_sg(chan,
719 dev_err(priv->dev, "unable to prep scatterlist DMA\n");
723 /* submit the transaction to the DMA controller */
724 cookie = tx->tx_submit(tx);
725 if (dma_submit_error(cookie)) {
726 dev_err(priv->dev, "unable to submit scatterlist DMA\n");
730 /* Prepare the re-read of the SYS-FPGA block */
731 dst = sg_dma_address(dst_sg) + (NUM_FPGA * REG_BLOCK_SIZE);
732 src = SYS_FPGA_BLOCK;
733 tx = chan->device->device_prep_dma_memcpy(chan, dst, src,
737 dev_err(priv->dev, "unable to prep SYS-FPGA DMA\n");
741 /* Setup the callback */
742 tx->callback = data_dma_cb;
743 tx->callback_param = priv;
745 /* submit the transaction to the DMA controller */
746 cookie = tx->tx_submit(tx);
747 if (dma_submit_error(cookie)) {
748 dev_err(priv->dev, "unable to submit SYS-FPGA DMA\n");
755 #define CORL_DONE 0x1
758 static irqreturn_t data_irq(int irq, void *dev_id)
760 struct fpga_device *priv = dev_id;
761 bool submitted = false;
762 struct data_buf *buf;
766 /* detect spurious interrupts via FPGA status */
767 for (i = 0; i < 4; i++) {
768 status = fpga_read_reg(priv, i, MMAP_REG_STATUS);
769 if (!(status & (CORL_DONE | CORL_ERR))) {
770 dev_err(priv->dev, "spurious irq detected (FPGA)\n");
775 /* detect spurious interrupts via raw IRQ pin readback */
776 status = ioread32be(priv->regs + SYS_IRQ_INPUT_DATA);
777 if (status & IRQ_CORL_DONE) {
778 dev_err(priv->dev, "spurious irq detected (IRQ)\n");
782 spin_lock(&priv->lock);
785 * This is an error case that should never happen.
787 * If this driver has a bug and manages to re-enable interrupts while
788 * a DMA is in progress, then we will hit this statement and should
789 * start paying attention immediately.
791 BUG_ON(priv->inflight != NULL);
793 /* hide the interrupt by switching the IRQ driver to GPIO */
794 data_disable_interrupts(priv);
796 /* If there are no free buffers, drop this data */
797 if (list_empty(&priv->free)) {
802 buf = list_first_entry(&priv->free, struct data_buf, entry);
803 list_del_init(&buf->entry);
804 BUG_ON(buf->size != priv->bufsize);
806 /* Submit a DMA transfer to get the correlation data */
807 if (data_submit_dma(priv, buf)) {
808 dev_err(priv->dev, "Unable to setup DMA transfer\n");
809 list_move_tail(&buf->entry, &priv->free);
813 /* Save the buffer for the DMA callback */
814 priv->inflight = buf;
817 /* Start the DMA Engine */
818 dma_async_issue_pending(priv->chan);
821 /* If no DMA was submitted, re-enable interrupts */
823 data_enable_interrupts(priv);
825 spin_unlock(&priv->lock);
830 * Realtime Device Enable Helpers
834 * data_device_enable() - enable the device for buffered dumping
835 * @priv: the driver's private data structure
837 * Enable the device for buffered dumping. Allocates buffers and hooks up
838 * the interrupt handler. When this finishes, data will come pouring in.
840 * LOCKING: must hold dev->mutex
841 * CONTEXT: user context only
843 * Returns 0 on success, -ERRNO otherwise
845 static int data_device_enable(struct fpga_device *priv)
851 /* multiple enables are safe: they do nothing */
852 spin_lock_irq(&priv->lock);
853 enabled = priv->enabled;
854 spin_unlock_irq(&priv->lock);
858 /* check that the FPGAs are programmed */
859 val = ioread32be(priv->regs + SYS_FPGA_CONFIG_STATUS);
860 if (!(val & (1 << 18))) {
861 dev_err(priv->dev, "DATA-FPGAs are not enabled\n");
865 /* read the FPGAs to calculate the buffer size */
866 ret = data_calculate_bufsize(priv);
868 dev_err(priv->dev, "unable to calculate buffer size\n");
872 /* allocate the correlation data buffers */
873 ret = data_alloc_buffers(priv);
875 dev_err(priv->dev, "unable to allocate buffers\n");
879 /* setup the source scatterlist for dumping correlation data */
880 ret = data_setup_corl_table(priv);
882 dev_err(priv->dev, "unable to setup correlation DMA table\n");
886 /* prevent the FPGAs from generating interrupts */
887 data_disable_interrupts(priv);
889 /* hookup the irq handler */
890 ret = request_irq(priv->irq, data_irq, IRQF_SHARED, drv_name, priv);
892 dev_err(priv->dev, "unable to request IRQ handler\n");
896 /* allow the DMA callback to re-enable FPGA interrupts */
897 spin_lock_irq(&priv->lock);
898 priv->enabled = true;
899 spin_unlock_irq(&priv->lock);
901 /* allow the FPGAs to generate interrupts */
902 data_enable_interrupts(priv);
906 sg_free_table(&priv->corl_table);
907 priv->corl_nents = 0;
909 data_free_buffers(priv);
914 * data_device_disable() - disable the device for buffered dumping
915 * @priv: the driver's private data structure
917 * Disable the device for buffered dumping. Stops new DMA transactions from
918 * being generated, waits for all outstanding DMA to complete, and then frees
921 * LOCKING: must hold dev->mutex
924 * Returns 0 on success, -ERRNO otherwise
926 static int data_device_disable(struct fpga_device *priv)
928 spin_lock_irq(&priv->lock);
930 /* allow multiple disable */
931 if (!priv->enabled) {
932 spin_unlock_irq(&priv->lock);
937 * Mark the device disabled
939 * This stops DMA callbacks from re-enabling interrupts
941 priv->enabled = false;
943 /* prevent the FPGAs from generating interrupts */
944 data_disable_interrupts(priv);
946 /* wait until all ongoing DMA has finished */
947 while (priv->inflight != NULL) {
948 spin_unlock_irq(&priv->lock);
949 wait_event(priv->wait, priv->inflight == NULL);
950 spin_lock_irq(&priv->lock);
953 spin_unlock_irq(&priv->lock);
955 /* unhook the irq handler */
956 free_irq(priv->irq, priv);
958 /* free the correlation table */
959 sg_free_table(&priv->corl_table);
960 priv->corl_nents = 0;
962 /* free all buffers: the free and used lists are not being changed */
963 data_free_buffers(priv);
970 #ifdef CONFIG_DEBUG_FS
973 * Count the number of entries in the given list
975 static unsigned int list_num_entries(struct list_head *list)
977 struct list_head *entry;
978 unsigned int ret = 0;
980 list_for_each(entry, list)
986 static int data_debug_show(struct seq_file *f, void *offset)
988 struct fpga_device *priv = f->private;
990 spin_lock_irq(&priv->lock);
992 seq_printf(f, "enabled: %d\n", priv->enabled);
993 seq_printf(f, "bufsize: %d\n", priv->bufsize);
994 seq_printf(f, "num_buffers: %d\n", priv->num_buffers);
995 seq_printf(f, "num_free: %d\n", list_num_entries(&priv->free));
996 seq_printf(f, "inflight: %d\n", priv->inflight != NULL);
997 seq_printf(f, "num_used: %d\n", list_num_entries(&priv->used));
998 seq_printf(f, "num_dropped: %d\n", priv->num_dropped);
1000 spin_unlock_irq(&priv->lock);
1004 static int data_debug_open(struct inode *inode, struct file *file)
1006 return single_open(file, data_debug_show, inode->i_private);
1009 static const struct file_operations data_debug_fops = {
1010 .owner = THIS_MODULE,
1011 .open = data_debug_open,
1013 .llseek = seq_lseek,
1014 .release = single_release,
1017 static int data_debugfs_init(struct fpga_device *priv)
1019 priv->dbg_entry = debugfs_create_file(drv_name, S_IRUGO, NULL, priv,
1021 return PTR_ERR_OR_ZERO(priv->dbg_entry);
1024 static void data_debugfs_exit(struct fpga_device *priv)
1026 debugfs_remove(priv->dbg_entry);
1031 static inline int data_debugfs_init(struct fpga_device *priv)
1036 static inline void data_debugfs_exit(struct fpga_device *priv)
1040 #endif /* CONFIG_DEBUG_FS */
1046 static ssize_t data_en_show(struct device *dev, struct device_attribute *attr,
1049 struct fpga_device *priv = dev_get_drvdata(dev);
1052 spin_lock_irq(&priv->lock);
1053 ret = snprintf(buf, PAGE_SIZE, "%u\n", priv->enabled);
1054 spin_unlock_irq(&priv->lock);
1059 static ssize_t data_en_set(struct device *dev, struct device_attribute *attr,
1060 const char *buf, size_t count)
1062 struct fpga_device *priv = dev_get_drvdata(dev);
1063 unsigned long enable;
1066 ret = kstrtoul(buf, 0, &enable);
1068 dev_err(priv->dev, "unable to parse enable input\n");
1072 /* protect against concurrent enable/disable */
1073 ret = mutex_lock_interruptible(&priv->mutex);
1078 ret = data_device_enable(priv);
1080 ret = data_device_disable(priv);
1083 dev_err(priv->dev, "device %s failed\n",
1084 enable ? "enable" : "disable");
1090 mutex_unlock(&priv->mutex);
1094 static DEVICE_ATTR(enable, S_IWUSR | S_IRUGO, data_en_show, data_en_set);
1096 static struct attribute *data_sysfs_attrs[] = {
1097 &dev_attr_enable.attr,
1101 static const struct attribute_group rt_sysfs_attr_group = {
1102 .attrs = data_sysfs_attrs,
1106 * FPGA Realtime Data Character Device
1109 static int data_open(struct inode *inode, struct file *filp)
1112 * The miscdevice layer puts our struct miscdevice into the
1113 * filp->private_data field. We use this to find our private
1114 * data and then overwrite it with our own private structure.
1116 struct fpga_device *priv = container_of(filp->private_data,
1117 struct fpga_device, miscdev);
1118 struct fpga_reader *reader;
1121 /* allocate private data */
1122 reader = kzalloc(sizeof(*reader), GFP_KERNEL);
1126 reader->priv = priv;
1129 filp->private_data = reader;
1130 ret = nonseekable_open(inode, filp);
1132 dev_err(priv->dev, "nonseekable-open failed\n");
1138 * success, increase the reference count of the private data structure
1139 * so that it doesn't disappear if the device is unbound
1141 kref_get(&priv->ref);
1145 static int data_release(struct inode *inode, struct file *filp)
1147 struct fpga_reader *reader = filp->private_data;
1148 struct fpga_device *priv = reader->priv;
1150 /* free the per-reader structure */
1151 data_free_buffer(reader->buf);
1153 filp->private_data = NULL;
1155 /* decrement our reference count to the private data */
1156 kref_put(&priv->ref, fpga_device_release);
1160 static ssize_t data_read(struct file *filp, char __user *ubuf, size_t count,
1163 struct fpga_reader *reader = filp->private_data;
1164 struct fpga_device *priv = reader->priv;
1165 struct list_head *used = &priv->used;
1166 bool drop_buffer = false;
1167 struct data_buf *dbuf;
1172 /* check if we already have a partial buffer */
1178 spin_lock_irq(&priv->lock);
1180 /* Block until there is at least one buffer on the used list */
1181 while (list_empty(used)) {
1182 spin_unlock_irq(&priv->lock);
1184 if (filp->f_flags & O_NONBLOCK)
1187 ret = wait_event_interruptible(priv->wait, !list_empty(used));
1191 spin_lock_irq(&priv->lock);
1194 /* Grab the first buffer off of the used list */
1195 dbuf = list_first_entry(used, struct data_buf, entry);
1196 list_del_init(&dbuf->entry);
1198 spin_unlock_irq(&priv->lock);
1200 /* Buffers are always mapped: unmap it */
1201 carma_dma_unmap(priv->dev, dbuf);
1203 /* save the buffer for later */
1205 reader->buf_start = 0;
1208 /* Get the number of bytes available */
1209 avail = dbuf->size - reader->buf_start;
1210 data = dbuf->vaddr + reader->buf_start;
1212 /* Get the number of bytes we can transfer */
1213 count = min(count, avail);
1215 /* Copy the data to the userspace buffer */
1216 if (copy_to_user(ubuf, data, count))
1219 /* Update the amount of available space */
1223 * If there is still some data available, save the buffer for the
1224 * next userspace call to read() and return
1227 reader->buf_start += count;
1233 * Get the buffer ready to be reused for DMA
1235 * If it fails, we pretend that the read never happed and return
1236 * -EFAULT to userspace. The read will be retried.
1238 ret = carma_dma_map(priv->dev, dbuf);
1240 dev_err(priv->dev, "unable to remap buffer for DMA\n");
1244 /* Lock against concurrent enable/disable */
1245 spin_lock_irq(&priv->lock);
1247 /* the reader is finished with this buffer */
1251 * One of two things has happened, the device is disabled, or the
1252 * device has been reconfigured underneath us. In either case, we
1253 * should just throw away the buffer.
1255 * Lockdep complains if this is done under the spinlock, so we
1256 * handle it during the unlock path.
1258 if (!priv->enabled || dbuf->size != priv->bufsize) {
1263 /* The buffer is safe to reuse, so add it back to the free list */
1264 list_add_tail(&dbuf->entry, &priv->free);
1267 spin_unlock_irq(&priv->lock);
1270 carma_dma_unmap(priv->dev, dbuf);
1271 data_free_buffer(dbuf);
1277 static unsigned int data_poll(struct file *filp, struct poll_table_struct *tbl)
1279 struct fpga_reader *reader = filp->private_data;
1280 struct fpga_device *priv = reader->priv;
1281 unsigned int mask = 0;
1283 poll_wait(filp, &priv->wait, tbl);
1285 if (!list_empty(&priv->used))
1286 mask |= POLLIN | POLLRDNORM;
1291 static int data_mmap(struct file *filp, struct vm_area_struct *vma)
1293 struct fpga_reader *reader = filp->private_data;
1294 struct fpga_device *priv = reader->priv;
1295 unsigned long offset, vsize, psize, addr;
1297 /* VMA properties */
1298 offset = vma->vm_pgoff << PAGE_SHIFT;
1299 vsize = vma->vm_end - vma->vm_start;
1300 psize = priv->phys_size - offset;
1301 addr = (priv->phys_addr + offset) >> PAGE_SHIFT;
1303 /* Check against the FPGA region's physical memory size */
1304 if (vsize > psize) {
1305 dev_err(priv->dev, "requested mmap mapping too large\n");
1309 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1311 return io_remap_pfn_range(vma, vma->vm_start, addr, vsize,
1315 static const struct file_operations data_fops = {
1316 .owner = THIS_MODULE,
1318 .release = data_release,
1322 .llseek = no_llseek,
1326 * OpenFirmware Device Subsystem
1329 static bool dma_filter(struct dma_chan *chan, void *data)
1332 * DMA Channel #0 is used for the FPGA Programmer, so ignore it
1334 * This probably won't survive an unload/load cycle of the Freescale
1335 * DMAEngine driver, but that won't be a problem
1337 if (chan->chan_id == 0 && chan->device->dev_id == 0)
1343 static int data_of_probe(struct platform_device *op)
1345 struct device_node *of_node = op->dev.of_node;
1346 struct device *this_device;
1347 struct fpga_device *priv;
1348 struct resource res;
1349 dma_cap_mask_t mask;
1352 /* Allocate private data */
1353 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1355 dev_err(&op->dev, "Unable to allocate device private data\n");
1360 platform_set_drvdata(op, priv);
1361 priv->dev = &op->dev;
1362 kref_init(&priv->ref);
1363 mutex_init(&priv->mutex);
1365 dev_set_drvdata(priv->dev, priv);
1366 spin_lock_init(&priv->lock);
1367 INIT_LIST_HEAD(&priv->free);
1368 INIT_LIST_HEAD(&priv->used);
1369 init_waitqueue_head(&priv->wait);
1371 /* Setup the misc device */
1372 priv->miscdev.minor = MISC_DYNAMIC_MINOR;
1373 priv->miscdev.name = drv_name;
1374 priv->miscdev.fops = &data_fops;
1376 /* Get the physical address of the FPGA registers */
1377 ret = of_address_to_resource(of_node, 0, &res);
1379 dev_err(&op->dev, "Unable to find FPGA physical address\n");
1384 priv->phys_addr = res.start;
1385 priv->phys_size = resource_size(&res);
1387 /* ioremap the registers for use */
1388 priv->regs = of_iomap(of_node, 0);
1390 dev_err(&op->dev, "Unable to ioremap registers\n");
1396 dma_cap_set(DMA_MEMCPY, mask);
1397 dma_cap_set(DMA_INTERRUPT, mask);
1398 dma_cap_set(DMA_SLAVE, mask);
1399 dma_cap_set(DMA_SG, mask);
1401 /* Request a DMA channel */
1402 priv->chan = dma_request_channel(mask, dma_filter, NULL);
1404 dev_err(&op->dev, "Unable to request DMA channel\n");
1406 goto out_unmap_regs;
1409 /* Find the correct IRQ number */
1410 priv->irq = irq_of_parse_and_map(of_node, 0);
1411 if (priv->irq == NO_IRQ) {
1412 dev_err(&op->dev, "Unable to find IRQ line\n");
1414 goto out_release_dma;
1417 /* Drive the GPIO for FPGA IRQ high (no interrupt) */
1418 iowrite32be(IRQ_CORL_DONE, priv->regs + SYS_IRQ_OUTPUT_DATA);
1420 /* Register the miscdevice */
1421 ret = misc_register(&priv->miscdev);
1423 dev_err(&op->dev, "Unable to register miscdevice\n");
1424 goto out_irq_dispose_mapping;
1427 /* Create the debugfs files */
1428 ret = data_debugfs_init(priv);
1430 dev_err(&op->dev, "Unable to create debugfs files\n");
1431 goto out_misc_deregister;
1434 /* Create the sysfs files */
1435 this_device = priv->miscdev.this_device;
1436 dev_set_drvdata(this_device, priv);
1437 ret = sysfs_create_group(&this_device->kobj, &rt_sysfs_attr_group);
1439 dev_err(&op->dev, "Unable to create sysfs files\n");
1440 goto out_data_debugfs_exit;
1443 dev_info(&op->dev, "CARMA FPGA Realtime Data Driver Loaded\n");
1446 out_data_debugfs_exit:
1447 data_debugfs_exit(priv);
1448 out_misc_deregister:
1449 misc_deregister(&priv->miscdev);
1450 out_irq_dispose_mapping:
1451 irq_dispose_mapping(priv->irq);
1453 dma_release_channel(priv->chan);
1455 iounmap(priv->regs);
1457 kref_put(&priv->ref, fpga_device_release);
1462 static int data_of_remove(struct platform_device *op)
1464 struct fpga_device *priv = platform_get_drvdata(op);
1465 struct device *this_device = priv->miscdev.this_device;
1467 /* remove all sysfs files, now the device cannot be re-enabled */
1468 sysfs_remove_group(&this_device->kobj, &rt_sysfs_attr_group);
1470 /* remove all debugfs files */
1471 data_debugfs_exit(priv);
1473 /* disable the device from generating data */
1474 data_device_disable(priv);
1476 /* remove the character device to stop new readers from appearing */
1477 misc_deregister(&priv->miscdev);
1479 /* cleanup everything not needed by readers */
1480 irq_dispose_mapping(priv->irq);
1481 dma_release_channel(priv->chan);
1482 iounmap(priv->regs);
1484 /* release our reference */
1485 kref_put(&priv->ref, fpga_device_release);
1489 static struct of_device_id data_of_match[] = {
1490 { .compatible = "carma,carma-fpga", },
1494 static struct platform_driver data_of_driver = {
1495 .probe = data_of_probe,
1496 .remove = data_of_remove,
1499 .of_match_table = data_of_match,
1503 module_platform_driver(data_of_driver);
1505 MODULE_AUTHOR("Ira W. Snyder <iws@ovro.caltech.edu>");
1506 MODULE_DESCRIPTION("CARMA DATA-FPGA Access Driver");
1507 MODULE_LICENSE("GPL");