Commit | Line | Data |
---|---|---|
50352fa7 | 1 | // SPDX-License-Identifier: GPL-2.0 |
ba82664c AS |
2 | /* |
3 | * Intel(R) Trace Hub Memory Storage Unit | |
4 | * | |
5 | * Copyright (C) 2014-2015 Intel Corporation. | |
ba82664c AS |
6 | */ |
7 | ||
8 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
9 | ||
10 | #include <linux/types.h> | |
11 | #include <linux/module.h> | |
12 | #include <linux/device.h> | |
13 | #include <linux/uaccess.h> | |
14 | #include <linux/sizes.h> | |
15 | #include <linux/printk.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/mm.h> | |
18 | #include <linux/fs.h> | |
19 | #include <linux/io.h> | |
20 | #include <linux/dma-mapping.h> | |
21 | ||
0c14dac9 LA |
22 | #ifdef CONFIG_X86 |
23 | #include <asm/set_memory.h> | |
24 | #endif | |
ba82664c AS |
25 | |
26 | #include "intel_th.h" | |
27 | #include "msu.h" | |
28 | ||
29 | #define msc_dev(x) (&(x)->thdev->dev) | |
30 | ||
ba82664c AS |
31 | /** |
32 | * struct msc_window - multiblock mode window descriptor | |
33 | * @entry: window list linkage (msc::win_list) | |
34 | * @pgoff: page offset into the buffer that this window starts at | |
35 | * @nr_blocks: number of blocks (pages) in this window | |
ba39bd83 | 36 | * @sgt: array of block descriptors |
ba82664c AS |
37 | */ |
38 | struct msc_window { | |
39 | struct list_head entry; | |
40 | unsigned long pgoff; | |
41 | unsigned int nr_blocks; | |
42 | struct msc *msc; | |
ba39bd83 | 43 | struct sg_table sgt; |
ba82664c AS |
44 | }; |
45 | ||
46 | /** | |
47 | * struct msc_iter - iterator for msc buffer | |
48 | * @entry: msc::iter_list linkage | |
49 | * @msc: pointer to the MSC device | |
50 | * @start_win: oldest window | |
51 | * @win: current window | |
52 | * @offset: current logical offset into the buffer | |
53 | * @start_block: oldest block in the window | |
54 | * @block: block number in the window | |
55 | * @block_off: offset into current block | |
56 | * @wrap_count: block wrapping handling | |
57 | * @eof: end of buffer reached | |
58 | */ | |
59 | struct msc_iter { | |
60 | struct list_head entry; | |
61 | struct msc *msc; | |
62 | struct msc_window *start_win; | |
63 | struct msc_window *win; | |
64 | unsigned long offset; | |
65 | int start_block; | |
66 | int block; | |
67 | unsigned int block_off; | |
68 | unsigned int wrap_count; | |
69 | unsigned int eof; | |
70 | }; | |
71 | ||
72 | /** | |
73 | * struct msc - MSC device representation | |
74 | * @reg_base: register window base address | |
75 | * @thdev: intel_th_device pointer | |
76 | * @win_list: list of windows in multiblock mode | |
4e0eaf23 | 77 | * @single_sgt: single mode buffer |
aad14ad3 | 78 | * @cur_win: current window |
ba82664c AS |
79 | * @nr_pages: total number of pages allocated for this buffer |
80 | * @single_sz: amount of data in single mode | |
81 | * @single_wrap: single mode wrap occurred | |
82 | * @base: buffer's base pointer | |
83 | * @base_addr: buffer's base address | |
84 | * @user_count: number of users of the buffer | |
85 | * @mmap_count: number of mappings | |
86 | * @buf_mutex: mutex to serialize access to buffer-related bits | |
87 | ||
88 | * @enabled: MSC is enabled | |
89 | * @wrap: wrapping is enabled | |
90 | * @mode: MSC operating mode | |
91 | * @burst_len: write burst length | |
92 | * @index: number of this MSC in the MSU | |
93 | */ | |
94 | struct msc { | |
95 | void __iomem *reg_base; | |
aac8da65 | 96 | void __iomem *msu_base; |
ba82664c AS |
97 | struct intel_th_device *thdev; |
98 | ||
99 | struct list_head win_list; | |
4e0eaf23 | 100 | struct sg_table single_sgt; |
aad14ad3 | 101 | struct msc_window *cur_win; |
ba82664c AS |
102 | unsigned long nr_pages; |
103 | unsigned long single_sz; | |
104 | unsigned int single_wrap : 1; | |
105 | void *base; | |
106 | dma_addr_t base_addr; | |
107 | ||
108 | /* <0: no buffer, 0: no users, >0: active users */ | |
109 | atomic_t user_count; | |
110 | ||
111 | atomic_t mmap_count; | |
112 | struct mutex buf_mutex; | |
113 | ||
ba82664c AS |
114 | struct list_head iter_list; |
115 | ||
116 | /* config */ | |
117 | unsigned int enabled : 1, | |
aac8da65 AS |
118 | wrap : 1, |
119 | do_irq : 1; | |
ba82664c AS |
120 | unsigned int mode; |
121 | unsigned int burst_len; | |
122 | unsigned int index; | |
123 | }; | |
124 | ||
125 | static inline bool msc_block_is_empty(struct msc_block_desc *bdesc) | |
126 | { | |
127 | /* header hasn't been written */ | |
128 | if (!bdesc->valid_dw) | |
129 | return true; | |
130 | ||
131 | /* valid_dw includes the header */ | |
132 | if (!msc_data_sz(bdesc)) | |
133 | return true; | |
134 | ||
135 | return false; | |
136 | } | |
137 | ||
ba39bd83 AS |
138 | static inline struct msc_block_desc * |
139 | msc_win_block(struct msc_window *win, unsigned int block) | |
140 | { | |
141 | return sg_virt(&win->sgt.sgl[block]); | |
142 | } | |
143 | ||
144 | static inline dma_addr_t | |
145 | msc_win_baddr(struct msc_window *win, unsigned int block) | |
146 | { | |
147 | return sg_dma_address(&win->sgt.sgl[block]); | |
148 | } | |
149 | ||
150 | static inline unsigned long | |
151 | msc_win_bpfn(struct msc_window *win, unsigned int block) | |
152 | { | |
153 | return msc_win_baddr(win, block) >> PAGE_SHIFT; | |
154 | } | |
155 | ||
aad14ad3 AS |
156 | /** |
157 | * msc_is_last_win() - check if a window is the last one for a given MSC | |
158 | * @win: window | |
159 | * Return: true if @win is the last window in MSC's multiblock buffer | |
160 | */ | |
161 | static inline bool msc_is_last_win(struct msc_window *win) | |
162 | { | |
163 | return win->entry.next == &win->msc->win_list; | |
164 | } | |
165 | ||
166 | /** | |
167 | * msc_next_window() - return next window in the multiblock buffer | |
168 | * @win: current window | |
169 | * | |
170 | * Return: window following the current one | |
171 | */ | |
172 | static struct msc_window *msc_next_window(struct msc_window *win) | |
173 | { | |
174 | if (msc_is_last_win(win)) | |
175 | return list_first_entry(&win->msc->win_list, struct msc_window, | |
176 | entry); | |
177 | ||
178 | return list_next_entry(win, entry); | |
179 | } | |
180 | ||
ba82664c AS |
181 | /** |
182 | * msc_oldest_window() - locate the window with oldest data | |
183 | * @msc: MSC device | |
184 | * | |
185 | * This should only be used in multiblock mode. Caller should hold the | |
186 | * msc::user_count reference. | |
187 | * | |
188 | * Return: the oldest window with valid data | |
189 | */ | |
190 | static struct msc_window *msc_oldest_window(struct msc *msc) | |
191 | { | |
aad14ad3 | 192 | struct msc_window *win, *next = msc_next_window(msc->cur_win); |
ba82664c AS |
193 | unsigned int found = 0; |
194 | ||
195 | if (list_empty(&msc->win_list)) | |
196 | return NULL; | |
197 | ||
198 | /* | |
199 | * we might need a radix tree for this, depending on how | |
200 | * many windows a typical user would allocate; ideally it's | |
201 | * something like 2, in which case we're good | |
202 | */ | |
203 | list_for_each_entry(win, &msc->win_list, entry) { | |
aad14ad3 | 204 | if (win == next) |
ba82664c AS |
205 | found++; |
206 | ||
207 | /* skip the empty ones */ | |
ba39bd83 | 208 | if (msc_block_is_empty(msc_win_block(win, 0))) |
ba82664c AS |
209 | continue; |
210 | ||
211 | if (found) | |
212 | return win; | |
213 | } | |
214 | ||
0de9e035 | 215 | return list_first_entry(&msc->win_list, struct msc_window, entry); |
ba82664c AS |
216 | } |
217 | ||
218 | /** | |
219 | * msc_win_oldest_block() - locate the oldest block in a given window | |
220 | * @win: window to look at | |
221 | * | |
222 | * Return: index of the block with the oldest data | |
223 | */ | |
224 | static unsigned int msc_win_oldest_block(struct msc_window *win) | |
225 | { | |
226 | unsigned int blk; | |
ba39bd83 | 227 | struct msc_block_desc *bdesc = msc_win_block(win, 0); |
ba82664c AS |
228 | |
229 | /* without wrapping, first block is the oldest */ | |
230 | if (!msc_block_wrapped(bdesc)) | |
231 | return 0; | |
232 | ||
233 | /* | |
234 | * with wrapping, last written block contains both the newest and the | |
235 | * oldest data for this window. | |
236 | */ | |
237 | for (blk = 0; blk < win->nr_blocks; blk++) { | |
ba39bd83 | 238 | bdesc = msc_win_block(win, blk); |
ba82664c AS |
239 | |
240 | if (msc_block_last_written(bdesc)) | |
241 | return blk; | |
242 | } | |
243 | ||
244 | return 0; | |
245 | } | |
246 | ||
ba82664c AS |
247 | static struct msc_block_desc *msc_iter_bdesc(struct msc_iter *iter) |
248 | { | |
ba39bd83 | 249 | return msc_win_block(iter->win, iter->block); |
ba82664c AS |
250 | } |
251 | ||
252 | static void msc_iter_init(struct msc_iter *iter) | |
253 | { | |
254 | memset(iter, 0, sizeof(*iter)); | |
255 | iter->start_block = -1; | |
256 | iter->block = -1; | |
257 | } | |
258 | ||
259 | static struct msc_iter *msc_iter_install(struct msc *msc) | |
260 | { | |
261 | struct msc_iter *iter; | |
262 | ||
263 | iter = kzalloc(sizeof(*iter), GFP_KERNEL); | |
264 | if (!iter) | |
a45ff6ed AS |
265 | return ERR_PTR(-ENOMEM); |
266 | ||
267 | mutex_lock(&msc->buf_mutex); | |
268 | ||
269 | /* | |
270 | * Reading and tracing are mutually exclusive; if msc is | |
271 | * enabled, open() will fail; otherwise existing readers | |
272 | * will prevent enabling the msc and the rest of fops don't | |
273 | * need to worry about it. | |
274 | */ | |
275 | if (msc->enabled) { | |
276 | kfree(iter); | |
277 | iter = ERR_PTR(-EBUSY); | |
278 | goto unlock; | |
279 | } | |
ba82664c AS |
280 | |
281 | msc_iter_init(iter); | |
282 | iter->msc = msc; | |
283 | ||
ba82664c | 284 | list_add_tail(&iter->entry, &msc->iter_list); |
a45ff6ed AS |
285 | unlock: |
286 | mutex_unlock(&msc->buf_mutex); | |
ba82664c AS |
287 | |
288 | return iter; | |
289 | } | |
290 | ||
291 | static void msc_iter_remove(struct msc_iter *iter, struct msc *msc) | |
292 | { | |
a45ff6ed | 293 | mutex_lock(&msc->buf_mutex); |
ba82664c | 294 | list_del(&iter->entry); |
a45ff6ed | 295 | mutex_unlock(&msc->buf_mutex); |
ba82664c AS |
296 | |
297 | kfree(iter); | |
298 | } | |
299 | ||
300 | static void msc_iter_block_start(struct msc_iter *iter) | |
301 | { | |
302 | if (iter->start_block != -1) | |
303 | return; | |
304 | ||
305 | iter->start_block = msc_win_oldest_block(iter->win); | |
306 | iter->block = iter->start_block; | |
307 | iter->wrap_count = 0; | |
308 | ||
309 | /* | |
310 | * start with the block with oldest data; if data has wrapped | |
311 | * in this window, it should be in this block | |
312 | */ | |
313 | if (msc_block_wrapped(msc_iter_bdesc(iter))) | |
314 | iter->wrap_count = 2; | |
315 | ||
316 | } | |
317 | ||
318 | static int msc_iter_win_start(struct msc_iter *iter, struct msc *msc) | |
319 | { | |
320 | /* already started, nothing to do */ | |
321 | if (iter->start_win) | |
322 | return 0; | |
323 | ||
324 | iter->start_win = msc_oldest_window(msc); | |
325 | if (!iter->start_win) | |
326 | return -EINVAL; | |
327 | ||
328 | iter->win = iter->start_win; | |
329 | iter->start_block = -1; | |
330 | ||
331 | msc_iter_block_start(iter); | |
332 | ||
333 | return 0; | |
334 | } | |
335 | ||
336 | static int msc_iter_win_advance(struct msc_iter *iter) | |
337 | { | |
338 | iter->win = msc_next_window(iter->win); | |
339 | iter->start_block = -1; | |
340 | ||
341 | if (iter->win == iter->start_win) { | |
342 | iter->eof++; | |
343 | return 1; | |
344 | } | |
345 | ||
346 | msc_iter_block_start(iter); | |
347 | ||
348 | return 0; | |
349 | } | |
350 | ||
351 | static int msc_iter_block_advance(struct msc_iter *iter) | |
352 | { | |
353 | iter->block_off = 0; | |
354 | ||
355 | /* wrapping */ | |
356 | if (iter->wrap_count && iter->block == iter->start_block) { | |
357 | iter->wrap_count--; | |
358 | if (!iter->wrap_count) | |
359 | /* copied newest data from the wrapped block */ | |
360 | return msc_iter_win_advance(iter); | |
361 | } | |
362 | ||
363 | /* no wrapping, check for last written block */ | |
364 | if (!iter->wrap_count && msc_block_last_written(msc_iter_bdesc(iter))) | |
365 | /* copied newest data for the window */ | |
366 | return msc_iter_win_advance(iter); | |
367 | ||
368 | /* block advance */ | |
369 | if (++iter->block == iter->win->nr_blocks) | |
370 | iter->block = 0; | |
371 | ||
372 | /* no wrapping, sanity check in case there is no last written block */ | |
373 | if (!iter->wrap_count && iter->block == iter->start_block) | |
374 | return msc_iter_win_advance(iter); | |
375 | ||
376 | return 0; | |
377 | } | |
378 | ||
379 | /** | |
380 | * msc_buffer_iterate() - go through multiblock buffer's data | |
381 | * @iter: iterator structure | |
382 | * @size: amount of data to scan | |
383 | * @data: callback's private data | |
384 | * @fn: iterator callback | |
385 | * | |
386 | * This will start at the window which will be written to next (containing | |
387 | * the oldest data) and work its way to the current window, calling @fn | |
388 | * for each chunk of data as it goes. | |
389 | * | |
390 | * Caller should have msc::user_count reference to make sure the buffer | |
391 | * doesn't disappear from under us. | |
392 | * | |
393 | * Return: amount of data actually scanned. | |
394 | */ | |
395 | static ssize_t | |
396 | msc_buffer_iterate(struct msc_iter *iter, size_t size, void *data, | |
397 | unsigned long (*fn)(void *, void *, size_t)) | |
398 | { | |
399 | struct msc *msc = iter->msc; | |
400 | size_t len = size; | |
401 | unsigned int advance; | |
402 | ||
403 | if (iter->eof) | |
404 | return 0; | |
405 | ||
406 | /* start with the oldest window */ | |
407 | if (msc_iter_win_start(iter, msc)) | |
408 | return 0; | |
409 | ||
410 | do { | |
411 | unsigned long data_bytes = msc_data_sz(msc_iter_bdesc(iter)); | |
412 | void *src = (void *)msc_iter_bdesc(iter) + MSC_BDESC; | |
413 | size_t tocopy = data_bytes, copied = 0; | |
414 | size_t remaining = 0; | |
415 | ||
416 | advance = 1; | |
417 | ||
418 | /* | |
419 | * If block wrapping happened, we need to visit the last block | |
420 | * twice, because it contains both the oldest and the newest | |
421 | * data in this window. | |
422 | * | |
423 | * First time (wrap_count==2), in the very beginning, to collect | |
424 | * the oldest data, which is in the range | |
425 | * (data_bytes..DATA_IN_PAGE). | |
426 | * | |
427 | * Second time (wrap_count==1), it's just like any other block, | |
428 | * containing data in the range of [MSC_BDESC..data_bytes]. | |
429 | */ | |
e4eca2a1 | 430 | if (iter->block == iter->start_block && iter->wrap_count == 2) { |
ba82664c AS |
431 | tocopy = DATA_IN_PAGE - data_bytes; |
432 | src += data_bytes; | |
433 | } | |
434 | ||
435 | if (!tocopy) | |
436 | goto next_block; | |
437 | ||
438 | tocopy -= iter->block_off; | |
439 | src += iter->block_off; | |
440 | ||
441 | if (len < tocopy) { | |
442 | tocopy = len; | |
443 | advance = 0; | |
444 | } | |
445 | ||
446 | remaining = fn(data, src, tocopy); | |
447 | ||
448 | if (remaining) | |
449 | advance = 0; | |
450 | ||
451 | copied = tocopy - remaining; | |
452 | len -= copied; | |
453 | iter->block_off += copied; | |
454 | iter->offset += copied; | |
455 | ||
456 | if (!advance) | |
457 | break; | |
458 | ||
459 | next_block: | |
460 | if (msc_iter_block_advance(iter)) | |
461 | break; | |
462 | ||
463 | } while (len); | |
464 | ||
465 | return size - len; | |
466 | } | |
467 | ||
468 | /** | |
469 | * msc_buffer_clear_hw_header() - clear hw header for multiblock | |
470 | * @msc: MSC device | |
471 | */ | |
472 | static void msc_buffer_clear_hw_header(struct msc *msc) | |
473 | { | |
474 | struct msc_window *win; | |
475 | ||
ba82664c AS |
476 | list_for_each_entry(win, &msc->win_list, entry) { |
477 | unsigned int blk; | |
478 | size_t hw_sz = sizeof(struct msc_block_desc) - | |
479 | offsetof(struct msc_block_desc, hw_tag); | |
480 | ||
481 | for (blk = 0; blk < win->nr_blocks; blk++) { | |
ba39bd83 | 482 | struct msc_block_desc *bdesc = msc_win_block(win, blk); |
ba82664c AS |
483 | |
484 | memset(&bdesc->hw_tag, 0, hw_sz); | |
485 | } | |
486 | } | |
ba82664c AS |
487 | } |
488 | ||
aac8da65 AS |
489 | static int intel_th_msu_init(struct msc *msc) |
490 | { | |
491 | u32 mintctl, msusts; | |
492 | ||
493 | if (!msc->do_irq) | |
494 | return 0; | |
495 | ||
496 | mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL); | |
497 | mintctl |= msc->index ? M1BLIE : M0BLIE; | |
498 | iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL); | |
499 | if (mintctl != ioread32(msc->msu_base + REG_MSU_MINTCTL)) { | |
500 | dev_info(msc_dev(msc), "MINTCTL ignores writes: no usable interrupts\n"); | |
501 | msc->do_irq = 0; | |
502 | return 0; | |
503 | } | |
504 | ||
505 | msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS); | |
506 | iowrite32(msusts, msc->msu_base + REG_MSU_MSUSTS); | |
507 | ||
508 | return 0; | |
509 | } | |
510 | ||
511 | static void intel_th_msu_deinit(struct msc *msc) | |
512 | { | |
513 | u32 mintctl; | |
514 | ||
515 | if (!msc->do_irq) | |
516 | return; | |
517 | ||
518 | mintctl = ioread32(msc->msu_base + REG_MSU_MINTCTL); | |
519 | mintctl &= msc->index ? ~M1BLIE : ~M0BLIE; | |
520 | iowrite32(mintctl, msc->msu_base + REG_MSU_MINTCTL); | |
521 | } | |
522 | ||
ba82664c AS |
523 | /** |
524 | * msc_configure() - set up MSC hardware | |
525 | * @msc: the MSC device to configure | |
526 | * | |
527 | * Program storage mode, wrapping, burst length and trace buffer address | |
a45ff6ed AS |
528 | * into a given MSC. Then, enable tracing and set msc::enabled. |
529 | * The latter is serialized on msc::buf_mutex, so make sure to hold it. | |
ba82664c AS |
530 | */ |
531 | static int msc_configure(struct msc *msc) | |
532 | { | |
533 | u32 reg; | |
534 | ||
a45ff6ed AS |
535 | lockdep_assert_held(&msc->buf_mutex); |
536 | ||
ba82664c AS |
537 | if (msc->mode > MSC_MODE_MULTI) |
538 | return -ENOTSUPP; | |
539 | ||
540 | if (msc->mode == MSC_MODE_MULTI) | |
541 | msc_buffer_clear_hw_header(msc); | |
542 | ||
543 | reg = msc->base_addr >> PAGE_SHIFT; | |
544 | iowrite32(reg, msc->reg_base + REG_MSU_MSC0BAR); | |
545 | ||
546 | if (msc->mode == MSC_MODE_SINGLE) { | |
547 | reg = msc->nr_pages; | |
548 | iowrite32(reg, msc->reg_base + REG_MSU_MSC0SIZE); | |
549 | } | |
550 | ||
551 | reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL); | |
552 | reg &= ~(MSC_MODE | MSC_WRAPEN | MSC_EN | MSC_RD_HDR_OVRD); | |
553 | ||
a45ff6ed | 554 | reg |= MSC_EN; |
ba82664c AS |
555 | reg |= msc->mode << __ffs(MSC_MODE); |
556 | reg |= msc->burst_len << __ffs(MSC_LEN); | |
a45ff6ed | 557 | |
ba82664c AS |
558 | if (msc->wrap) |
559 | reg |= MSC_WRAPEN; | |
ba82664c AS |
560 | |
561 | iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL); | |
562 | ||
a45ff6ed AS |
563 | msc->thdev->output.multiblock = msc->mode == MSC_MODE_MULTI; |
564 | intel_th_trace_enable(msc->thdev); | |
565 | msc->enabled = 1; | |
566 | ||
ba82664c AS |
567 | |
568 | return 0; | |
569 | } | |
570 | ||
571 | /** | |
572 | * msc_disable() - disable MSC hardware | |
573 | * @msc: MSC device to disable | |
574 | * | |
575 | * If @msc is enabled, disable tracing on the switch and then disable MSC | |
a45ff6ed | 576 | * storage. Caller must hold msc::buf_mutex. |
ba82664c AS |
577 | */ |
578 | static void msc_disable(struct msc *msc) | |
579 | { | |
ba82664c AS |
580 | u32 reg; |
581 | ||
a45ff6ed | 582 | lockdep_assert_held(&msc->buf_mutex); |
ba82664c AS |
583 | |
584 | intel_th_trace_disable(msc->thdev); | |
585 | ||
ba82664c | 586 | if (msc->mode == MSC_MODE_SINGLE) { |
8d415512 | 587 | reg = ioread32(msc->reg_base + REG_MSU_MSC0STS); |
ba82664c AS |
588 | msc->single_wrap = !!(reg & MSCSTS_WRAPSTAT); |
589 | ||
590 | reg = ioread32(msc->reg_base + REG_MSU_MSC0MWP); | |
591 | msc->single_sz = reg & ((msc->nr_pages << PAGE_SHIFT) - 1); | |
592 | dev_dbg(msc_dev(msc), "MSCnMWP: %08x/%08lx, wrap: %d\n", | |
593 | reg, msc->single_sz, msc->single_wrap); | |
594 | } | |
595 | ||
596 | reg = ioread32(msc->reg_base + REG_MSU_MSC0CTL); | |
597 | reg &= ~MSC_EN; | |
598 | iowrite32(reg, msc->reg_base + REG_MSU_MSC0CTL); | |
599 | msc->enabled = 0; | |
600 | ||
601 | iowrite32(0, msc->reg_base + REG_MSU_MSC0BAR); | |
602 | iowrite32(0, msc->reg_base + REG_MSU_MSC0SIZE); | |
603 | ||
604 | dev_dbg(msc_dev(msc), "MSCnNWSA: %08x\n", | |
605 | ioread32(msc->reg_base + REG_MSU_MSC0NWSA)); | |
606 | ||
607 | reg = ioread32(msc->reg_base + REG_MSU_MSC0STS); | |
608 | dev_dbg(msc_dev(msc), "MSCnSTS: %08x\n", reg); | |
609 | } | |
610 | ||
611 | static int intel_th_msc_activate(struct intel_th_device *thdev) | |
612 | { | |
613 | struct msc *msc = dev_get_drvdata(&thdev->dev); | |
a45ff6ed | 614 | int ret = -EBUSY; |
ba82664c AS |
615 | |
616 | if (!atomic_inc_unless_negative(&msc->user_count)) | |
617 | return -ENODEV; | |
618 | ||
a45ff6ed | 619 | mutex_lock(&msc->buf_mutex); |
ba82664c | 620 | |
a45ff6ed AS |
621 | /* if there are readers, refuse */ |
622 | if (list_empty(&msc->iter_list)) | |
623 | ret = msc_configure(msc); | |
ba82664c | 624 | |
a45ff6ed AS |
625 | mutex_unlock(&msc->buf_mutex); |
626 | ||
627 | if (ret) | |
628 | atomic_dec(&msc->user_count); | |
ba82664c | 629 | |
a45ff6ed | 630 | return ret; |
ba82664c AS |
631 | } |
632 | ||
633 | static void intel_th_msc_deactivate(struct intel_th_device *thdev) | |
634 | { | |
635 | struct msc *msc = dev_get_drvdata(&thdev->dev); | |
636 | ||
a45ff6ed AS |
637 | mutex_lock(&msc->buf_mutex); |
638 | if (msc->enabled) { | |
639 | msc_disable(msc); | |
640 | atomic_dec(&msc->user_count); | |
641 | } | |
642 | mutex_unlock(&msc->buf_mutex); | |
ba82664c AS |
643 | } |
644 | ||
645 | /** | |
646 | * msc_buffer_contig_alloc() - allocate a contiguous buffer for SINGLE mode | |
647 | * @msc: MSC device | |
648 | * @size: allocation size in bytes | |
649 | * | |
650 | * This modifies msc::base, which requires msc::buf_mutex to serialize, so the | |
651 | * caller is expected to hold it. | |
652 | * | |
653 | * Return: 0 on success, -errno otherwise. | |
654 | */ | |
655 | static int msc_buffer_contig_alloc(struct msc *msc, unsigned long size) | |
656 | { | |
4e0eaf23 | 657 | unsigned long nr_pages = size >> PAGE_SHIFT; |
ba82664c AS |
658 | unsigned int order = get_order(size); |
659 | struct page *page; | |
4e0eaf23 | 660 | int ret; |
ba82664c AS |
661 | |
662 | if (!size) | |
663 | return 0; | |
664 | ||
4e0eaf23 AS |
665 | ret = sg_alloc_table(&msc->single_sgt, 1, GFP_KERNEL); |
666 | if (ret) | |
667 | goto err_out; | |
668 | ||
669 | ret = -ENOMEM; | |
ba82664c AS |
670 | page = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); |
671 | if (!page) | |
4e0eaf23 | 672 | goto err_free_sgt; |
ba82664c AS |
673 | |
674 | split_page(page, order); | |
4e0eaf23 AS |
675 | sg_set_buf(msc->single_sgt.sgl, page_address(page), size); |
676 | ||
677 | ret = dma_map_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, 1, | |
678 | DMA_FROM_DEVICE); | |
679 | if (ret < 0) | |
680 | goto err_free_pages; | |
681 | ||
682 | msc->nr_pages = nr_pages; | |
ba82664c | 683 | msc->base = page_address(page); |
4e0eaf23 | 684 | msc->base_addr = sg_dma_address(msc->single_sgt.sgl); |
ba82664c AS |
685 | |
686 | return 0; | |
4e0eaf23 AS |
687 | |
688 | err_free_pages: | |
689 | __free_pages(page, order); | |
690 | ||
691 | err_free_sgt: | |
692 | sg_free_table(&msc->single_sgt); | |
693 | ||
694 | err_out: | |
695 | return ret; | |
ba82664c AS |
696 | } |
697 | ||
698 | /** | |
699 | * msc_buffer_contig_free() - free a contiguous buffer | |
700 | * @msc: MSC configured in SINGLE mode | |
701 | */ | |
702 | static void msc_buffer_contig_free(struct msc *msc) | |
703 | { | |
704 | unsigned long off; | |
705 | ||
4e0eaf23 AS |
706 | dma_unmap_sg(msc_dev(msc)->parent->parent, msc->single_sgt.sgl, |
707 | 1, DMA_FROM_DEVICE); | |
708 | sg_free_table(&msc->single_sgt); | |
709 | ||
ba82664c AS |
710 | for (off = 0; off < msc->nr_pages << PAGE_SHIFT; off += PAGE_SIZE) { |
711 | struct page *page = virt_to_page(msc->base + off); | |
712 | ||
713 | page->mapping = NULL; | |
714 | __free_page(page); | |
715 | } | |
716 | ||
717 | msc->nr_pages = 0; | |
718 | } | |
719 | ||
720 | /** | |
721 | * msc_buffer_contig_get_page() - find a page at a given offset | |
722 | * @msc: MSC configured in SINGLE mode | |
723 | * @pgoff: page offset | |
724 | * | |
725 | * Return: page, if @pgoff is within the range, NULL otherwise. | |
726 | */ | |
727 | static struct page *msc_buffer_contig_get_page(struct msc *msc, | |
728 | unsigned long pgoff) | |
729 | { | |
730 | if (pgoff >= msc->nr_pages) | |
731 | return NULL; | |
732 | ||
733 | return virt_to_page(msc->base + (pgoff << PAGE_SHIFT)); | |
734 | } | |
735 | ||
ba39bd83 AS |
736 | static int __msc_buffer_win_alloc(struct msc_window *win, |
737 | unsigned int nr_blocks) | |
738 | { | |
739 | struct scatterlist *sg_ptr; | |
740 | void *block; | |
741 | int i, ret; | |
742 | ||
743 | ret = sg_alloc_table(&win->sgt, nr_blocks, GFP_KERNEL); | |
744 | if (ret) | |
745 | return -ENOMEM; | |
746 | ||
747 | for_each_sg(win->sgt.sgl, sg_ptr, nr_blocks, i) { | |
748 | block = dma_alloc_coherent(msc_dev(win->msc)->parent->parent, | |
749 | PAGE_SIZE, &sg_dma_address(sg_ptr), | |
750 | GFP_KERNEL); | |
751 | if (!block) | |
752 | goto err_nomem; | |
753 | ||
754 | sg_set_buf(sg_ptr, block, PAGE_SIZE); | |
755 | } | |
756 | ||
757 | return nr_blocks; | |
758 | ||
759 | err_nomem: | |
760 | for (i--; i >= 0; i--) | |
761 | dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE, | |
762 | msc_win_block(win, i), | |
763 | msc_win_baddr(win, i)); | |
764 | ||
765 | sg_free_table(&win->sgt); | |
766 | ||
767 | return -ENOMEM; | |
768 | } | |
769 | ||
b96fb368 SZ |
770 | #ifdef CONFIG_X86 |
771 | static void msc_buffer_set_uc(struct msc_window *win, unsigned int nr_blocks) | |
772 | { | |
773 | int i; | |
774 | ||
775 | for (i = 0; i < nr_blocks; i++) | |
776 | /* Set the page as uncached */ | |
777 | set_memory_uc((unsigned long)msc_win_block(win, i), 1); | |
778 | } | |
779 | ||
780 | static void msc_buffer_set_wb(struct msc_window *win) | |
781 | { | |
782 | int i; | |
783 | ||
784 | for (i = 0; i < win->nr_blocks; i++) | |
785 | /* Reset the page to write-back */ | |
786 | set_memory_wb((unsigned long)msc_win_block(win, i), 1); | |
787 | } | |
788 | #else /* !X86 */ | |
789 | static inline void | |
790 | msc_buffer_set_uc(struct msc_window *win, unsigned int nr_blocks) {} | |
791 | static inline void msc_buffer_set_wb(struct msc_window *win) {} | |
792 | #endif /* CONFIG_X86 */ | |
793 | ||
ba82664c AS |
794 | /** |
795 | * msc_buffer_win_alloc() - alloc a window for a multiblock mode | |
796 | * @msc: MSC device | |
797 | * @nr_blocks: number of pages in this window | |
798 | * | |
799 | * This modifies msc::win_list and msc::base, which requires msc::buf_mutex | |
800 | * to serialize, so the caller is expected to hold it. | |
801 | * | |
802 | * Return: 0 on success, -errno otherwise. | |
803 | */ | |
804 | static int msc_buffer_win_alloc(struct msc *msc, unsigned int nr_blocks) | |
805 | { | |
806 | struct msc_window *win; | |
b96fb368 | 807 | int ret = -ENOMEM; |
ba82664c AS |
808 | |
809 | if (!nr_blocks) | |
810 | return 0; | |
811 | ||
ba39bd83 AS |
812 | /* |
813 | * This limitation hold as long as we need random access to the | |
814 | * block. When that changes, this can go away. | |
815 | */ | |
816 | if (nr_blocks > SG_MAX_SINGLE_ALLOC) | |
817 | return -EINVAL; | |
818 | ||
819 | win = kzalloc(sizeof(*win), GFP_KERNEL); | |
ba82664c AS |
820 | if (!win) |
821 | return -ENOMEM; | |
822 | ||
ba39bd83 AS |
823 | win->msc = msc; |
824 | ||
ba82664c | 825 | if (!list_empty(&msc->win_list)) { |
0de9e035 AS |
826 | struct msc_window *prev = list_last_entry(&msc->win_list, |
827 | struct msc_window, | |
828 | entry); | |
ba82664c | 829 | |
ba39bd83 | 830 | /* This works as long as blocks are page-sized */ |
ba82664c AS |
831 | win->pgoff = prev->pgoff + prev->nr_blocks; |
832 | } | |
833 | ||
ba39bd83 AS |
834 | ret = __msc_buffer_win_alloc(win, nr_blocks); |
835 | if (ret < 0) | |
836 | goto err_nomem; | |
ba82664c | 837 | |
b96fb368 | 838 | msc_buffer_set_uc(win, ret); |
ba82664c | 839 | |
ba39bd83 | 840 | win->nr_blocks = ret; |
ba82664c AS |
841 | |
842 | if (list_empty(&msc->win_list)) { | |
ba39bd83 AS |
843 | msc->base = msc_win_block(win, 0); |
844 | msc->base_addr = msc_win_baddr(win, 0); | |
aad14ad3 | 845 | msc->cur_win = win; |
ba82664c AS |
846 | } |
847 | ||
848 | list_add_tail(&win->entry, &msc->win_list); | |
849 | msc->nr_pages += nr_blocks; | |
850 | ||
851 | return 0; | |
852 | ||
853 | err_nomem: | |
ba82664c AS |
854 | kfree(win); |
855 | ||
856 | return ret; | |
857 | } | |
858 | ||
ba39bd83 AS |
859 | static void __msc_buffer_win_free(struct msc *msc, struct msc_window *win) |
860 | { | |
861 | int i; | |
862 | ||
863 | for (i = 0; i < win->nr_blocks; i++) { | |
864 | struct page *page = sg_page(&win->sgt.sgl[i]); | |
865 | ||
866 | page->mapping = NULL; | |
867 | dma_free_coherent(msc_dev(win->msc)->parent->parent, PAGE_SIZE, | |
868 | msc_win_block(win, i), msc_win_baddr(win, i)); | |
869 | } | |
870 | sg_free_table(&win->sgt); | |
871 | } | |
872 | ||
ba82664c AS |
873 | /** |
874 | * msc_buffer_win_free() - free a window from MSC's window list | |
875 | * @msc: MSC device | |
876 | * @win: window to free | |
877 | * | |
878 | * This modifies msc::win_list and msc::base, which requires msc::buf_mutex | |
879 | * to serialize, so the caller is expected to hold it. | |
880 | */ | |
881 | static void msc_buffer_win_free(struct msc *msc, struct msc_window *win) | |
882 | { | |
ba82664c AS |
883 | msc->nr_pages -= win->nr_blocks; |
884 | ||
885 | list_del(&win->entry); | |
886 | if (list_empty(&msc->win_list)) { | |
887 | msc->base = NULL; | |
888 | msc->base_addr = 0; | |
889 | } | |
890 | ||
b96fb368 | 891 | msc_buffer_set_wb(win); |
ba39bd83 AS |
892 | |
893 | __msc_buffer_win_free(msc, win); | |
ba82664c AS |
894 | |
895 | kfree(win); | |
896 | } | |
897 | ||
898 | /** | |
899 | * msc_buffer_relink() - set up block descriptors for multiblock mode | |
900 | * @msc: MSC device | |
901 | * | |
902 | * This traverses msc::win_list, which requires msc::buf_mutex to serialize, | |
903 | * so the caller is expected to hold it. | |
904 | */ | |
905 | static void msc_buffer_relink(struct msc *msc) | |
906 | { | |
907 | struct msc_window *win, *next_win; | |
908 | ||
909 | /* call with msc::mutex locked */ | |
910 | list_for_each_entry(win, &msc->win_list, entry) { | |
911 | unsigned int blk; | |
912 | u32 sw_tag = 0; | |
913 | ||
914 | /* | |
915 | * Last window's next_win should point to the first window | |
916 | * and MSC_SW_TAG_LASTWIN should be set. | |
917 | */ | |
918 | if (msc_is_last_win(win)) { | |
919 | sw_tag |= MSC_SW_TAG_LASTWIN; | |
0de9e035 AS |
920 | next_win = list_first_entry(&msc->win_list, |
921 | struct msc_window, entry); | |
ba82664c | 922 | } else { |
0de9e035 | 923 | next_win = list_next_entry(win, entry); |
ba82664c AS |
924 | } |
925 | ||
926 | for (blk = 0; blk < win->nr_blocks; blk++) { | |
ba39bd83 | 927 | struct msc_block_desc *bdesc = msc_win_block(win, blk); |
ba82664c AS |
928 | |
929 | memset(bdesc, 0, sizeof(*bdesc)); | |
930 | ||
ba39bd83 | 931 | bdesc->next_win = msc_win_bpfn(next_win, 0); |
ba82664c AS |
932 | |
933 | /* | |
934 | * Similarly to last window, last block should point | |
935 | * to the first one. | |
936 | */ | |
937 | if (blk == win->nr_blocks - 1) { | |
938 | sw_tag |= MSC_SW_TAG_LASTBLK; | |
ba39bd83 | 939 | bdesc->next_blk = msc_win_bpfn(win, 0); |
ba82664c | 940 | } else { |
ba39bd83 | 941 | bdesc->next_blk = msc_win_bpfn(win, blk + 1); |
ba82664c AS |
942 | } |
943 | ||
944 | bdesc->sw_tag = sw_tag; | |
945 | bdesc->block_sz = PAGE_SIZE / 64; | |
946 | } | |
947 | } | |
948 | ||
949 | /* | |
950 | * Make the above writes globally visible before tracing is | |
951 | * enabled to make sure hardware sees them coherently. | |
952 | */ | |
953 | wmb(); | |
954 | } | |
955 | ||
956 | static void msc_buffer_multi_free(struct msc *msc) | |
957 | { | |
958 | struct msc_window *win, *iter; | |
959 | ||
960 | list_for_each_entry_safe(win, iter, &msc->win_list, entry) | |
961 | msc_buffer_win_free(msc, win); | |
962 | } | |
963 | ||
964 | static int msc_buffer_multi_alloc(struct msc *msc, unsigned long *nr_pages, | |
965 | unsigned int nr_wins) | |
966 | { | |
967 | int ret, i; | |
968 | ||
969 | for (i = 0; i < nr_wins; i++) { | |
970 | ret = msc_buffer_win_alloc(msc, nr_pages[i]); | |
971 | if (ret) { | |
972 | msc_buffer_multi_free(msc); | |
973 | return ret; | |
974 | } | |
975 | } | |
976 | ||
977 | msc_buffer_relink(msc); | |
978 | ||
979 | return 0; | |
980 | } | |
981 | ||
982 | /** | |
983 | * msc_buffer_free() - free buffers for MSC | |
984 | * @msc: MSC device | |
985 | * | |
986 | * Free MSC's storage buffers. | |
987 | * | |
988 | * This modifies msc::win_list and msc::base, which requires msc::buf_mutex to | |
989 | * serialize, so the caller is expected to hold it. | |
990 | */ | |
991 | static void msc_buffer_free(struct msc *msc) | |
992 | { | |
993 | if (msc->mode == MSC_MODE_SINGLE) | |
994 | msc_buffer_contig_free(msc); | |
995 | else if (msc->mode == MSC_MODE_MULTI) | |
996 | msc_buffer_multi_free(msc); | |
997 | } | |
998 | ||
999 | /** | |
1000 | * msc_buffer_alloc() - allocate a buffer for MSC | |
1001 | * @msc: MSC device | |
1002 | * @size: allocation size in bytes | |
1003 | * | |
1004 | * Allocate a storage buffer for MSC, depending on the msc::mode, it will be | |
1005 | * either done via msc_buffer_contig_alloc() for SINGLE operation mode or | |
1006 | * msc_buffer_win_alloc() for multiblock operation. The latter allocates one | |
1007 | * window per invocation, so in multiblock mode this can be called multiple | |
1008 | * times for the same MSC to allocate multiple windows. | |
1009 | * | |
1010 | * This modifies msc::win_list and msc::base, which requires msc::buf_mutex | |
1011 | * to serialize, so the caller is expected to hold it. | |
1012 | * | |
1013 | * Return: 0 on success, -errno otherwise. | |
1014 | */ | |
1015 | static int msc_buffer_alloc(struct msc *msc, unsigned long *nr_pages, | |
1016 | unsigned int nr_wins) | |
1017 | { | |
1018 | int ret; | |
1019 | ||
1020 | /* -1: buffer not allocated */ | |
1021 | if (atomic_read(&msc->user_count) != -1) | |
1022 | return -EBUSY; | |
1023 | ||
1024 | if (msc->mode == MSC_MODE_SINGLE) { | |
1025 | if (nr_wins != 1) | |
1026 | return -EINVAL; | |
1027 | ||
1028 | ret = msc_buffer_contig_alloc(msc, nr_pages[0] << PAGE_SHIFT); | |
1029 | } else if (msc->mode == MSC_MODE_MULTI) { | |
1030 | ret = msc_buffer_multi_alloc(msc, nr_pages, nr_wins); | |
1031 | } else { | |
1032 | ret = -ENOTSUPP; | |
1033 | } | |
1034 | ||
1035 | if (!ret) { | |
1036 | /* allocation should be visible before the counter goes to 0 */ | |
1037 | smp_mb__before_atomic(); | |
1038 | ||
1039 | if (WARN_ON_ONCE(atomic_cmpxchg(&msc->user_count, -1, 0) != -1)) | |
1040 | return -EINVAL; | |
1041 | } | |
1042 | ||
1043 | return ret; | |
1044 | } | |
1045 | ||
1046 | /** | |
1047 | * msc_buffer_unlocked_free_unless_used() - free a buffer unless it's in use | |
1048 | * @msc: MSC device | |
1049 | * | |
1050 | * This will free MSC buffer unless it is in use or there is no allocated | |
1051 | * buffer. | |
1052 | * Caller needs to hold msc::buf_mutex. | |
1053 | * | |
1054 | * Return: 0 on successful deallocation or if there was no buffer to | |
1055 | * deallocate, -EBUSY if there are active users. | |
1056 | */ | |
1057 | static int msc_buffer_unlocked_free_unless_used(struct msc *msc) | |
1058 | { | |
1059 | int count, ret = 0; | |
1060 | ||
1061 | count = atomic_cmpxchg(&msc->user_count, 0, -1); | |
1062 | ||
1063 | /* > 0: buffer is allocated and has users */ | |
1064 | if (count > 0) | |
1065 | ret = -EBUSY; | |
1066 | /* 0: buffer is allocated, no users */ | |
1067 | else if (!count) | |
1068 | msc_buffer_free(msc); | |
1069 | /* < 0: no buffer, nothing to do */ | |
1070 | ||
1071 | return ret; | |
1072 | } | |
1073 | ||
1074 | /** | |
1075 | * msc_buffer_free_unless_used() - free a buffer unless it's in use | |
1076 | * @msc: MSC device | |
1077 | * | |
1078 | * This is a locked version of msc_buffer_unlocked_free_unless_used(). | |
1079 | */ | |
1080 | static int msc_buffer_free_unless_used(struct msc *msc) | |
1081 | { | |
1082 | int ret; | |
1083 | ||
1084 | mutex_lock(&msc->buf_mutex); | |
1085 | ret = msc_buffer_unlocked_free_unless_used(msc); | |
1086 | mutex_unlock(&msc->buf_mutex); | |
1087 | ||
1088 | return ret; | |
1089 | } | |
1090 | ||
1091 | /** | |
1092 | * msc_buffer_get_page() - get MSC buffer page at a given offset | |
1093 | * @msc: MSC device | |
1094 | * @pgoff: page offset into the storage buffer | |
1095 | * | |
1096 | * This traverses msc::win_list, so holding msc::buf_mutex is expected from | |
1097 | * the caller. | |
1098 | * | |
1099 | * Return: page if @pgoff corresponds to a valid buffer page or NULL. | |
1100 | */ | |
1101 | static struct page *msc_buffer_get_page(struct msc *msc, unsigned long pgoff) | |
1102 | { | |
1103 | struct msc_window *win; | |
1104 | ||
1105 | if (msc->mode == MSC_MODE_SINGLE) | |
1106 | return msc_buffer_contig_get_page(msc, pgoff); | |
1107 | ||
1108 | list_for_each_entry(win, &msc->win_list, entry) | |
1109 | if (pgoff >= win->pgoff && pgoff < win->pgoff + win->nr_blocks) | |
1110 | goto found; | |
1111 | ||
1112 | return NULL; | |
1113 | ||
1114 | found: | |
1115 | pgoff -= win->pgoff; | |
ba39bd83 | 1116 | return sg_page(&win->sgt.sgl[pgoff]); |
ba82664c AS |
1117 | } |
1118 | ||
1119 | /** | |
1120 | * struct msc_win_to_user_struct - data for copy_to_user() callback | |
1121 | * @buf: userspace buffer to copy data to | |
1122 | * @offset: running offset | |
1123 | */ | |
1124 | struct msc_win_to_user_struct { | |
1125 | char __user *buf; | |
1126 | unsigned long offset; | |
1127 | }; | |
1128 | ||
1129 | /** | |
1130 | * msc_win_to_user() - iterator for msc_buffer_iterate() to copy data to user | |
1131 | * @data: callback's private data | |
1132 | * @src: source buffer | |
1133 | * @len: amount of data to copy from the source buffer | |
1134 | */ | |
1135 | static unsigned long msc_win_to_user(void *data, void *src, size_t len) | |
1136 | { | |
1137 | struct msc_win_to_user_struct *u = data; | |
1138 | unsigned long ret; | |
1139 | ||
1140 | ret = copy_to_user(u->buf + u->offset, src, len); | |
1141 | u->offset += len - ret; | |
1142 | ||
1143 | return ret; | |
1144 | } | |
1145 | ||
1146 | ||
1147 | /* | |
1148 | * file operations' callbacks | |
1149 | */ | |
1150 | ||
1151 | static int intel_th_msc_open(struct inode *inode, struct file *file) | |
1152 | { | |
1153 | struct intel_th_device *thdev = file->private_data; | |
1154 | struct msc *msc = dev_get_drvdata(&thdev->dev); | |
1155 | struct msc_iter *iter; | |
1156 | ||
1157 | if (!capable(CAP_SYS_RAWIO)) | |
1158 | return -EPERM; | |
1159 | ||
1160 | iter = msc_iter_install(msc); | |
a45ff6ed AS |
1161 | if (IS_ERR(iter)) |
1162 | return PTR_ERR(iter); | |
ba82664c AS |
1163 | |
1164 | file->private_data = iter; | |
1165 | ||
1166 | return nonseekable_open(inode, file); | |
1167 | } | |
1168 | ||
1169 | static int intel_th_msc_release(struct inode *inode, struct file *file) | |
1170 | { | |
1171 | struct msc_iter *iter = file->private_data; | |
1172 | struct msc *msc = iter->msc; | |
1173 | ||
1174 | msc_iter_remove(iter, msc); | |
1175 | ||
1176 | return 0; | |
1177 | } | |
1178 | ||
1179 | static ssize_t | |
1180 | msc_single_to_user(struct msc *msc, char __user *buf, loff_t off, size_t len) | |
1181 | { | |
ed392688 | 1182 | unsigned long size = msc->nr_pages << PAGE_SHIFT, rem = len; |
ba82664c AS |
1183 | unsigned long start = off, tocopy = 0; |
1184 | ||
1185 | if (msc->single_wrap) { | |
1186 | start += msc->single_sz; | |
1187 | if (start < size) { | |
1188 | tocopy = min(rem, size - start); | |
1189 | if (copy_to_user(buf, msc->base + start, tocopy)) | |
1190 | return -EFAULT; | |
1191 | ||
1192 | buf += tocopy; | |
1193 | rem -= tocopy; | |
1194 | start += tocopy; | |
1195 | } | |
1196 | ||
1197 | start &= size - 1; | |
1198 | if (rem) { | |
1199 | tocopy = min(rem, msc->single_sz - start); | |
1200 | if (copy_to_user(buf, msc->base + start, tocopy)) | |
1201 | return -EFAULT; | |
1202 | ||
1203 | rem -= tocopy; | |
1204 | } | |
1205 | ||
1206 | return len - rem; | |
1207 | } | |
1208 | ||
1209 | if (copy_to_user(buf, msc->base + start, rem)) | |
1210 | return -EFAULT; | |
1211 | ||
1212 | return len; | |
1213 | } | |
1214 | ||
1215 | static ssize_t intel_th_msc_read(struct file *file, char __user *buf, | |
1216 | size_t len, loff_t *ppos) | |
1217 | { | |
1218 | struct msc_iter *iter = file->private_data; | |
1219 | struct msc *msc = iter->msc; | |
1220 | size_t size; | |
1221 | loff_t off = *ppos; | |
1222 | ssize_t ret = 0; | |
1223 | ||
1224 | if (!atomic_inc_unless_negative(&msc->user_count)) | |
1225 | return 0; | |
1226 | ||
ba82664c AS |
1227 | if (msc->mode == MSC_MODE_SINGLE && !msc->single_wrap) |
1228 | size = msc->single_sz; | |
1229 | else | |
1230 | size = msc->nr_pages << PAGE_SHIFT; | |
1231 | ||
1232 | if (!size) | |
2bed074a | 1233 | goto put_count; |
ba82664c | 1234 | |
2bed074a | 1235 | if (off >= size) |
ba82664c | 1236 | goto put_count; |
2bed074a | 1237 | |
ba82664c AS |
1238 | if (off + len >= size) |
1239 | len = size - off; | |
1240 | ||
1241 | if (msc->mode == MSC_MODE_SINGLE) { | |
1242 | ret = msc_single_to_user(msc, buf, off, len); | |
1243 | if (ret >= 0) | |
1244 | *ppos += ret; | |
1245 | } else if (msc->mode == MSC_MODE_MULTI) { | |
1246 | struct msc_win_to_user_struct u = { | |
1247 | .buf = buf, | |
1248 | .offset = 0, | |
1249 | }; | |
1250 | ||
1251 | ret = msc_buffer_iterate(iter, len, &u, msc_win_to_user); | |
1252 | if (ret >= 0) | |
1253 | *ppos = iter->offset; | |
1254 | } else { | |
1255 | ret = -ENOTSUPP; | |
1256 | } | |
1257 | ||
1258 | put_count: | |
1259 | atomic_dec(&msc->user_count); | |
1260 | ||
1261 | return ret; | |
1262 | } | |
1263 | ||
1264 | /* | |
1265 | * vm operations callbacks (vm_ops) | |
1266 | */ | |
1267 | ||
1268 | static void msc_mmap_open(struct vm_area_struct *vma) | |
1269 | { | |
1270 | struct msc_iter *iter = vma->vm_file->private_data; | |
1271 | struct msc *msc = iter->msc; | |
1272 | ||
1273 | atomic_inc(&msc->mmap_count); | |
1274 | } | |
1275 | ||
1276 | static void msc_mmap_close(struct vm_area_struct *vma) | |
1277 | { | |
1278 | struct msc_iter *iter = vma->vm_file->private_data; | |
1279 | struct msc *msc = iter->msc; | |
1280 | unsigned long pg; | |
1281 | ||
1282 | if (!atomic_dec_and_mutex_lock(&msc->mmap_count, &msc->buf_mutex)) | |
1283 | return; | |
1284 | ||
0139aa7b | 1285 | /* drop page _refcounts */ |
ba82664c AS |
1286 | for (pg = 0; pg < msc->nr_pages; pg++) { |
1287 | struct page *page = msc_buffer_get_page(msc, pg); | |
1288 | ||
1289 | if (WARN_ON_ONCE(!page)) | |
1290 | continue; | |
1291 | ||
1292 | if (page->mapping) | |
1293 | page->mapping = NULL; | |
1294 | } | |
1295 | ||
1296 | /* last mapping -- drop user_count */ | |
1297 | atomic_dec(&msc->user_count); | |
1298 | mutex_unlock(&msc->buf_mutex); | |
1299 | } | |
1300 | ||
42df0509 | 1301 | static vm_fault_t msc_mmap_fault(struct vm_fault *vmf) |
ba82664c | 1302 | { |
11bac800 | 1303 | struct msc_iter *iter = vmf->vma->vm_file->private_data; |
ba82664c AS |
1304 | struct msc *msc = iter->msc; |
1305 | ||
1306 | vmf->page = msc_buffer_get_page(msc, vmf->pgoff); | |
1307 | if (!vmf->page) | |
1308 | return VM_FAULT_SIGBUS; | |
1309 | ||
1310 | get_page(vmf->page); | |
11bac800 | 1311 | vmf->page->mapping = vmf->vma->vm_file->f_mapping; |
ba82664c AS |
1312 | vmf->page->index = vmf->pgoff; |
1313 | ||
1314 | return 0; | |
1315 | } | |
1316 | ||
1317 | static const struct vm_operations_struct msc_mmap_ops = { | |
1318 | .open = msc_mmap_open, | |
1319 | .close = msc_mmap_close, | |
1320 | .fault = msc_mmap_fault, | |
1321 | }; | |
1322 | ||
1323 | static int intel_th_msc_mmap(struct file *file, struct vm_area_struct *vma) | |
1324 | { | |
1325 | unsigned long size = vma->vm_end - vma->vm_start; | |
1326 | struct msc_iter *iter = vma->vm_file->private_data; | |
1327 | struct msc *msc = iter->msc; | |
1328 | int ret = -EINVAL; | |
1329 | ||
1330 | if (!size || offset_in_page(size)) | |
1331 | return -EINVAL; | |
1332 | ||
1333 | if (vma->vm_pgoff) | |
1334 | return -EINVAL; | |
1335 | ||
1336 | /* grab user_count once per mmap; drop in msc_mmap_close() */ | |
1337 | if (!atomic_inc_unless_negative(&msc->user_count)) | |
1338 | return -EINVAL; | |
1339 | ||
1340 | if (msc->mode != MSC_MODE_SINGLE && | |
1341 | msc->mode != MSC_MODE_MULTI) | |
1342 | goto out; | |
1343 | ||
1344 | if (size >> PAGE_SHIFT != msc->nr_pages) | |
1345 | goto out; | |
1346 | ||
1347 | atomic_set(&msc->mmap_count, 1); | |
1348 | ret = 0; | |
1349 | ||
1350 | out: | |
1351 | if (ret) | |
1352 | atomic_dec(&msc->user_count); | |
1353 | ||
1354 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); | |
1355 | vma->vm_flags |= VM_DONTEXPAND | VM_DONTCOPY; | |
1356 | vma->vm_ops = &msc_mmap_ops; | |
1357 | return ret; | |
1358 | } | |
1359 | ||
1360 | static const struct file_operations intel_th_msc_fops = { | |
1361 | .open = intel_th_msc_open, | |
1362 | .release = intel_th_msc_release, | |
1363 | .read = intel_th_msc_read, | |
1364 | .mmap = intel_th_msc_mmap, | |
1365 | .llseek = no_llseek, | |
8e9a2beb | 1366 | .owner = THIS_MODULE, |
ba82664c AS |
1367 | }; |
1368 | ||
8d415512 AS |
1369 | static void intel_th_msc_wait_empty(struct intel_th_device *thdev) |
1370 | { | |
1371 | struct msc *msc = dev_get_drvdata(&thdev->dev); | |
1372 | unsigned long count; | |
1373 | u32 reg; | |
1374 | ||
1375 | for (reg = 0, count = MSC_PLE_WAITLOOP_DEPTH; | |
1376 | count && !(reg & MSCSTS_PLE); count--) { | |
1377 | reg = __raw_readl(msc->reg_base + REG_MSU_MSC0STS); | |
1378 | cpu_relax(); | |
1379 | } | |
1380 | ||
1381 | if (!count) | |
1382 | dev_dbg(msc_dev(msc), "timeout waiting for MSC0 PLE\n"); | |
1383 | } | |
1384 | ||
ba82664c AS |
1385 | static int intel_th_msc_init(struct msc *msc) |
1386 | { | |
1387 | atomic_set(&msc->user_count, -1); | |
1388 | ||
1389 | msc->mode = MSC_MODE_MULTI; | |
1390 | mutex_init(&msc->buf_mutex); | |
1391 | INIT_LIST_HEAD(&msc->win_list); | |
ba82664c AS |
1392 | INIT_LIST_HEAD(&msc->iter_list); |
1393 | ||
1394 | msc->burst_len = | |
1395 | (ioread32(msc->reg_base + REG_MSU_MSC0CTL) & MSC_LEN) >> | |
1396 | __ffs(MSC_LEN); | |
1397 | ||
1398 | return 0; | |
1399 | } | |
1400 | ||
aad14ad3 AS |
1401 | static void msc_win_switch(struct msc *msc) |
1402 | { | |
1403 | struct msc_window *last, *first; | |
1404 | ||
1405 | first = list_first_entry(&msc->win_list, struct msc_window, entry); | |
1406 | last = list_last_entry(&msc->win_list, struct msc_window, entry); | |
1407 | ||
1408 | if (msc_is_last_win(msc->cur_win)) | |
1409 | msc->cur_win = first; | |
1410 | else | |
1411 | msc->cur_win = list_next_entry(msc->cur_win, entry); | |
1412 | ||
1413 | msc->base = msc_win_block(msc->cur_win, 0); | |
1414 | msc->base_addr = msc_win_baddr(msc->cur_win, 0); | |
1415 | ||
1416 | intel_th_trace_switch(msc->thdev); | |
1417 | } | |
1418 | ||
aac8da65 AS |
1419 | static irqreturn_t intel_th_msc_interrupt(struct intel_th_device *thdev) |
1420 | { | |
1421 | struct msc *msc = dev_get_drvdata(&thdev->dev); | |
1422 | u32 msusts = ioread32(msc->msu_base + REG_MSU_MSUSTS); | |
1423 | u32 mask = msc->index ? MSUSTS_MSC1BLAST : MSUSTS_MSC0BLAST; | |
1424 | ||
1425 | if (!(msusts & mask)) { | |
1426 | if (msc->enabled) | |
1427 | return IRQ_HANDLED; | |
1428 | return IRQ_NONE; | |
1429 | } | |
1430 | ||
1431 | return IRQ_HANDLED; | |
1432 | } | |
1433 | ||
ba82664c AS |
1434 | static const char * const msc_mode[] = { |
1435 | [MSC_MODE_SINGLE] = "single", | |
1436 | [MSC_MODE_MULTI] = "multi", | |
1437 | [MSC_MODE_EXI] = "ExI", | |
1438 | [MSC_MODE_DEBUG] = "debug", | |
1439 | }; | |
1440 | ||
1441 | static ssize_t | |
1442 | wrap_show(struct device *dev, struct device_attribute *attr, char *buf) | |
1443 | { | |
1444 | struct msc *msc = dev_get_drvdata(dev); | |
1445 | ||
1446 | return scnprintf(buf, PAGE_SIZE, "%d\n", msc->wrap); | |
1447 | } | |
1448 | ||
1449 | static ssize_t | |
1450 | wrap_store(struct device *dev, struct device_attribute *attr, const char *buf, | |
1451 | size_t size) | |
1452 | { | |
1453 | struct msc *msc = dev_get_drvdata(dev); | |
1454 | unsigned long val; | |
1455 | int ret; | |
1456 | ||
1457 | ret = kstrtoul(buf, 10, &val); | |
1458 | if (ret) | |
1459 | return ret; | |
1460 | ||
1461 | msc->wrap = !!val; | |
1462 | ||
1463 | return size; | |
1464 | } | |
1465 | ||
1466 | static DEVICE_ATTR_RW(wrap); | |
1467 | ||
1468 | static ssize_t | |
1469 | mode_show(struct device *dev, struct device_attribute *attr, char *buf) | |
1470 | { | |
1471 | struct msc *msc = dev_get_drvdata(dev); | |
1472 | ||
1473 | return scnprintf(buf, PAGE_SIZE, "%s\n", msc_mode[msc->mode]); | |
1474 | } | |
1475 | ||
1476 | static ssize_t | |
1477 | mode_store(struct device *dev, struct device_attribute *attr, const char *buf, | |
1478 | size_t size) | |
1479 | { | |
1480 | struct msc *msc = dev_get_drvdata(dev); | |
1481 | size_t len = size; | |
1482 | char *cp; | |
1483 | int i, ret; | |
1484 | ||
1485 | if (!capable(CAP_SYS_RAWIO)) | |
1486 | return -EPERM; | |
1487 | ||
1488 | cp = memchr(buf, '\n', len); | |
1489 | if (cp) | |
1490 | len = cp - buf; | |
1491 | ||
1492 | for (i = 0; i < ARRAY_SIZE(msc_mode); i++) | |
1493 | if (!strncmp(msc_mode[i], buf, len)) | |
1494 | goto found; | |
1495 | ||
1496 | return -EINVAL; | |
1497 | ||
1498 | found: | |
1499 | mutex_lock(&msc->buf_mutex); | |
1500 | ret = msc_buffer_unlocked_free_unless_used(msc); | |
1501 | if (!ret) | |
1502 | msc->mode = i; | |
1503 | mutex_unlock(&msc->buf_mutex); | |
1504 | ||
1505 | return ret ? ret : size; | |
1506 | } | |
1507 | ||
1508 | static DEVICE_ATTR_RW(mode); | |
1509 | ||
1510 | static ssize_t | |
1511 | nr_pages_show(struct device *dev, struct device_attribute *attr, char *buf) | |
1512 | { | |
1513 | struct msc *msc = dev_get_drvdata(dev); | |
1514 | struct msc_window *win; | |
1515 | size_t count = 0; | |
1516 | ||
1517 | mutex_lock(&msc->buf_mutex); | |
1518 | ||
1519 | if (msc->mode == MSC_MODE_SINGLE) | |
1520 | count = scnprintf(buf, PAGE_SIZE, "%ld\n", msc->nr_pages); | |
1521 | else if (msc->mode == MSC_MODE_MULTI) { | |
1522 | list_for_each_entry(win, &msc->win_list, entry) { | |
1523 | count += scnprintf(buf + count, PAGE_SIZE - count, | |
1524 | "%d%c", win->nr_blocks, | |
1525 | msc_is_last_win(win) ? '\n' : ','); | |
1526 | } | |
1527 | } else { | |
1528 | count = scnprintf(buf, PAGE_SIZE, "unsupported\n"); | |
1529 | } | |
1530 | ||
1531 | mutex_unlock(&msc->buf_mutex); | |
1532 | ||
1533 | return count; | |
1534 | } | |
1535 | ||
1536 | static ssize_t | |
1537 | nr_pages_store(struct device *dev, struct device_attribute *attr, | |
1538 | const char *buf, size_t size) | |
1539 | { | |
1540 | struct msc *msc = dev_get_drvdata(dev); | |
1541 | unsigned long val, *win = NULL, *rewin; | |
1542 | size_t len = size; | |
1543 | const char *p = buf; | |
1544 | char *end, *s; | |
1545 | int ret, nr_wins = 0; | |
1546 | ||
1547 | if (!capable(CAP_SYS_RAWIO)) | |
1548 | return -EPERM; | |
1549 | ||
1550 | ret = msc_buffer_free_unless_used(msc); | |
1551 | if (ret) | |
1552 | return ret; | |
1553 | ||
1554 | /* scan the comma-separated list of allocation sizes */ | |
1555 | end = memchr(buf, '\n', len); | |
1556 | if (end) | |
1557 | len = end - buf; | |
1558 | ||
1559 | do { | |
1560 | end = memchr(p, ',', len); | |
1561 | s = kstrndup(p, end ? end - p : len, GFP_KERNEL); | |
6575cbd6 AS |
1562 | if (!s) { |
1563 | ret = -ENOMEM; | |
1564 | goto free_win; | |
1565 | } | |
1566 | ||
ba82664c AS |
1567 | ret = kstrtoul(s, 10, &val); |
1568 | kfree(s); | |
1569 | ||
1570 | if (ret || !val) | |
1571 | goto free_win; | |
1572 | ||
1573 | if (nr_wins && msc->mode == MSC_MODE_SINGLE) { | |
1574 | ret = -EINVAL; | |
1575 | goto free_win; | |
1576 | } | |
1577 | ||
1578 | nr_wins++; | |
1579 | rewin = krealloc(win, sizeof(*win) * nr_wins, GFP_KERNEL); | |
1580 | if (!rewin) { | |
1581 | kfree(win); | |
1582 | return -ENOMEM; | |
1583 | } | |
1584 | ||
1585 | win = rewin; | |
1586 | win[nr_wins - 1] = val; | |
1587 | ||
1588 | if (!end) | |
1589 | break; | |
1590 | ||
ec5b5ad6 AS |
1591 | /* consume the number and the following comma, hence +1 */ |
1592 | len -= end - p + 1; | |
ba82664c AS |
1593 | p = end + 1; |
1594 | } while (len); | |
1595 | ||
1596 | mutex_lock(&msc->buf_mutex); | |
1597 | ret = msc_buffer_alloc(msc, win, nr_wins); | |
1598 | mutex_unlock(&msc->buf_mutex); | |
1599 | ||
1600 | free_win: | |
1601 | kfree(win); | |
1602 | ||
1603 | return ret ? ret : size; | |
1604 | } | |
1605 | ||
1606 | static DEVICE_ATTR_RW(nr_pages); | |
1607 | ||
6cac7866 AS |
1608 | static ssize_t |
1609 | win_switch_store(struct device *dev, struct device_attribute *attr, | |
1610 | const char *buf, size_t size) | |
1611 | { | |
1612 | struct msc *msc = dev_get_drvdata(dev); | |
1613 | unsigned long val; | |
1614 | int ret; | |
1615 | ||
1616 | ret = kstrtoul(buf, 10, &val); | |
1617 | if (ret) | |
1618 | return ret; | |
1619 | ||
1620 | if (val != 1) | |
1621 | return -EINVAL; | |
1622 | ||
1623 | mutex_lock(&msc->buf_mutex); | |
1624 | if (msc->mode != MSC_MODE_MULTI) | |
1625 | ret = -ENOTSUPP; | |
1626 | else | |
aad14ad3 | 1627 | msc_win_switch(msc); |
6cac7866 AS |
1628 | mutex_unlock(&msc->buf_mutex); |
1629 | ||
1630 | return ret ? ret : size; | |
1631 | } | |
1632 | ||
1633 | static DEVICE_ATTR_WO(win_switch); | |
1634 | ||
ba82664c AS |
1635 | static struct attribute *msc_output_attrs[] = { |
1636 | &dev_attr_wrap.attr, | |
1637 | &dev_attr_mode.attr, | |
1638 | &dev_attr_nr_pages.attr, | |
6cac7866 | 1639 | &dev_attr_win_switch.attr, |
ba82664c AS |
1640 | NULL, |
1641 | }; | |
1642 | ||
1643 | static struct attribute_group msc_output_group = { | |
1644 | .attrs = msc_output_attrs, | |
1645 | }; | |
1646 | ||
1647 | static int intel_th_msc_probe(struct intel_th_device *thdev) | |
1648 | { | |
1649 | struct device *dev = &thdev->dev; | |
1650 | struct resource *res; | |
1651 | struct msc *msc; | |
1652 | void __iomem *base; | |
1653 | int err; | |
1654 | ||
1655 | res = intel_th_device_get_resource(thdev, IORESOURCE_MEM, 0); | |
1656 | if (!res) | |
1657 | return -ENODEV; | |
1658 | ||
1659 | base = devm_ioremap(dev, res->start, resource_size(res)); | |
73061da0 DC |
1660 | if (!base) |
1661 | return -ENOMEM; | |
ba82664c AS |
1662 | |
1663 | msc = devm_kzalloc(dev, sizeof(*msc), GFP_KERNEL); | |
1664 | if (!msc) | |
1665 | return -ENOMEM; | |
1666 | ||
aac8da65 AS |
1667 | res = intel_th_device_get_resource(thdev, IORESOURCE_IRQ, 1); |
1668 | if (!res) | |
1669 | msc->do_irq = 1; | |
1670 | ||
ba82664c AS |
1671 | msc->index = thdev->id; |
1672 | ||
1673 | msc->thdev = thdev; | |
1674 | msc->reg_base = base + msc->index * 0x100; | |
aac8da65 AS |
1675 | msc->msu_base = base; |
1676 | ||
1677 | err = intel_th_msu_init(msc); | |
1678 | if (err) | |
1679 | return err; | |
ba82664c AS |
1680 | |
1681 | err = intel_th_msc_init(msc); | |
1682 | if (err) | |
1683 | return err; | |
1684 | ||
ba82664c AS |
1685 | dev_set_drvdata(dev, msc); |
1686 | ||
1687 | return 0; | |
1688 | } | |
1689 | ||
1690 | static void intel_th_msc_remove(struct intel_th_device *thdev) | |
1691 | { | |
f152dfee AS |
1692 | struct msc *msc = dev_get_drvdata(&thdev->dev); |
1693 | int ret; | |
1694 | ||
1695 | intel_th_msc_deactivate(thdev); | |
aac8da65 | 1696 | intel_th_msu_deinit(msc); |
f152dfee AS |
1697 | |
1698 | /* | |
1699 | * Buffers should not be used at this point except if the | |
1700 | * output character device is still open and the parent | |
1701 | * device gets detached from its bus, which is a FIXME. | |
1702 | */ | |
1703 | ret = msc_buffer_free_unless_used(msc); | |
1704 | WARN_ON_ONCE(ret); | |
ba82664c AS |
1705 | } |
1706 | ||
1707 | static struct intel_th_driver intel_th_msc_driver = { | |
1708 | .probe = intel_th_msc_probe, | |
1709 | .remove = intel_th_msc_remove, | |
aac8da65 | 1710 | .irq = intel_th_msc_interrupt, |
8d415512 | 1711 | .wait_empty = intel_th_msc_wait_empty, |
ba82664c AS |
1712 | .activate = intel_th_msc_activate, |
1713 | .deactivate = intel_th_msc_deactivate, | |
1714 | .fops = &intel_th_msc_fops, | |
9d482aed | 1715 | .attr_group = &msc_output_group, |
ba82664c AS |
1716 | .driver = { |
1717 | .name = "msc", | |
1718 | .owner = THIS_MODULE, | |
1719 | }, | |
1720 | }; | |
1721 | ||
1722 | module_driver(intel_th_msc_driver, | |
1723 | intel_th_driver_register, | |
1724 | intel_th_driver_unregister); | |
1725 | ||
1726 | MODULE_LICENSE("GPL v2"); | |
1727 | MODULE_DESCRIPTION("Intel(R) Trace Hub Memory Storage Unit driver"); | |
1728 | MODULE_AUTHOR("Alexander Shishkin <alexander.shishkin@linux.intel.com>"); |