projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
powerpc/powernv/vas: Define copy/paste interfaces
[linux-2.6-block.git] / arch / powerpc / platforms / powernv / vas-window.c
CommitLineData
4dea2d1a
SB		 1/*
2 * Copyright 2016-17 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
180fe15a
SB		 10#define pr_fmt(fmt) "vas: " fmt
11
4dea2d1a
SB		 12#include <linux/types.h>
13#include <linux/mutex.h>
180fe15a
SB		 14#include <linux/slab.h>
15#include <linux/io.h>
b25b33ac 16#include <linux/log2.h>
62c4eda4
SB		 17#include <linux/rcupdate.h>
18#include <linux/cred.h>
4dea2d1a
SB		 19
20#include "vas.h"
2392c8c8 21#include "copy-paste.h"
4dea2d1a 22
180fe15a
SB		 23/*
24 * Compute the paste address region for the window @window using the
25 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
26 */
27static void compute_paste_address(struct vas_window *window, u64 *addr, int *len)
28{
29 int winid;
30 u64 base, shift;
31
32 base = window->vinst->paste_base_addr;
33 shift = window->vinst->paste_win_id_shift;
34 winid = window->winid;
35
36 *addr = base + (winid << shift);
37 if (len)
38 *len = PAGE_SIZE;
39
40 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
41}
42
43static inline void get_hvwc_mmio_bar(struct vas_window *window,
44 u64 *start, int *len)
45{
46 u64 pbaddr;
47
48 pbaddr = window->vinst->hvwc_bar_start;
49 *start = pbaddr + window->winid * VAS_HVWC_SIZE;
50 *len = VAS_HVWC_SIZE;
51}
52
53static inline void get_uwc_mmio_bar(struct vas_window *window,
54 u64 *start, int *len)
55{
56 u64 pbaddr;
57
58 pbaddr = window->vinst->uwc_bar_start;
59 *start = pbaddr + window->winid * VAS_UWC_SIZE;
60 *len = VAS_UWC_SIZE;
61}
62
63/*
64 * Map the paste bus address of the given send window into kernel address
65 * space. Unlike MMIO regions (map_mmio_region() below), paste region must
66 * be mapped cache-able and is only applicable to send windows.
67 */
5239af67 68static void *map_paste_region(struct vas_window *txwin)
180fe15a
SB		 69{
70 int len;
71 void *map;
72 char *name;
73 u64 start;
74
75 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
76 txwin->winid);
77 if (!name)
78 goto free_name;
79
80 txwin->paste_addr_name = name;
81 compute_paste_address(txwin, &start, &len);
82
83 if (!request_mem_region(start, len, name)) {
84 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
85 __func__, start, len);
86 goto free_name;
87 }
88
89 map = ioremap_cache(start, len);
90 if (!map) {
91 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
92 start, len);
93 goto free_name;
94 }
95
96 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
97 return map;
98
99free_name:
100 kfree(name);
101 return ERR_PTR(-ENOMEM);
102}
103
180fe15a
SB		 104static void *map_mmio_region(char *name, u64 start, int len)
105{
106 void *map;
107
108 if (!request_mem_region(start, len, name)) {
109 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
110 __func__, start, len);
111 return NULL;
112 }
113
114 map = ioremap(start, len);
115 if (!map) {
116 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
117 len);
118 return NULL;
119 }
120
121 return map;
122}
123
124static void unmap_region(void *addr, u64 start, int len)
125{
126 iounmap(addr);
127 release_mem_region((phys_addr_t)start, len);
128}
129
130/*
131 * Unmap the paste address region for a window.
132 */
98271d41 133static void unmap_paste_region(struct vas_window *window)
180fe15a
SB		 134{
135 int len;
136 u64 busaddr_start;
137
138 if (window->paste_kaddr) {
139 compute_paste_address(window, &busaddr_start, &len);
140 unmap_region(window->paste_kaddr, busaddr_start, len);
141 window->paste_kaddr = NULL;
142 kfree(window->paste_addr_name);
143 window->paste_addr_name = NULL;
144 }
145}
146
147/*
148 * Unmap the MMIO regions for a window.
149 */
150static void unmap_winctx_mmio_bars(struct vas_window *window)
151{
152 int len;
153 u64 busaddr_start;
154
155 if (window->hvwc_map) {
156 get_hvwc_mmio_bar(window, &busaddr_start, &len);
157 unmap_region(window->hvwc_map, busaddr_start, len);
158 window->hvwc_map = NULL;
159 }
160
161 if (window->uwc_map) {
162 get_uwc_mmio_bar(window, &busaddr_start, &len);
163 unmap_region(window->uwc_map, busaddr_start, len);
164 window->uwc_map = NULL;
165 }
166}
167
168/*
169 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
170 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
171 * Map these bus addresses and save the mapped kernel addresses in @window.
172 */
173int map_winctx_mmio_bars(struct vas_window *window)
174{
175 int len;
176 u64 start;
177
178 get_hvwc_mmio_bar(window, &start, &len);
179 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
180
181 get_uwc_mmio_bar(window, &start, &len);
182 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
183
184 if (!window->hvwc_map || !window->uwc_map) {
185 unmap_winctx_mmio_bars(window);
186 return -1;
187 }
188
189 return 0;
190}
191
b25b33ac
SB		 192/*
193 * Reset all valid registers in the HV and OS/User Window Contexts for
194 * the window identified by @window.
195 *
196 * NOTE: We cannot really use a for loop to reset window context. Not all
197 * offsets in a window context are valid registers and the valid
198 * registers are not sequential. And, we can only write to offsets
199 * with valid registers.
200 */
201void reset_window_regs(struct vas_window *window)
202{
203 write_hvwc_reg(window, VREG(LPID), 0ULL);
204 write_hvwc_reg(window, VREG(PID), 0ULL);
205 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
206 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
207 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
208 write_hvwc_reg(window, VREG(AMR), 0ULL);
209 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
210 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
211 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
212 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
213 write_hvwc_reg(window, VREG(PSWID), 0ULL);
214 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
215 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
216 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
217 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
218 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
219 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
220 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
221 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
222 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
223 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
224 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
225 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
226 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
227 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
228 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
229 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
230 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
231 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
232 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
233 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
234 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
235 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
236
237 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
238
239 /*
240 * The send and receive window credit adder registers are also
241 * accessible from HVWC and have been initialized above. We don't
242 * need to initialize from the OS/User Window Context, so skip
243 * following calls:
244 *
245 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
246 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
247 */
248}
249
250/*
251 * Initialize window context registers related to Address Translation.
252 * These registers are common to send/receive windows although they
253 * differ for user/kernel windows. As we resolve the TODOs we may
254 * want to add fields to vas_winctx and move the initialization to
255 * init_vas_winctx_regs().
256 */
257static void init_xlate_regs(struct vas_window *window, bool user_win)
258{
259 u64 lpcr, val;
260
261 /*
262 * MSR_TA, MSR_US are false for both kernel and user.
263 * MSR_DR and MSR_PR are false for kernel.
264 */
265 val = 0ULL;
266 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
267 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
268 if (user_win) {
269 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
270 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
271 }
272 write_hvwc_reg(window, VREG(XLATE_MSR), val);
273
274 lpcr = mfspr(SPRN_LPCR);
275 val = 0ULL;
276 /*
277 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
278 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
279 *
280 * NOTE: From Section 1.3.1, Address Translation Context of the
281 * Nest MMU Workbook, LPCR_SC should be 0 for Power9.
282 */
283 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
284 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
285 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
286 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
287 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
288
289 /*
290 * Section 1.3.1 (Address translation Context) of NMMU workbook.
291 * 0b00 Hashed Page Table mode
292 * 0b01 Reserved
293 * 0b10 Radix on HPT
294 * 0b11 Radix on Radix
295 */
296 val = 0ULL;
297 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
298 write_hvwc_reg(window, VREG(XLATE_CTL), val);
299
300 /*
301 * TODO: Can we mfspr(AMR) even for user windows?
302 */
303 val = 0ULL;
304 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
305 write_hvwc_reg(window, VREG(AMR), val);
306
307 val = 0ULL;
308 val = SET_FIELD(VAS_SEIDR, val, 0);
309 write_hvwc_reg(window, VREG(SEIDR), val);
310}
311
312/*
313 * Initialize Reserved Send Buffer Count for the send window. It involves
314 * writing to the register, reading it back to confirm that the hardware
315 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
316 *
317 * Since we can only make a best-effort attempt to fulfill the request,
318 * we don't return any errors if we cannot.
319 *
320 * TODO: Reserved (aka dedicated) send buffers are not supported yet.
321 */
322static void init_rsvd_tx_buf_count(struct vas_window *txwin,
323 struct vas_winctx *winctx)
324{
325 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
326}
327
328/*
329 * init_winctx_regs()
330 * Initialize window context registers for a receive window.
331 * Except for caching control and marking window open, the registers
332 * are initialized in the order listed in Section 3.1.4 (Window Context
333 * Cache Register Details) of the VAS workbook although they don't need
334 * to be.
335 *
336 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
337 * (so that it can get a large contiguous area) and passes that buffer
338 * to kernel via device tree. We now write that buffer address to the
339 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
340 * write the per-chip RX FIFO addresses to the windows during boot-up
341 * as a one-time task? That could work for NX but what about other
342 * receivers? Let the receivers tell us the rx-fifo buffers for now.
343 */
344int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx)
345{
346 u64 val;
347 int fifo_size;
348
349 reset_window_regs(window);
350
351 val = 0ULL;
352 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
353 write_hvwc_reg(window, VREG(LPID), val);
354
355 val = 0ULL;
356 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
357 write_hvwc_reg(window, VREG(PID), val);
358
359 init_xlate_regs(window, winctx->user_win);
360
361 val = 0ULL;
362 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0);
363 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
364
365 /* In PowerNV, interrupts go to HV. */
366 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
367
368 val = 0ULL;
369 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
370 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
371
372 val = 0ULL;
373 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
374 write_hvwc_reg(window, VREG(PSWID), val);
375
376 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
377 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
378 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
379
380 /*
381 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
382 * register as is - do NOT shift the address into VAS_LFIFO_BAR
383 * bit fields! Ok to set the page migration select fields -
384 * VAS ignores the lower 10+ bits in the address anyway, because
385 * the minimum FIFO size is 1K?
386 *
387 * See also: Design note in function header.
388 */
389 val = __pa(winctx->rx_fifo);
390 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
391 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
392
393 val = 0ULL;
394 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
395 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
396
397 val = 0ULL;
398 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
399 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
400 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
401
402 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
403 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
404 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
405
406 val = 0ULL;
407 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
408 write_hvwc_reg(window, VREG(LRX_WCRED), val);
409
410 val = 0ULL;
411 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
412 write_hvwc_reg(window, VREG(TX_WCRED), val);
413
414 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
415 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
416
417 fifo_size = winctx->rx_fifo_size / 1024;
418
419 val = 0ULL;
420 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
421 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
422
423 /* Update window control and caching control registers last so
424 * we mark the window open only after fully initializing it and
425 * pushing context to cache.
426 */
427
428 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
429
430 init_rsvd_tx_buf_count(window, winctx);
431
432 /* for a send window, point to the matching receive window */
433 val = 0ULL;
434 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
435 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
436
437 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
438
439 val = 0ULL;
440 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
441 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
442 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
443 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
444 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
445
446 val = 0ULL;
447 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
448 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
449
450 val = 0ULL;
451 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
452 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
453
454 val = 0ULL;
455 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
456 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
457
458 val = 0ULL;
459 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
460 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
461 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
462
463 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
464
465 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
466 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
467 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
468
469 /* Finally, push window context to memory and... */
470 val = 0ULL;
471 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
472 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
473
474 /* ... mark the window open for business */
475 val = 0ULL;
476 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
477 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
478 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
479 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
480 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
481 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
482 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
483 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
484 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
485 write_hvwc_reg(window, VREG(WINCTL), val);
486
487 return 0;
488}
489
bbfe59f8
SB		 490static DEFINE_SPINLOCK(vas_ida_lock);
491
492static void vas_release_window_id(struct ida *ida, int winid)
493{
494 spin_lock(&vas_ida_lock);
495 ida_remove(ida, winid);
496 spin_unlock(&vas_ida_lock);
497}
498
499static int vas_assign_window_id(struct ida *ida)
500{
501 int rc, winid;
502
503 do {
504 rc = ida_pre_get(ida, GFP_KERNEL);
505 if (!rc)
506 return -EAGAIN;
507
508 spin_lock(&vas_ida_lock);
509 rc = ida_get_new(ida, &winid);
510 spin_unlock(&vas_ida_lock);
511 } while (rc == -EAGAIN);
512
513 if (rc)
514 return rc;
515
516 if (winid > VAS_WINDOWS_PER_CHIP) {
517 pr_err("Too many (%d) open windows\n", winid);
518 vas_release_window_id(ida, winid);
519 return -EAGAIN;
520 }
521
522 return winid;
523}
524
98271d41 525static void vas_window_free(struct vas_window *window)
bbfe59f8
SB		 526{
527 int winid = window->winid;
528 struct vas_instance *vinst = window->vinst;
529
530 unmap_winctx_mmio_bars(window);
531 kfree(window);
532
533 vas_release_window_id(&vinst->ida, winid);
534}
535
62c4eda4 536static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
bbfe59f8
SB		 537{
538 int winid;
539 struct vas_window *window;
540
541 winid = vas_assign_window_id(&vinst->ida);
542 if (winid < 0)
543 return ERR_PTR(winid);
544
545 window = kzalloc(sizeof(*window), GFP_KERNEL);
546 if (!window)
547 goto out_free;
548
549 window->vinst = vinst;
550 window->winid = winid;
551
552 if (map_winctx_mmio_bars(window))
553 goto out_free;
554
555 return window;
556
557out_free:
558 kfree(window);
559 vas_release_window_id(&vinst->ida, winid);
560 return ERR_PTR(-ENOMEM);
561}
562
98271d41
SB		 563static void put_rx_win(struct vas_window *rxwin)
564{
565 /* Better not be a send window! */
566 WARN_ON_ONCE(rxwin->tx_win);
567
568 atomic_dec(&rxwin->num_txwins);
569}
570
62c4eda4
SB		 571/*
572 * Get the VAS receive window associated with NX engine identified
573 * by @cop and if applicable, @pswid.
574 *
575 * See also function header of set_vinst_win().
576 */
5239af67 577static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
62c4eda4
SB		 578 enum vas_cop_type cop, u32 pswid)
579{
580 struct vas_window *rxwin;
581
582 mutex_lock(&vinst->mutex);
583
584 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI)
585 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
586 else
587 rxwin = ERR_PTR(-EINVAL);
588
589 if (!IS_ERR(rxwin))
590 atomic_inc(&rxwin->num_txwins);
591
592 mutex_unlock(&vinst->mutex);
593
594 return rxwin;
595}
596
597/*
598 * We have two tables of windows in a VAS instance. The first one,
599 * ->windows[], contains all the windows in the instance and allows
600 * looking up a window by its id. It is used to look up send windows
601 * during fault handling and receive windows when pairing user space
602 * send/receive windows.
603 *
604 * The second table, ->rxwin[], contains receive windows that are
605 * associated with NX engines. This table has VAS_COP_TYPE_MAX
606 * entries and is used to look up a receive window by its
607 * coprocessor type.
608 *
609 * Here, we save @window in the ->windows[] table. If it is a receive
610 * window, we also save the window in the ->rxwin[] table.
611 */
612static void set_vinst_win(struct vas_instance *vinst,
613 struct vas_window *window)
614{
615 int id = window->winid;
616
617 mutex_lock(&vinst->mutex);
618
619 /*
620 * There should only be one receive window for a coprocessor type
621 * unless its a user (FTW) window.
622 */
623 if (!window->user_win && !window->tx_win) {
624 WARN_ON_ONCE(vinst->rxwin[window->cop]);
625 vinst->rxwin[window->cop] = window;
626 }
627
628 WARN_ON_ONCE(vinst->windows[id] != NULL);
629 vinst->windows[id] = window;
630
631 mutex_unlock(&vinst->mutex);
632}
633
634/*
635 * Clear this window from the table(s) of windows for this VAS instance.
636 * See also function header of set_vinst_win().
637 */
98271d41 638static void clear_vinst_win(struct vas_window *window)
62c4eda4
SB		 639{
640 int id = window->winid;
641 struct vas_instance *vinst = window->vinst;
642
643 mutex_lock(&vinst->mutex);
644
645 if (!window->user_win && !window->tx_win) {
646 WARN_ON_ONCE(!vinst->rxwin[window->cop]);
647 vinst->rxwin[window->cop] = NULL;
648 }
649
650 WARN_ON_ONCE(vinst->windows[id] != window);
651 vinst->windows[id] = NULL;
652
653 mutex_unlock(&vinst->mutex);
654}
655
656static void init_winctx_for_rxwin(struct vas_window *rxwin,
657 struct vas_rx_win_attr *rxattr,
658 struct vas_winctx *winctx)
659{
660 /*
661 * We first zero (memset()) all fields and only set non-zero fields.
662 * Following fields are 0/false but maybe deserve a comment:
663 *
664 * ->notify_os_intr_reg In powerNV, send intrs to HV
665 * ->notify_disable False for NX windows
666 * ->intr_disable False for Fault Windows
667 * ->xtra_write False for NX windows
668 * ->notify_early NA for NX windows
669 * ->rsvd_txbuf_count NA for Rx windows
670 * ->lpid, ->pid, ->tid NA for Rx windows
671 */
672
673 memset(winctx, 0, sizeof(struct vas_winctx));
674
675 winctx->rx_fifo = rxattr->rx_fifo;
676 winctx->rx_fifo_size = rxattr->rx_fifo_size;
677 winctx->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
678 winctx->pin_win = rxattr->pin_win;
679
680 winctx->nx_win = rxattr->nx_win;
681 winctx->fault_win = rxattr->fault_win;
682 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
683 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
684 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
685 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
686
687 if (winctx->nx_win) {
688 winctx->data_stamp = true;
689 winctx->intr_disable = true;
690 winctx->pin_win = true;
691
692 WARN_ON_ONCE(winctx->fault_win);
693 WARN_ON_ONCE(!winctx->rx_word_mode);
694 WARN_ON_ONCE(!winctx->tx_word_mode);
695 WARN_ON_ONCE(winctx->notify_after_count);
696 } else if (winctx->fault_win) {
697 winctx->notify_disable = true;
698 } else if (winctx->user_win) {
699 /*
700 * Section 1.8.1 Low Latency Core-Core Wake up of
701 * the VAS workbook:
702 *
703 * - disable credit checks ([tr]x_wcred_mode = false)
704 * - disable FIFO writes
705 * - enable ASB_Notify, disable interrupt
706 */
707 winctx->fifo_disable = true;
708 winctx->intr_disable = true;
709 winctx->rx_fifo = NULL;
710 }
711
712 winctx->lnotify_lpid = rxattr->lnotify_lpid;
713 winctx->lnotify_pid = rxattr->lnotify_pid;
714 winctx->lnotify_tid = rxattr->lnotify_tid;
715 winctx->pswid = rxattr->pswid;
716 winctx->dma_type = VAS_DMA_TYPE_INJECT;
717 winctx->tc_mode = rxattr->tc_mode;
718
719 winctx->min_scope = VAS_SCOPE_LOCAL;
720 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
721}
722
723static bool rx_win_args_valid(enum vas_cop_type cop,
724 struct vas_rx_win_attr *attr)
725{
726 dump_rx_win_attr(attr);
727
728 if (cop >= VAS_COP_TYPE_MAX)
729 return false;
730
731 if (cop != VAS_COP_TYPE_FTW &&
732 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
733 return false;
734
735 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
736 return false;
737
738 if (attr->nx_win) {
739 /* cannot be fault or user window if it is nx */
740 if (attr->fault_win || attr->user_win)
741 return false;
742 /*
743 * Section 3.1.4.32: NX Windows must not disable notification,
744 * and must not enable interrupts or early notification.
745 */
746 if (attr->notify_disable || !attr->intr_disable ||
747 attr->notify_early)
748 return false;
749 } else if (attr->fault_win) {
750 /* cannot be both fault and user window */
751 if (attr->user_win)
752 return false;
753
754 /*
755 * Section 3.1.4.32: Fault windows must disable notification
756 * but not interrupts.
757 */
758 if (!attr->notify_disable || attr->intr_disable)
759 return false;
760
761 } else if (attr->user_win) {
762 /*
763 * User receive windows are only for fast-thread-wakeup
764 * (FTW). They don't need a FIFO and must disable interrupts
765 */
766 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
767 return false;
768 } else {
769 /* Rx window must be one of NX or Fault or User window. */
770 return false;
771 }
772
773 return true;
774}
775
776void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
777{
778 memset(rxattr, 0, sizeof(*rxattr));
779
780 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
781 rxattr->pin_win = true;
782 rxattr->nx_win = true;
783 rxattr->fault_win = false;
784 rxattr->intr_disable = true;
785 rxattr->rx_wcred_mode = true;
786 rxattr->tx_wcred_mode = true;
787 rxattr->rx_win_ord_mode = true;
788 rxattr->tx_win_ord_mode = true;
789 } else if (cop == VAS_COP_TYPE_FAULT) {
790 rxattr->pin_win = true;
791 rxattr->fault_win = true;
792 rxattr->notify_disable = true;
793 rxattr->rx_wcred_mode = true;
794 rxattr->tx_wcred_mode = true;
795 rxattr->rx_win_ord_mode = true;
796 rxattr->tx_win_ord_mode = true;
797 } else if (cop == VAS_COP_TYPE_FTW) {
798 rxattr->user_win = true;
799 rxattr->intr_disable = true;
800
801 /*
802 * As noted in the VAS Workbook we disable credit checks.
803 * If we enable credit checks in the future, we must also
804 * implement a mechanism to return the user credits or new
805 * paste operations will fail.
806 */
807 }
808}
809EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
810
811struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
812 struct vas_rx_win_attr *rxattr)
813{
814 struct vas_window *rxwin;
815 struct vas_winctx winctx;
816 struct vas_instance *vinst;
817
818 if (!rx_win_args_valid(cop, rxattr))
819 return ERR_PTR(-EINVAL);
820
821 vinst = find_vas_instance(vasid);
822 if (!vinst) {
823 pr_devel("vasid %d not found!\n", vasid);
824 return ERR_PTR(-EINVAL);
825 }
826 pr_devel("Found instance %d\n", vasid);
827
828 rxwin = vas_window_alloc(vinst);
829 if (IS_ERR(rxwin)) {
830 pr_devel("Unable to allocate memory for Rx window\n");
831 return rxwin;
832 }
833
834 rxwin->tx_win = false;
835 rxwin->nx_win = rxattr->nx_win;
836 rxwin->user_win = rxattr->user_win;
837 rxwin->cop = cop;
838 if (rxattr->user_win)
839 rxwin->pid = task_pid_vnr(current);
840
841 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
842 init_winctx_regs(rxwin, &winctx);
843
844 set_vinst_win(vinst, rxwin);
845
846 return rxwin;
847}
848EXPORT_SYMBOL_GPL(vas_rx_win_open);
849
5239af67
SB		 850void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
851{
852 memset(txattr, 0, sizeof(*txattr));
853
854 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
855 txattr->rej_no_credit = false;
856 txattr->rx_wcred_mode = true;
857 txattr->tx_wcred_mode = true;
858 txattr->rx_win_ord_mode = true;
859 txattr->tx_win_ord_mode = true;
860 } else if (cop == VAS_COP_TYPE_FTW) {
861 txattr->user_win = true;
862 }
863}
864EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
865
866static void init_winctx_for_txwin(struct vas_window *txwin,
867 struct vas_tx_win_attr *txattr,
868 struct vas_winctx *winctx)
869{
870 /*
871 * We first zero all fields and only set non-zero ones. Following
872 * are some fields set to 0/false for the stated reason:
873 *
874 * ->notify_os_intr_reg In powernv, send intrs to HV
875 * ->rsvd_txbuf_count Not supported yet.
876 * ->notify_disable False for NX windows
877 * ->xtra_write False for NX windows
878 * ->notify_early NA for NX windows
879 * ->lnotify_lpid NA for Tx windows
880 * ->lnotify_pid NA for Tx windows
881 * ->lnotify_tid NA for Tx windows
882 * ->tx_win_cred_mode Ignore for now for NX windows
883 * ->rx_win_cred_mode Ignore for now for NX windows
884 */
885 memset(winctx, 0, sizeof(struct vas_winctx));
886
887 winctx->wcreds_max = txattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
888
889 winctx->user_win = txattr->user_win;
890 winctx->nx_win = txwin->rxwin->nx_win;
891 winctx->pin_win = txattr->pin_win;
892
893 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
894 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
895 winctx->rx_word_mode = txattr->rx_win_ord_mode;
896 winctx->tx_word_mode = txattr->tx_win_ord_mode;
897
898 if (winctx->nx_win) {
899 winctx->data_stamp = true;
900 winctx->intr_disable = true;
901 }
902
903 winctx->lpid = txattr->lpid;
904 winctx->pidr = txattr->pidr;
905 winctx->rx_win_id = txwin->rxwin->winid;
906
907 winctx->dma_type = VAS_DMA_TYPE_INJECT;
908 winctx->tc_mode = txattr->tc_mode;
909 winctx->min_scope = VAS_SCOPE_LOCAL;
910 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
911
912 winctx->pswid = 0;
913}
914
915static bool tx_win_args_valid(enum vas_cop_type cop,
916 struct vas_tx_win_attr *attr)
917{
918 if (attr->tc_mode != VAS_THRESH_DISABLED)
919 return false;
920
921 if (cop > VAS_COP_TYPE_MAX)
922 return false;
923
924 if (attr->user_win &&
925 (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count))
926 return false;
927
928 return true;
929}
930
931struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
932 struct vas_tx_win_attr *attr)
933{
934 int rc;
935 struct vas_window *txwin;
936 struct vas_window *rxwin;
937 struct vas_winctx winctx;
938 struct vas_instance *vinst;
939
940 if (!tx_win_args_valid(cop, attr))
941 return ERR_PTR(-EINVAL);
942
943 vinst = find_vas_instance(vasid);
944 if (!vinst) {
945 pr_devel("vasid %d not found!\n", vasid);
946 return ERR_PTR(-EINVAL);
947 }
948
949 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
950 if (IS_ERR(rxwin)) {
951 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
952 return rxwin;
953 }
954
955 txwin = vas_window_alloc(vinst);
956 if (IS_ERR(txwin)) {
957 rc = PTR_ERR(txwin);
958 goto put_rxwin;
959 }
960
961 txwin->tx_win = 1;
962 txwin->rxwin = rxwin;
963 txwin->nx_win = txwin->rxwin->nx_win;
964 txwin->pid = attr->pid;
965 txwin->user_win = attr->user_win;
966
967 init_winctx_for_txwin(txwin, attr, &winctx);
968
969 init_winctx_regs(txwin, &winctx);
970
971 /*
972 * If its a kernel send window, map the window address into the
973 * kernel's address space. For user windows, user must issue an
974 * mmap() to map the window into their address space.
975 *
976 * NOTE: If kernel ever resubmits a user CRB after handling a page
977 * fault, we will need to map this into kernel as well.
978 */
979 if (!txwin->user_win) {
980 txwin->paste_kaddr = map_paste_region(txwin);
981 if (IS_ERR(txwin->paste_kaddr)) {
982 rc = PTR_ERR(txwin->paste_kaddr);
983 goto free_window;
984 }
985 }
986
987 set_vinst_win(vinst, txwin);
988
989 return txwin;
990
991free_window:
992 vas_window_free(txwin);
993
994put_rxwin:
995 put_rx_win(rxwin);
996 return ERR_PTR(rc);
997
998}
999EXPORT_SYMBOL_GPL(vas_tx_win_open);
1000
2392c8c8
SB		 1001int vas_copy_crb(void *crb, int offset)
1002{
1003 return vas_copy(crb, offset);
1004}
1005EXPORT_SYMBOL_GPL(vas_copy_crb);
1006
1007#define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
1008int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1009{
1010 int rc;
1011 void *addr;
1012 uint64_t val;
1013
1014 /*
1015 * Only NX windows are supported for now and hardware assumes
1016 * report-enable flag is set for NX windows. Ensure software
1017 * complies too.
1018 */
1019 WARN_ON_ONCE(txwin->nx_win && !re);
1020
1021 addr = txwin->paste_kaddr;
1022 if (re) {
1023 /*
1024 * Set the REPORT_ENABLE bit (equivalent to writing
1025 * to 1K offset of the paste address)
1026 */
1027 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1028 addr += val;
1029 }
1030
1031 /*
1032 * Map the raw CR value from vas_paste() to an error code (there
1033 * is just pass or fail for now though).
1034 */
1035 rc = vas_paste(addr, offset);
1036 if (rc == 2)
1037 rc = 0;
1038 else
1039 rc = -EINVAL;
1040
1041 print_fifo_msg_count(txwin);
1042
1043 return rc;
1044}
1045EXPORT_SYMBOL_GPL(vas_paste_crb);
1046
98271d41
SB		 1047static void poll_window_busy_state(struct vas_window *window)
1048{
1049 int busy;
1050 u64 val;
1051
1052retry:
1053 /*
1054 * Poll Window Busy flag
1055 */
1056 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1057 busy = GET_FIELD(VAS_WIN_BUSY, val);
1058 if (busy) {
1059 val = 0;
1060 set_current_state(TASK_UNINTERRUPTIBLE);
1061 schedule_timeout(HZ);
1062 goto retry;
1063 }
1064}
1065
1066static void poll_window_castout(struct vas_window *window)
1067{
1068 int cached;
1069 u64 val;
1070
1071 /* Cast window context out of the cache */
1072retry:
1073 val = read_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL));
1074 cached = GET_FIELD(VAS_WIN_CACHE_STATUS, val);
1075 if (cached) {
1076 val = 0ULL;
1077 val = SET_FIELD(VAS_CASTOUT_REQ, val, 1);
1078 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 0);
1079 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
1080
1081 set_current_state(TASK_UNINTERRUPTIBLE);
1082 schedule_timeout(HZ);
1083 goto retry;
1084 }
1085}
1086
1087/*
1088 * Close a window.
1089 *
1090 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1091 * - Disable new paste operations (unmap paste address)
1092 * - Poll for the "Window Busy" bit to be cleared
1093 * - Clear the Open/Enable bit for the Window.
1094 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1095 * - Unpin and cast window context out of cache
1096 *
1097 * Besides the hardware, kernel has some bookkeeping of course.
1098 */
4dea2d1a
SB		 1099int vas_win_close(struct vas_window *window)
1100{
98271d41
SB		 1101 u64 val;
1102
1103 if (!window)
1104 return 0;
1105
1106 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1107 pr_devel("Attempting to close an active Rx window!\n");
1108 WARN_ON_ONCE(1);
1109 return -EBUSY;
1110 }
1111
1112 unmap_paste_region(window);
1113
1114 clear_vinst_win(window);
1115
1116 poll_window_busy_state(window);
1117
1118 /* Unpin window from cache and close it */
1119 val = read_hvwc_reg(window, VREG(WINCTL));
1120 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1121 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1122 write_hvwc_reg(window, VREG(WINCTL), val);
1123
1124 poll_window_castout(window);
1125
1126 /* if send window, drop reference to matching receive window */
1127 if (window->tx_win)
1128 put_rx_win(window->rxwin);
1129
1130 vas_window_free(window);
1131
1132 return 0;
4dea2d1a 1133}
98271d41 1134EXPORT_SYMBOL_GPL(vas_win_close);
Linux 6.x block layer and io_uring tree(s)