Commit | Line | Data |
---|---|---|
96768914 SB |
1 | /* |
2 | * Copyright 2016-17 IBM Corp. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #ifndef _VAS_H | |
11 | #define _VAS_H | |
12 | #include <linux/atomic.h> | |
13 | #include <linux/idr.h> | |
14 | #include <asm/vas.h> | |
b25b33ac | 15 | #include <linux/io.h> |
ece4e512 SB |
16 | #include <linux/dcache.h> |
17 | #include <linux/mutex.h> | |
5c35a02c | 18 | #include <linux/stringify.h> |
96768914 SB |
19 | |
20 | /* | |
21 | * Overview of Virtual Accelerator Switchboard (VAS). | |
22 | * | |
23 | * VAS is a hardware "switchboard" that allows senders and receivers to | |
24 | * exchange messages with _minimal_ kernel involvment. The receivers are | |
25 | * typically NX coprocessor engines that perform compression or encryption | |
26 | * in hardware, but receivers can also be other software threads. | |
27 | * | |
28 | * Senders are user/kernel threads that submit compression/encryption or | |
29 | * other requests to the receivers. Senders must format their messages as | |
30 | * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and | |
31 | * "paste" instructions which were introduced in Power9. | |
32 | * | |
33 | * A Power node can have (upto?) 8 Power chips. There is one instance of | |
34 | * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports, | |
35 | * Senders and receivers must each connect to a separate window before they | |
36 | * can exchange messages through the switchboard. | |
37 | * | |
38 | * Each window is described by two types of window contexts: | |
39 | * | |
40 | * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes | |
41 | * | |
42 | * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes. | |
43 | * | |
44 | * A window context can be viewed as a set of 64-bit registers. The settings | |
45 | * in these registers configure/control/determine the behavior of the VAS | |
46 | * hardware when messages are sent/received through the window. The registers | |
47 | * in the HVWC are configured by the kernel while the registers in the UWC can | |
48 | * be configured by the kernel or by the user space application that is using | |
49 | * the window. | |
50 | * | |
51 | * The HVWCs for all windows on a specific instance of VAS are in a contiguous | |
52 | * range of hardware addresses or Base address region (BAR) referred to as the | |
53 | * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance | |
54 | * are referred to as the UWC BAR for the instance. | |
55 | * | |
56 | * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet | |
57 | * and available to the kernel in the VAS node's "reg" property in the device | |
58 | * tree: | |
59 | * | |
60 | * /proc/device-tree/vasm@.../reg | |
61 | * | |
62 | * (see vas_probe() for details on the reg property). | |
63 | * | |
64 | * The kernel maps the HVWC and UWC BAR regions into the kernel address | |
65 | * space (hvwc_map and uwc_map). The kernel can then access the window | |
66 | * contexts of a specific window using: | |
67 | * | |
68 | * hvwc = hvwc_map + winid * VAS_HVWC_SIZE. | |
69 | * uwc = uwc_map + winid * VAS_UWC_SIZE. | |
70 | * | |
71 | * where winid is the window index (0..64K). | |
72 | * | |
73 | * As mentioned, a window context is used to "configure" a window. Besides | |
74 | * this configuration address, each _send_ window also has a unique hardware | |
75 | * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()). | |
76 | * | |
77 | * The hardware paste address for a window is computed using the "paste | |
78 | * base address" and "paste win id shift" reg properties in the VAS device | |
79 | * tree node using: | |
80 | * | |
81 | * paste_addr = paste_base + ((winid << paste_win_id_shift)) | |
82 | * | |
83 | * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift). | |
84 | * | |
85 | * The kernel maps this hardware address into the sender's address space | |
86 | * after which they can use the 'paste' instruction (new in Power9) to | |
87 | * send a message (submit a request aka CRB) to the coprocessor. | |
88 | * | |
89 | * NOTE: In the initial version, senders can only in-kernel drivers/threads. | |
90 | * Support for user space threads will be added in follow-on patches. | |
91 | * | |
92 | * TODO: Do we need to map the UWC into user address space so they can return | |
93 | * credits? Its NA for NX but may be needed for other receive windows. | |
94 | * | |
95 | */ | |
96 | ||
97 | #define VAS_WINDOWS_PER_CHIP (64 << 10) | |
98 | ||
99 | /* | |
100 | * Hypervisor and OS/USer Window Context sizes | |
101 | */ | |
102 | #define VAS_HVWC_SIZE 512 | |
103 | #define VAS_UWC_SIZE PAGE_SIZE | |
104 | ||
105 | /* | |
106 | * Initial per-process credits. | |
107 | * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED) | |
108 | * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED) | |
109 | * | |
110 | * TODO: Needs tuning for per-process credits | |
111 | */ | |
51b53712 SB |
112 | #define VAS_RX_WCREDS_MAX ((64 << 10) - 1) |
113 | #define VAS_TX_WCREDS_MAX ((4 << 10) - 1) | |
96768914 SB |
114 | #define VAS_WCREDS_DEFAULT (1 << 10) |
115 | ||
116 | /* | |
117 | * VAS Window Context Register Offsets and bitmasks. | |
118 | * See Section 3.1.4 of VAS Work book | |
119 | */ | |
120 | #define VAS_LPID_OFFSET 0x010 | |
121 | #define VAS_LPID PPC_BITMASK(0, 11) | |
122 | ||
123 | #define VAS_PID_OFFSET 0x018 | |
124 | #define VAS_PID_ID PPC_BITMASK(0, 19) | |
125 | ||
126 | #define VAS_XLATE_MSR_OFFSET 0x020 | |
127 | #define VAS_XLATE_MSR_DR PPC_BIT(0) | |
128 | #define VAS_XLATE_MSR_TA PPC_BIT(1) | |
129 | #define VAS_XLATE_MSR_PR PPC_BIT(2) | |
130 | #define VAS_XLATE_MSR_US PPC_BIT(3) | |
131 | #define VAS_XLATE_MSR_HV PPC_BIT(4) | |
132 | #define VAS_XLATE_MSR_SF PPC_BIT(5) | |
133 | ||
134 | #define VAS_XLATE_LPCR_OFFSET 0x028 | |
135 | #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2) | |
136 | #define VAS_XLATE_LPCR_ISL PPC_BIT(3) | |
137 | #define VAS_XLATE_LPCR_TC PPC_BIT(4) | |
138 | #define VAS_XLATE_LPCR_SC PPC_BIT(5) | |
139 | ||
140 | #define VAS_XLATE_CTL_OFFSET 0x030 | |
141 | #define VAS_XLATE_MODE PPC_BITMASK(0, 1) | |
142 | ||
143 | #define VAS_AMR_OFFSET 0x040 | |
144 | #define VAS_AMR PPC_BITMASK(0, 63) | |
145 | ||
146 | #define VAS_SEIDR_OFFSET 0x048 | |
147 | #define VAS_SEIDR PPC_BITMASK(0, 63) | |
148 | ||
149 | #define VAS_FAULT_TX_WIN_OFFSET 0x050 | |
150 | #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63) | |
151 | ||
152 | #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060 | |
153 | #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63) | |
154 | ||
155 | #define VAS_HV_INTR_SRC_RA_OFFSET 0x070 | |
156 | #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63) | |
157 | ||
158 | #define VAS_PSWID_OFFSET 0x078 | |
159 | #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31) | |
160 | ||
161 | #define VAS_SPARE1_OFFSET 0x080 | |
162 | #define VAS_SPARE2_OFFSET 0x088 | |
163 | #define VAS_SPARE3_OFFSET 0x090 | |
164 | #define VAS_SPARE4_OFFSET 0x130 | |
165 | #define VAS_SPARE5_OFFSET 0x160 | |
166 | #define VAS_SPARE6_OFFSET 0x188 | |
167 | ||
168 | #define VAS_LFIFO_BAR_OFFSET 0x0A0 | |
169 | #define VAS_LFIFO_BAR PPC_BITMASK(8, 53) | |
170 | #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56) | |
171 | ||
172 | #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8 | |
173 | #define VAS_LDATA_STAMP PPC_BITMASK(0, 1) | |
174 | #define VAS_XTRA_WRITE PPC_BIT(2) | |
175 | ||
176 | #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0 | |
177 | #define VAS_LDMA_TYPE PPC_BITMASK(0, 1) | |
178 | #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2) | |
179 | ||
180 | #define VAS_LRFIFO_PUSH_OFFSET 0x0B8 | |
181 | #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15) | |
182 | ||
183 | #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0 | |
184 | #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7) | |
185 | ||
186 | #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8 | |
187 | #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7) | |
188 | ||
189 | #define VAS_LRX_WCRED_OFFSET 0x0E0 | |
190 | #define VAS_LRX_WCRED PPC_BITMASK(0, 15) | |
191 | ||
192 | #define VAS_LRX_WCRED_ADDER_OFFSET 0x190 | |
193 | #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15) | |
194 | ||
195 | #define VAS_TX_WCRED_OFFSET 0x0F0 | |
196 | #define VAS_TX_WCRED PPC_BITMASK(4, 15) | |
197 | ||
198 | #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0 | |
199 | #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15) | |
200 | ||
201 | #define VAS_LFIFO_SIZE_OFFSET 0x100 | |
202 | #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3) | |
203 | ||
204 | #define VAS_WINCTL_OFFSET 0x108 | |
205 | #define VAS_WINCTL_OPEN PPC_BIT(0) | |
206 | #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1) | |
207 | #define VAS_WINCTL_PIN PPC_BIT(2) | |
208 | #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3) | |
209 | #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4) | |
210 | #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5) | |
211 | #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6) | |
212 | #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7) | |
213 | #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9) | |
214 | #define VAS_WINCTL_FAULT_WIN PPC_BIT(10) | |
215 | #define VAS_WINCTL_NX_WIN PPC_BIT(11) | |
216 | ||
217 | #define VAS_WIN_STATUS_OFFSET 0x110 | |
218 | #define VAS_WIN_BUSY PPC_BIT(1) | |
219 | ||
220 | #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118 | |
221 | #define VAS_CASTOUT_REQ PPC_BIT(0) | |
222 | #define VAS_PUSH_TO_MEM PPC_BIT(1) | |
223 | #define VAS_WIN_CACHE_STATUS PPC_BIT(4) | |
224 | ||
225 | #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120 | |
226 | #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63) | |
227 | ||
228 | #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128 | |
229 | #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15) | |
230 | ||
231 | /* | |
232 | * Local Notification Control Register controls what happens in _response_ | |
233 | * to a paste command and hence applies only to receive windows. | |
234 | */ | |
235 | #define VAS_LNOTIFY_CTL_OFFSET 0x138 | |
236 | #define VAS_NOTIFY_DISABLE PPC_BIT(0) | |
237 | #define VAS_INTR_DISABLE PPC_BIT(1) | |
238 | #define VAS_NOTIFY_EARLY PPC_BIT(2) | |
239 | #define VAS_NOTIFY_OSU_INTR PPC_BIT(3) | |
240 | ||
241 | #define VAS_LNOTIFY_PID_OFFSET 0x140 | |
242 | #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19) | |
243 | ||
244 | #define VAS_LNOTIFY_LPID_OFFSET 0x148 | |
245 | #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11) | |
246 | ||
247 | #define VAS_LNOTIFY_TID_OFFSET 0x150 | |
248 | #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15) | |
249 | ||
250 | #define VAS_LNOTIFY_SCOPE_OFFSET 0x158 | |
251 | #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1) | |
252 | #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3) | |
253 | ||
254 | #define VAS_NX_UTIL_OFFSET 0x1B0 | |
255 | #define VAS_NX_UTIL PPC_BITMASK(0, 63) | |
256 | ||
257 | /* SE: Side effects */ | |
258 | #define VAS_NX_UTIL_SE_OFFSET 0x1B8 | |
259 | #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63) | |
260 | ||
261 | #define VAS_NX_UTIL_ADDER_OFFSET 0x180 | |
262 | #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63) | |
263 | ||
0a2c2c24 SB |
264 | /* |
265 | * VREG(x): | |
266 | * Expand a register's short name (eg: LPID) into two parameters: | |
267 | * - the register's short name in string form ("LPID"), and | |
268 | * - the name of the macro (eg: VAS_LPID_OFFSET), defining the | |
269 | * register's offset in the window context | |
270 | */ | |
271 | #define VREG_SFX(n, s) __stringify(n), VAS_##n##s | |
272 | #define VREG(r) VREG_SFX(r, _OFFSET) | |
273 | ||
96768914 SB |
274 | /* |
275 | * Local Notify Scope Control Register. (Receive windows only). | |
276 | */ | |
277 | enum vas_notify_scope { | |
278 | VAS_SCOPE_LOCAL, | |
279 | VAS_SCOPE_GROUP, | |
280 | VAS_SCOPE_VECTORED_GROUP, | |
281 | VAS_SCOPE_UNUSED, | |
282 | }; | |
283 | ||
284 | /* | |
285 | * Local DMA Cache Control Register (Receive windows only). | |
286 | */ | |
287 | enum vas_dma_type { | |
288 | VAS_DMA_TYPE_INJECT, | |
289 | VAS_DMA_TYPE_WRITE, | |
290 | }; | |
291 | ||
292 | /* | |
293 | * Local Notify Scope Control Register. (Receive windows only). | |
294 | * Not applicable to NX receive windows. | |
295 | */ | |
296 | enum vas_notify_after_count { | |
297 | VAS_NOTIFY_AFTER_256 = 0, | |
298 | VAS_NOTIFY_NONE, | |
299 | VAS_NOTIFY_AFTER_2 | |
300 | }; | |
301 | ||
302 | /* | |
303 | * One per instance of VAS. Each instance will have a separate set of | |
304 | * receive windows, one per coprocessor type. | |
62c4eda4 SB |
305 | * |
306 | * See also function header of set_vinst_win() for details on ->windows[] | |
307 | * and ->rxwin[] tables. | |
96768914 SB |
308 | */ |
309 | struct vas_instance { | |
310 | int vas_id; | |
311 | struct ida ida; | |
312 | struct list_head node; | |
313 | struct platform_device *pdev; | |
314 | ||
315 | u64 hvwc_bar_start; | |
316 | u64 uwc_bar_start; | |
317 | u64 paste_base_addr; | |
318 | u64 paste_win_id_shift; | |
319 | ||
320 | struct mutex mutex; | |
321 | struct vas_window *rxwin[VAS_COP_TYPE_MAX]; | |
322 | struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; | |
ece4e512 SB |
323 | |
324 | char *dbgname; | |
325 | struct dentry *dbgdir; | |
96768914 SB |
326 | }; |
327 | ||
328 | /* | |
329 | * In-kernel state a VAS window. One per window. | |
330 | */ | |
331 | struct vas_window { | |
332 | /* Fields common to send and receive windows */ | |
333 | struct vas_instance *vinst; | |
334 | int winid; | |
335 | bool tx_win; /* True if send window */ | |
336 | bool nx_win; /* True if NX window */ | |
337 | bool user_win; /* True if user space window */ | |
338 | void *hvwc_map; /* HV window context */ | |
339 | void *uwc_map; /* OS/User window context */ | |
340 | pid_t pid; /* Linux process id of owner */ | |
62f659e0 | 341 | int wcreds_max; /* Window credits */ |
96768914 | 342 | |
ece4e512 SB |
343 | char *dbgname; |
344 | struct dentry *dbgdir; | |
345 | ||
96768914 SB |
346 | /* Fields applicable only to send windows */ |
347 | void *paste_kaddr; | |
348 | char *paste_addr_name; | |
349 | struct vas_window *rxwin; | |
350 | ||
351 | /* Feilds applicable only to receive windows */ | |
352 | enum vas_cop_type cop; | |
353 | atomic_t num_txwins; | |
354 | }; | |
355 | ||
356 | /* | |
357 | * Container for the hardware state of a window. One per-window. | |
358 | * | |
359 | * A VAS Window context is a 512-byte area in the hardware that contains | |
360 | * a set of 64-bit registers. Individual bit-fields in these registers | |
361 | * determine the configuration/operation of the hardware. struct vas_winctx | |
362 | * is a container for the register fields in the window context. | |
363 | */ | |
364 | struct vas_winctx { | |
365 | void *rx_fifo; | |
366 | int rx_fifo_size; | |
367 | int wcreds_max; | |
368 | int rsvd_txbuf_count; | |
369 | ||
370 | bool user_win; | |
371 | bool nx_win; | |
372 | bool fault_win; | |
373 | bool rsvd_txbuf_enable; | |
374 | bool pin_win; | |
375 | bool rej_no_credit; | |
376 | bool tx_wcred_mode; | |
377 | bool rx_wcred_mode; | |
378 | bool tx_word_mode; | |
379 | bool rx_word_mode; | |
380 | bool data_stamp; | |
381 | bool xtra_write; | |
382 | bool notify_disable; | |
383 | bool intr_disable; | |
384 | bool fifo_disable; | |
385 | bool notify_early; | |
386 | bool notify_os_intr_reg; | |
387 | ||
388 | int lpid; | |
389 | int pidr; /* value from SPRN_PID, not linux pid */ | |
390 | int lnotify_lpid; | |
391 | int lnotify_pid; | |
392 | int lnotify_tid; | |
393 | u32 pswid; | |
394 | int rx_win_id; | |
395 | int fault_win_id; | |
396 | int tc_mode; | |
397 | ||
398 | u64 irq_port; | |
399 | ||
400 | enum vas_dma_type dma_type; | |
401 | enum vas_notify_scope min_scope; | |
402 | enum vas_notify_scope max_scope; | |
403 | enum vas_notify_after_count notify_after_count; | |
404 | }; | |
405 | ||
ece4e512 SB |
406 | extern struct mutex vas_mutex; |
407 | ||
4dea2d1a | 408 | extern struct vas_instance *find_vas_instance(int vasid); |
ece4e512 SB |
409 | extern void vas_init_dbgdir(void); |
410 | extern void vas_instance_init_dbgdir(struct vas_instance *vinst); | |
411 | extern void vas_window_init_dbgdir(struct vas_window *win); | |
412 | extern void vas_window_free_dbgdir(struct vas_window *win); | |
4dea2d1a | 413 | |
b25b33ac SB |
414 | static inline void vas_log_write(struct vas_window *win, char *name, |
415 | void *regptr, u64 val) | |
416 | { | |
417 | if (val) | |
0a2c2c24 | 418 | pr_debug("%swin #%d: %s reg %p, val 0x%016llx\n", |
b25b33ac SB |
419 | win->tx_win ? "Tx" : "Rx", win->winid, name, |
420 | regptr, val); | |
421 | } | |
422 | ||
b25b33ac SB |
423 | static inline void write_uwc_reg(struct vas_window *win, char *name, |
424 | s32 reg, u64 val) | |
425 | { | |
426 | void *regptr; | |
427 | ||
428 | regptr = win->uwc_map + reg; | |
429 | vas_log_write(win, name, regptr, val); | |
430 | ||
431 | out_be64(regptr, val); | |
432 | } | |
433 | ||
434 | static inline void write_hvwc_reg(struct vas_window *win, char *name, | |
435 | s32 reg, u64 val) | |
436 | { | |
437 | void *regptr; | |
438 | ||
439 | regptr = win->hvwc_map + reg; | |
440 | vas_log_write(win, name, regptr, val); | |
441 | ||
442 | out_be64(regptr, val); | |
443 | } | |
444 | ||
445 | static inline u64 read_hvwc_reg(struct vas_window *win, | |
446 | char *name __maybe_unused, s32 reg) | |
447 | { | |
448 | return in_be64(win->hvwc_map+reg); | |
449 | } | |
450 | ||
61f3cca8 SB |
451 | static inline void decode_pswid(u32 pswid, int *vasid, int *winid) |
452 | { | |
453 | if (vasid) | |
454 | *vasid = pswid >> (31 - 7) & 0xFF; | |
455 | ||
456 | if (winid) | |
457 | *winid = pswid & 0xFFFF; | |
458 | } | |
96768914 | 459 | #endif /* _VAS_H */ |