Commit | Line | Data |
---|---|---|
96768914 SB |
1 | /* |
2 | * Copyright 2016-17 IBM Corp. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; either version | |
7 | * 2 of the License, or (at your option) any later version. | |
8 | */ | |
9 | ||
10 | #ifndef _VAS_H | |
11 | #define _VAS_H | |
12 | #include <linux/atomic.h> | |
13 | #include <linux/idr.h> | |
14 | #include <asm/vas.h> | |
b25b33ac | 15 | #include <linux/io.h> |
96768914 SB |
16 | |
17 | /* | |
18 | * Overview of Virtual Accelerator Switchboard (VAS). | |
19 | * | |
20 | * VAS is a hardware "switchboard" that allows senders and receivers to | |
21 | * exchange messages with _minimal_ kernel involvment. The receivers are | |
22 | * typically NX coprocessor engines that perform compression or encryption | |
23 | * in hardware, but receivers can also be other software threads. | |
24 | * | |
25 | * Senders are user/kernel threads that submit compression/encryption or | |
26 | * other requests to the receivers. Senders must format their messages as | |
27 | * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and | |
28 | * "paste" instructions which were introduced in Power9. | |
29 | * | |
30 | * A Power node can have (upto?) 8 Power chips. There is one instance of | |
31 | * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports, | |
32 | * Senders and receivers must each connect to a separate window before they | |
33 | * can exchange messages through the switchboard. | |
34 | * | |
35 | * Each window is described by two types of window contexts: | |
36 | * | |
37 | * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes | |
38 | * | |
39 | * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes. | |
40 | * | |
41 | * A window context can be viewed as a set of 64-bit registers. The settings | |
42 | * in these registers configure/control/determine the behavior of the VAS | |
43 | * hardware when messages are sent/received through the window. The registers | |
44 | * in the HVWC are configured by the kernel while the registers in the UWC can | |
45 | * be configured by the kernel or by the user space application that is using | |
46 | * the window. | |
47 | * | |
48 | * The HVWCs for all windows on a specific instance of VAS are in a contiguous | |
49 | * range of hardware addresses or Base address region (BAR) referred to as the | |
50 | * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance | |
51 | * are referred to as the UWC BAR for the instance. | |
52 | * | |
53 | * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet | |
54 | * and available to the kernel in the VAS node's "reg" property in the device | |
55 | * tree: | |
56 | * | |
57 | * /proc/device-tree/vasm@.../reg | |
58 | * | |
59 | * (see vas_probe() for details on the reg property). | |
60 | * | |
61 | * The kernel maps the HVWC and UWC BAR regions into the kernel address | |
62 | * space (hvwc_map and uwc_map). The kernel can then access the window | |
63 | * contexts of a specific window using: | |
64 | * | |
65 | * hvwc = hvwc_map + winid * VAS_HVWC_SIZE. | |
66 | * uwc = uwc_map + winid * VAS_UWC_SIZE. | |
67 | * | |
68 | * where winid is the window index (0..64K). | |
69 | * | |
70 | * As mentioned, a window context is used to "configure" a window. Besides | |
71 | * this configuration address, each _send_ window also has a unique hardware | |
72 | * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()). | |
73 | * | |
74 | * The hardware paste address for a window is computed using the "paste | |
75 | * base address" and "paste win id shift" reg properties in the VAS device | |
76 | * tree node using: | |
77 | * | |
78 | * paste_addr = paste_base + ((winid << paste_win_id_shift)) | |
79 | * | |
80 | * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift). | |
81 | * | |
82 | * The kernel maps this hardware address into the sender's address space | |
83 | * after which they can use the 'paste' instruction (new in Power9) to | |
84 | * send a message (submit a request aka CRB) to the coprocessor. | |
85 | * | |
86 | * NOTE: In the initial version, senders can only in-kernel drivers/threads. | |
87 | * Support for user space threads will be added in follow-on patches. | |
88 | * | |
89 | * TODO: Do we need to map the UWC into user address space so they can return | |
90 | * credits? Its NA for NX but may be needed for other receive windows. | |
91 | * | |
92 | */ | |
93 | ||
94 | #define VAS_WINDOWS_PER_CHIP (64 << 10) | |
95 | ||
96 | /* | |
97 | * Hypervisor and OS/USer Window Context sizes | |
98 | */ | |
99 | #define VAS_HVWC_SIZE 512 | |
100 | #define VAS_UWC_SIZE PAGE_SIZE | |
101 | ||
102 | /* | |
103 | * Initial per-process credits. | |
104 | * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED) | |
105 | * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED) | |
106 | * | |
107 | * TODO: Needs tuning for per-process credits | |
108 | */ | |
109 | #define VAS_WCREDS_MIN 16 | |
110 | #define VAS_WCREDS_MAX ((64 << 10) - 1) | |
111 | #define VAS_WCREDS_DEFAULT (1 << 10) | |
112 | ||
113 | /* | |
114 | * VAS Window Context Register Offsets and bitmasks. | |
115 | * See Section 3.1.4 of VAS Work book | |
116 | */ | |
117 | #define VAS_LPID_OFFSET 0x010 | |
118 | #define VAS_LPID PPC_BITMASK(0, 11) | |
119 | ||
120 | #define VAS_PID_OFFSET 0x018 | |
121 | #define VAS_PID_ID PPC_BITMASK(0, 19) | |
122 | ||
123 | #define VAS_XLATE_MSR_OFFSET 0x020 | |
124 | #define VAS_XLATE_MSR_DR PPC_BIT(0) | |
125 | #define VAS_XLATE_MSR_TA PPC_BIT(1) | |
126 | #define VAS_XLATE_MSR_PR PPC_BIT(2) | |
127 | #define VAS_XLATE_MSR_US PPC_BIT(3) | |
128 | #define VAS_XLATE_MSR_HV PPC_BIT(4) | |
129 | #define VAS_XLATE_MSR_SF PPC_BIT(5) | |
130 | ||
131 | #define VAS_XLATE_LPCR_OFFSET 0x028 | |
132 | #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2) | |
133 | #define VAS_XLATE_LPCR_ISL PPC_BIT(3) | |
134 | #define VAS_XLATE_LPCR_TC PPC_BIT(4) | |
135 | #define VAS_XLATE_LPCR_SC PPC_BIT(5) | |
136 | ||
137 | #define VAS_XLATE_CTL_OFFSET 0x030 | |
138 | #define VAS_XLATE_MODE PPC_BITMASK(0, 1) | |
139 | ||
140 | #define VAS_AMR_OFFSET 0x040 | |
141 | #define VAS_AMR PPC_BITMASK(0, 63) | |
142 | ||
143 | #define VAS_SEIDR_OFFSET 0x048 | |
144 | #define VAS_SEIDR PPC_BITMASK(0, 63) | |
145 | ||
146 | #define VAS_FAULT_TX_WIN_OFFSET 0x050 | |
147 | #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63) | |
148 | ||
149 | #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060 | |
150 | #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63) | |
151 | ||
152 | #define VAS_HV_INTR_SRC_RA_OFFSET 0x070 | |
153 | #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63) | |
154 | ||
155 | #define VAS_PSWID_OFFSET 0x078 | |
156 | #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31) | |
157 | ||
158 | #define VAS_SPARE1_OFFSET 0x080 | |
159 | #define VAS_SPARE2_OFFSET 0x088 | |
160 | #define VAS_SPARE3_OFFSET 0x090 | |
161 | #define VAS_SPARE4_OFFSET 0x130 | |
162 | #define VAS_SPARE5_OFFSET 0x160 | |
163 | #define VAS_SPARE6_OFFSET 0x188 | |
164 | ||
165 | #define VAS_LFIFO_BAR_OFFSET 0x0A0 | |
166 | #define VAS_LFIFO_BAR PPC_BITMASK(8, 53) | |
167 | #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56) | |
168 | ||
169 | #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8 | |
170 | #define VAS_LDATA_STAMP PPC_BITMASK(0, 1) | |
171 | #define VAS_XTRA_WRITE PPC_BIT(2) | |
172 | ||
173 | #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0 | |
174 | #define VAS_LDMA_TYPE PPC_BITMASK(0, 1) | |
175 | #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2) | |
176 | ||
177 | #define VAS_LRFIFO_PUSH_OFFSET 0x0B8 | |
178 | #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15) | |
179 | ||
180 | #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0 | |
181 | #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7) | |
182 | ||
183 | #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8 | |
184 | #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7) | |
185 | ||
186 | #define VAS_LRX_WCRED_OFFSET 0x0E0 | |
187 | #define VAS_LRX_WCRED PPC_BITMASK(0, 15) | |
188 | ||
189 | #define VAS_LRX_WCRED_ADDER_OFFSET 0x190 | |
190 | #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15) | |
191 | ||
192 | #define VAS_TX_WCRED_OFFSET 0x0F0 | |
193 | #define VAS_TX_WCRED PPC_BITMASK(4, 15) | |
194 | ||
195 | #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0 | |
196 | #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15) | |
197 | ||
198 | #define VAS_LFIFO_SIZE_OFFSET 0x100 | |
199 | #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3) | |
200 | ||
201 | #define VAS_WINCTL_OFFSET 0x108 | |
202 | #define VAS_WINCTL_OPEN PPC_BIT(0) | |
203 | #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1) | |
204 | #define VAS_WINCTL_PIN PPC_BIT(2) | |
205 | #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3) | |
206 | #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4) | |
207 | #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5) | |
208 | #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6) | |
209 | #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7) | |
210 | #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9) | |
211 | #define VAS_WINCTL_FAULT_WIN PPC_BIT(10) | |
212 | #define VAS_WINCTL_NX_WIN PPC_BIT(11) | |
213 | ||
214 | #define VAS_WIN_STATUS_OFFSET 0x110 | |
215 | #define VAS_WIN_BUSY PPC_BIT(1) | |
216 | ||
217 | #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118 | |
218 | #define VAS_CASTOUT_REQ PPC_BIT(0) | |
219 | #define VAS_PUSH_TO_MEM PPC_BIT(1) | |
220 | #define VAS_WIN_CACHE_STATUS PPC_BIT(4) | |
221 | ||
222 | #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120 | |
223 | #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63) | |
224 | ||
225 | #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128 | |
226 | #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15) | |
227 | ||
228 | /* | |
229 | * Local Notification Control Register controls what happens in _response_ | |
230 | * to a paste command and hence applies only to receive windows. | |
231 | */ | |
232 | #define VAS_LNOTIFY_CTL_OFFSET 0x138 | |
233 | #define VAS_NOTIFY_DISABLE PPC_BIT(0) | |
234 | #define VAS_INTR_DISABLE PPC_BIT(1) | |
235 | #define VAS_NOTIFY_EARLY PPC_BIT(2) | |
236 | #define VAS_NOTIFY_OSU_INTR PPC_BIT(3) | |
237 | ||
238 | #define VAS_LNOTIFY_PID_OFFSET 0x140 | |
239 | #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19) | |
240 | ||
241 | #define VAS_LNOTIFY_LPID_OFFSET 0x148 | |
242 | #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11) | |
243 | ||
244 | #define VAS_LNOTIFY_TID_OFFSET 0x150 | |
245 | #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15) | |
246 | ||
247 | #define VAS_LNOTIFY_SCOPE_OFFSET 0x158 | |
248 | #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1) | |
249 | #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3) | |
250 | ||
251 | #define VAS_NX_UTIL_OFFSET 0x1B0 | |
252 | #define VAS_NX_UTIL PPC_BITMASK(0, 63) | |
253 | ||
254 | /* SE: Side effects */ | |
255 | #define VAS_NX_UTIL_SE_OFFSET 0x1B8 | |
256 | #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63) | |
257 | ||
258 | #define VAS_NX_UTIL_ADDER_OFFSET 0x180 | |
259 | #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63) | |
260 | ||
261 | /* | |
262 | * Local Notify Scope Control Register. (Receive windows only). | |
263 | */ | |
264 | enum vas_notify_scope { | |
265 | VAS_SCOPE_LOCAL, | |
266 | VAS_SCOPE_GROUP, | |
267 | VAS_SCOPE_VECTORED_GROUP, | |
268 | VAS_SCOPE_UNUSED, | |
269 | }; | |
270 | ||
271 | /* | |
272 | * Local DMA Cache Control Register (Receive windows only). | |
273 | */ | |
274 | enum vas_dma_type { | |
275 | VAS_DMA_TYPE_INJECT, | |
276 | VAS_DMA_TYPE_WRITE, | |
277 | }; | |
278 | ||
279 | /* | |
280 | * Local Notify Scope Control Register. (Receive windows only). | |
281 | * Not applicable to NX receive windows. | |
282 | */ | |
283 | enum vas_notify_after_count { | |
284 | VAS_NOTIFY_AFTER_256 = 0, | |
285 | VAS_NOTIFY_NONE, | |
286 | VAS_NOTIFY_AFTER_2 | |
287 | }; | |
288 | ||
289 | /* | |
290 | * One per instance of VAS. Each instance will have a separate set of | |
291 | * receive windows, one per coprocessor type. | |
62c4eda4 SB |
292 | * |
293 | * See also function header of set_vinst_win() for details on ->windows[] | |
294 | * and ->rxwin[] tables. | |
96768914 SB |
295 | */ |
296 | struct vas_instance { | |
297 | int vas_id; | |
298 | struct ida ida; | |
299 | struct list_head node; | |
300 | struct platform_device *pdev; | |
301 | ||
302 | u64 hvwc_bar_start; | |
303 | u64 uwc_bar_start; | |
304 | u64 paste_base_addr; | |
305 | u64 paste_win_id_shift; | |
306 | ||
307 | struct mutex mutex; | |
308 | struct vas_window *rxwin[VAS_COP_TYPE_MAX]; | |
309 | struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; | |
310 | }; | |
311 | ||
312 | /* | |
313 | * In-kernel state a VAS window. One per window. | |
314 | */ | |
315 | struct vas_window { | |
316 | /* Fields common to send and receive windows */ | |
317 | struct vas_instance *vinst; | |
318 | int winid; | |
319 | bool tx_win; /* True if send window */ | |
320 | bool nx_win; /* True if NX window */ | |
321 | bool user_win; /* True if user space window */ | |
322 | void *hvwc_map; /* HV window context */ | |
323 | void *uwc_map; /* OS/User window context */ | |
324 | pid_t pid; /* Linux process id of owner */ | |
325 | ||
326 | /* Fields applicable only to send windows */ | |
327 | void *paste_kaddr; | |
328 | char *paste_addr_name; | |
329 | struct vas_window *rxwin; | |
330 | ||
331 | /* Feilds applicable only to receive windows */ | |
332 | enum vas_cop_type cop; | |
333 | atomic_t num_txwins; | |
334 | }; | |
335 | ||
336 | /* | |
337 | * Container for the hardware state of a window. One per-window. | |
338 | * | |
339 | * A VAS Window context is a 512-byte area in the hardware that contains | |
340 | * a set of 64-bit registers. Individual bit-fields in these registers | |
341 | * determine the configuration/operation of the hardware. struct vas_winctx | |
342 | * is a container for the register fields in the window context. | |
343 | */ | |
344 | struct vas_winctx { | |
345 | void *rx_fifo; | |
346 | int rx_fifo_size; | |
347 | int wcreds_max; | |
348 | int rsvd_txbuf_count; | |
349 | ||
350 | bool user_win; | |
351 | bool nx_win; | |
352 | bool fault_win; | |
353 | bool rsvd_txbuf_enable; | |
354 | bool pin_win; | |
355 | bool rej_no_credit; | |
356 | bool tx_wcred_mode; | |
357 | bool rx_wcred_mode; | |
358 | bool tx_word_mode; | |
359 | bool rx_word_mode; | |
360 | bool data_stamp; | |
361 | bool xtra_write; | |
362 | bool notify_disable; | |
363 | bool intr_disable; | |
364 | bool fifo_disable; | |
365 | bool notify_early; | |
366 | bool notify_os_intr_reg; | |
367 | ||
368 | int lpid; | |
369 | int pidr; /* value from SPRN_PID, not linux pid */ | |
370 | int lnotify_lpid; | |
371 | int lnotify_pid; | |
372 | int lnotify_tid; | |
373 | u32 pswid; | |
374 | int rx_win_id; | |
375 | int fault_win_id; | |
376 | int tc_mode; | |
377 | ||
378 | u64 irq_port; | |
379 | ||
380 | enum vas_dma_type dma_type; | |
381 | enum vas_notify_scope min_scope; | |
382 | enum vas_notify_scope max_scope; | |
383 | enum vas_notify_after_count notify_after_count; | |
384 | }; | |
385 | ||
4dea2d1a SB |
386 | extern struct vas_instance *find_vas_instance(int vasid); |
387 | ||
b25b33ac SB |
388 | /* |
389 | * VREG(x): | |
390 | * Expand a register's short name (eg: LPID) into two parameters: | |
391 | * - the register's short name in string form ("LPID"), and | |
392 | * - the name of the macro (eg: VAS_LPID_OFFSET), defining the | |
393 | * register's offset in the window context | |
394 | */ | |
395 | #define VREG_SFX(n, s) __stringify(n), VAS_##n##s | |
396 | #define VREG(r) VREG_SFX(r, _OFFSET) | |
397 | ||
398 | #ifdef vas_debug | |
62c4eda4 SB |
399 | static inline void dump_rx_win_attr(struct vas_rx_win_attr *attr) |
400 | { | |
401 | pr_err("VAS: fault %d, notify %d, intr %d early %d\n", | |
402 | attr->fault_win, attr->notify_disable, | |
403 | attr->intr_disable, attr->notify_early); | |
404 | ||
405 | pr_err("VAS: rx_fifo_size %d, max value %d\n", | |
406 | attr->rx_fifo_size, VAS_RX_FIFO_SIZE_MAX); | |
407 | } | |
408 | ||
b25b33ac SB |
409 | static inline void vas_log_write(struct vas_window *win, char *name, |
410 | void *regptr, u64 val) | |
411 | { | |
412 | if (val) | |
413 | pr_err("%swin #%d: %s reg %p, val 0x%016llx\n", | |
414 | win->tx_win ? "Tx" : "Rx", win->winid, name, | |
415 | regptr, val); | |
416 | } | |
417 | ||
418 | #else /* vas_debug */ | |
419 | ||
420 | #define vas_log_write(win, name, reg, val) | |
62c4eda4 | 421 | #define dump_rx_win_attr(attr) |
b25b33ac SB |
422 | |
423 | #endif /* vas_debug */ | |
424 | ||
425 | static inline void write_uwc_reg(struct vas_window *win, char *name, | |
426 | s32 reg, u64 val) | |
427 | { | |
428 | void *regptr; | |
429 | ||
430 | regptr = win->uwc_map + reg; | |
431 | vas_log_write(win, name, regptr, val); | |
432 | ||
433 | out_be64(regptr, val); | |
434 | } | |
435 | ||
436 | static inline void write_hvwc_reg(struct vas_window *win, char *name, | |
437 | s32 reg, u64 val) | |
438 | { | |
439 | void *regptr; | |
440 | ||
441 | regptr = win->hvwc_map + reg; | |
442 | vas_log_write(win, name, regptr, val); | |
443 | ||
444 | out_be64(regptr, val); | |
445 | } | |
446 | ||
447 | static inline u64 read_hvwc_reg(struct vas_window *win, | |
448 | char *name __maybe_unused, s32 reg) | |
449 | { | |
450 | return in_be64(win->hvwc_map+reg); | |
451 | } | |
452 | ||
96768914 | 453 | #endif /* _VAS_H */ |