2 * Copyright 2016-17 IBM Corp.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
12 #include <linux/atomic.h>
13 #include <linux/idr.h>
18 * Overview of Virtual Accelerator Switchboard (VAS).
20 * VAS is a hardware "switchboard" that allows senders and receivers to
21 * exchange messages with _minimal_ kernel involvment. The receivers are
22 * typically NX coprocessor engines that perform compression or encryption
23 * in hardware, but receivers can also be other software threads.
25 * Senders are user/kernel threads that submit compression/encryption or
26 * other requests to the receivers. Senders must format their messages as
27 * Coprocessor Request Blocks (CRB)s and submit them using the "copy" and
28 * "paste" instructions which were introduced in Power9.
30 * A Power node can have (upto?) 8 Power chips. There is one instance of
31 * VAS in each Power9 chip. Each instance of VAS has 64K windows or ports,
32 * Senders and receivers must each connect to a separate window before they
33 * can exchange messages through the switchboard.
35 * Each window is described by two types of window contexts:
37 * Hypervisor Window Context (HVWC) of size VAS_HVWC_SIZE bytes
39 * OS/User Window Context (UWC) of size VAS_UWC_SIZE bytes.
41 * A window context can be viewed as a set of 64-bit registers. The settings
42 * in these registers configure/control/determine the behavior of the VAS
43 * hardware when messages are sent/received through the window. The registers
44 * in the HVWC are configured by the kernel while the registers in the UWC can
45 * be configured by the kernel or by the user space application that is using
48 * The HVWCs for all windows on a specific instance of VAS are in a contiguous
49 * range of hardware addresses or Base address region (BAR) referred to as the
50 * HVWC BAR for the instance. Similarly the UWCs for all windows on an instance
51 * are referred to as the UWC BAR for the instance.
53 * The two BARs for each instance are defined Power9 MMIO Ranges spreadsheet
54 * and available to the kernel in the VAS node's "reg" property in the device
57 * /proc/device-tree/vasm@.../reg
59 * (see vas_probe() for details on the reg property).
61 * The kernel maps the HVWC and UWC BAR regions into the kernel address
62 * space (hvwc_map and uwc_map). The kernel can then access the window
63 * contexts of a specific window using:
65 * hvwc = hvwc_map + winid * VAS_HVWC_SIZE.
66 * uwc = uwc_map + winid * VAS_UWC_SIZE.
68 * where winid is the window index (0..64K).
70 * As mentioned, a window context is used to "configure" a window. Besides
71 * this configuration address, each _send_ window also has a unique hardware
72 * "paste" address that is used to submit requests/CRBs (see vas_paste_crb()).
74 * The hardware paste address for a window is computed using the "paste
75 * base address" and "paste win id shift" reg properties in the VAS device
78 * paste_addr = paste_base + ((winid << paste_win_id_shift))
80 * (again, see vas_probe() for ->paste_base_addr and ->paste_win_id_shift).
82 * The kernel maps this hardware address into the sender's address space
83 * after which they can use the 'paste' instruction (new in Power9) to
84 * send a message (submit a request aka CRB) to the coprocessor.
86 * NOTE: In the initial version, senders can only in-kernel drivers/threads.
87 * Support for user space threads will be added in follow-on patches.
89 * TODO: Do we need to map the UWC into user address space so they can return
90 * credits? Its NA for NX but may be needed for other receive windows.
94 #define VAS_WINDOWS_PER_CHIP (64 << 10)
97 * Hypervisor and OS/USer Window Context sizes
99 #define VAS_HVWC_SIZE 512
100 #define VAS_UWC_SIZE PAGE_SIZE
103 * Initial per-process credits.
104 * Max send window credits: 4K-1 (12-bits in VAS_TX_WCRED)
105 * Max receive window credits: 64K-1 (16 bits in VAS_LRX_WCRED)
107 * TODO: Needs tuning for per-process credits
109 #define VAS_WCREDS_MIN 16
110 #define VAS_WCREDS_MAX ((64 << 10) - 1)
111 #define VAS_WCREDS_DEFAULT (1 << 10)
114 * VAS Window Context Register Offsets and bitmasks.
115 * See Section 3.1.4 of VAS Work book
117 #define VAS_LPID_OFFSET 0x010
118 #define VAS_LPID PPC_BITMASK(0, 11)
120 #define VAS_PID_OFFSET 0x018
121 #define VAS_PID_ID PPC_BITMASK(0, 19)
123 #define VAS_XLATE_MSR_OFFSET 0x020
124 #define VAS_XLATE_MSR_DR PPC_BIT(0)
125 #define VAS_XLATE_MSR_TA PPC_BIT(1)
126 #define VAS_XLATE_MSR_PR PPC_BIT(2)
127 #define VAS_XLATE_MSR_US PPC_BIT(3)
128 #define VAS_XLATE_MSR_HV PPC_BIT(4)
129 #define VAS_XLATE_MSR_SF PPC_BIT(5)
131 #define VAS_XLATE_LPCR_OFFSET 0x028
132 #define VAS_XLATE_LPCR_PAGE_SIZE PPC_BITMASK(0, 2)
133 #define VAS_XLATE_LPCR_ISL PPC_BIT(3)
134 #define VAS_XLATE_LPCR_TC PPC_BIT(4)
135 #define VAS_XLATE_LPCR_SC PPC_BIT(5)
137 #define VAS_XLATE_CTL_OFFSET 0x030
138 #define VAS_XLATE_MODE PPC_BITMASK(0, 1)
140 #define VAS_AMR_OFFSET 0x040
141 #define VAS_AMR PPC_BITMASK(0, 63)
143 #define VAS_SEIDR_OFFSET 0x048
144 #define VAS_SEIDR PPC_BITMASK(0, 63)
146 #define VAS_FAULT_TX_WIN_OFFSET 0x050
147 #define VAS_FAULT_TX_WIN PPC_BITMASK(48, 63)
149 #define VAS_OSU_INTR_SRC_RA_OFFSET 0x060
150 #define VAS_OSU_INTR_SRC_RA PPC_BITMASK(8, 63)
152 #define VAS_HV_INTR_SRC_RA_OFFSET 0x070
153 #define VAS_HV_INTR_SRC_RA PPC_BITMASK(8, 63)
155 #define VAS_PSWID_OFFSET 0x078
156 #define VAS_PSWID_EA_HANDLE PPC_BITMASK(0, 31)
158 #define VAS_SPARE1_OFFSET 0x080
159 #define VAS_SPARE2_OFFSET 0x088
160 #define VAS_SPARE3_OFFSET 0x090
161 #define VAS_SPARE4_OFFSET 0x130
162 #define VAS_SPARE5_OFFSET 0x160
163 #define VAS_SPARE6_OFFSET 0x188
165 #define VAS_LFIFO_BAR_OFFSET 0x0A0
166 #define VAS_LFIFO_BAR PPC_BITMASK(8, 53)
167 #define VAS_PAGE_MIGRATION_SELECT PPC_BITMASK(54, 56)
169 #define VAS_LDATA_STAMP_CTL_OFFSET 0x0A8
170 #define VAS_LDATA_STAMP PPC_BITMASK(0, 1)
171 #define VAS_XTRA_WRITE PPC_BIT(2)
173 #define VAS_LDMA_CACHE_CTL_OFFSET 0x0B0
174 #define VAS_LDMA_TYPE PPC_BITMASK(0, 1)
175 #define VAS_LDMA_FIFO_DISABLE PPC_BIT(2)
177 #define VAS_LRFIFO_PUSH_OFFSET 0x0B8
178 #define VAS_LRFIFO_PUSH PPC_BITMASK(0, 15)
180 #define VAS_CURR_MSG_COUNT_OFFSET 0x0C0
181 #define VAS_CURR_MSG_COUNT PPC_BITMASK(0, 7)
183 #define VAS_LNOTIFY_AFTER_COUNT_OFFSET 0x0C8
184 #define VAS_LNOTIFY_AFTER_COUNT PPC_BITMASK(0, 7)
186 #define VAS_LRX_WCRED_OFFSET 0x0E0
187 #define VAS_LRX_WCRED PPC_BITMASK(0, 15)
189 #define VAS_LRX_WCRED_ADDER_OFFSET 0x190
190 #define VAS_LRX_WCRED_ADDER PPC_BITMASK(0, 15)
192 #define VAS_TX_WCRED_OFFSET 0x0F0
193 #define VAS_TX_WCRED PPC_BITMASK(4, 15)
195 #define VAS_TX_WCRED_ADDER_OFFSET 0x1A0
196 #define VAS_TX_WCRED_ADDER PPC_BITMASK(4, 15)
198 #define VAS_LFIFO_SIZE_OFFSET 0x100
199 #define VAS_LFIFO_SIZE PPC_BITMASK(0, 3)
201 #define VAS_WINCTL_OFFSET 0x108
202 #define VAS_WINCTL_OPEN PPC_BIT(0)
203 #define VAS_WINCTL_REJ_NO_CREDIT PPC_BIT(1)
204 #define VAS_WINCTL_PIN PPC_BIT(2)
205 #define VAS_WINCTL_TX_WCRED_MODE PPC_BIT(3)
206 #define VAS_WINCTL_RX_WCRED_MODE PPC_BIT(4)
207 #define VAS_WINCTL_TX_WORD_MODE PPC_BIT(5)
208 #define VAS_WINCTL_RX_WORD_MODE PPC_BIT(6)
209 #define VAS_WINCTL_RSVD_TXBUF PPC_BIT(7)
210 #define VAS_WINCTL_THRESH_CTL PPC_BITMASK(8, 9)
211 #define VAS_WINCTL_FAULT_WIN PPC_BIT(10)
212 #define VAS_WINCTL_NX_WIN PPC_BIT(11)
214 #define VAS_WIN_STATUS_OFFSET 0x110
215 #define VAS_WIN_BUSY PPC_BIT(1)
217 #define VAS_WIN_CTX_CACHING_CTL_OFFSET 0x118
218 #define VAS_CASTOUT_REQ PPC_BIT(0)
219 #define VAS_PUSH_TO_MEM PPC_BIT(1)
220 #define VAS_WIN_CACHE_STATUS PPC_BIT(4)
222 #define VAS_TX_RSVD_BUF_COUNT_OFFSET 0x120
223 #define VAS_RXVD_BUF_COUNT PPC_BITMASK(58, 63)
225 #define VAS_LRFIFO_WIN_PTR_OFFSET 0x128
226 #define VAS_LRX_WIN_ID PPC_BITMASK(0, 15)
229 * Local Notification Control Register controls what happens in _response_
230 * to a paste command and hence applies only to receive windows.
232 #define VAS_LNOTIFY_CTL_OFFSET 0x138
233 #define VAS_NOTIFY_DISABLE PPC_BIT(0)
234 #define VAS_INTR_DISABLE PPC_BIT(1)
235 #define VAS_NOTIFY_EARLY PPC_BIT(2)
236 #define VAS_NOTIFY_OSU_INTR PPC_BIT(3)
238 #define VAS_LNOTIFY_PID_OFFSET 0x140
239 #define VAS_LNOTIFY_PID PPC_BITMASK(0, 19)
241 #define VAS_LNOTIFY_LPID_OFFSET 0x148
242 #define VAS_LNOTIFY_LPID PPC_BITMASK(0, 11)
244 #define VAS_LNOTIFY_TID_OFFSET 0x150
245 #define VAS_LNOTIFY_TID PPC_BITMASK(0, 15)
247 #define VAS_LNOTIFY_SCOPE_OFFSET 0x158
248 #define VAS_LNOTIFY_MIN_SCOPE PPC_BITMASK(0, 1)
249 #define VAS_LNOTIFY_MAX_SCOPE PPC_BITMASK(2, 3)
251 #define VAS_NX_UTIL_OFFSET 0x1B0
252 #define VAS_NX_UTIL PPC_BITMASK(0, 63)
254 /* SE: Side effects */
255 #define VAS_NX_UTIL_SE_OFFSET 0x1B8
256 #define VAS_NX_UTIL_SE PPC_BITMASK(0, 63)
258 #define VAS_NX_UTIL_ADDER_OFFSET 0x180
259 #define VAS_NX_UTIL_ADDER PPC_BITMASK(32, 63)
262 * Local Notify Scope Control Register. (Receive windows only).
264 enum vas_notify_scope {
267 VAS_SCOPE_VECTORED_GROUP,
272 * Local DMA Cache Control Register (Receive windows only).
280 * Local Notify Scope Control Register. (Receive windows only).
281 * Not applicable to NX receive windows.
283 enum vas_notify_after_count {
284 VAS_NOTIFY_AFTER_256 = 0,
290 * One per instance of VAS. Each instance will have a separate set of
291 * receive windows, one per coprocessor type.
293 * See also function header of set_vinst_win() for details on ->windows[]
294 * and ->rxwin[] tables.
296 struct vas_instance {
299 struct list_head node;
300 struct platform_device *pdev;
305 u64 paste_win_id_shift;
308 struct vas_window *rxwin[VAS_COP_TYPE_MAX];
309 struct vas_window *windows[VAS_WINDOWS_PER_CHIP];
313 * In-kernel state a VAS window. One per window.
316 /* Fields common to send and receive windows */
317 struct vas_instance *vinst;
319 bool tx_win; /* True if send window */
320 bool nx_win; /* True if NX window */
321 bool user_win; /* True if user space window */
322 void *hvwc_map; /* HV window context */
323 void *uwc_map; /* OS/User window context */
324 pid_t pid; /* Linux process id of owner */
326 /* Fields applicable only to send windows */
328 char *paste_addr_name;
329 struct vas_window *rxwin;
331 /* Feilds applicable only to receive windows */
332 enum vas_cop_type cop;
337 * Container for the hardware state of a window. One per-window.
339 * A VAS Window context is a 512-byte area in the hardware that contains
340 * a set of 64-bit registers. Individual bit-fields in these registers
341 * determine the configuration/operation of the hardware. struct vas_winctx
342 * is a container for the register fields in the window context.
348 int rsvd_txbuf_count;
353 bool rsvd_txbuf_enable;
366 bool notify_os_intr_reg;
369 int pidr; /* value from SPRN_PID, not linux pid */
380 enum vas_dma_type dma_type;
381 enum vas_notify_scope min_scope;
382 enum vas_notify_scope max_scope;
383 enum vas_notify_after_count notify_after_count;
386 extern struct vas_instance *find_vas_instance(int vasid);
390 * Expand a register's short name (eg: LPID) into two parameters:
391 * - the register's short name in string form ("LPID"), and
392 * - the name of the macro (eg: VAS_LPID_OFFSET), defining the
393 * register's offset in the window context
395 #define VREG_SFX(n, s) __stringify(n), VAS_##n##s
396 #define VREG(r) VREG_SFX(r, _OFFSET)
399 static inline void dump_rx_win_attr(struct vas_rx_win_attr *attr)
401 pr_err("VAS: fault %d, notify %d, intr %d early %d\n",
402 attr->fault_win, attr->notify_disable,
403 attr->intr_disable, attr->notify_early);
405 pr_err("VAS: rx_fifo_size %d, max value %d\n",
406 attr->rx_fifo_size, VAS_RX_FIFO_SIZE_MAX);
409 static inline void vas_log_write(struct vas_window *win, char *name,
410 void *regptr, u64 val)
413 pr_err("%swin #%d: %s reg %p, val 0x%016llx\n",
414 win->tx_win ? "Tx" : "Rx", win->winid, name,
418 #else /* vas_debug */
420 #define vas_log_write(win, name, reg, val)
421 #define dump_rx_win_attr(attr)
423 #endif /* vas_debug */
425 static inline void write_uwc_reg(struct vas_window *win, char *name,
430 regptr = win->uwc_map + reg;
431 vas_log_write(win, name, regptr, val);
433 out_be64(regptr, val);
436 static inline void write_hvwc_reg(struct vas_window *win, char *name,
441 regptr = win->hvwc_map + reg;
442 vas_log_write(win, name, regptr, val);
444 out_be64(regptr, val);
447 static inline u64 read_hvwc_reg(struct vas_window *win,
448 char *name __maybe_unused, s32 reg)
450 return in_be64(win->hvwc_map+reg);