Commit | Line | Data |
---|---|---|
2959ab24 NG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * CDX bus driver. | |
4 | * | |
5 | * Copyright (C) 2022-2023, Advanced Micro Devices, Inc. | |
6 | */ | |
7 | ||
8 | /* | |
9 | * Architecture Overview | |
10 | * ===================== | |
11 | * CDX is a Hardware Architecture designed for AMD FPGA devices. It | |
12 | * consists of sophisticated mechanism for interaction between FPGA, | |
13 | * Firmware and the APUs (Application CPUs). | |
14 | * | |
15 | * Firmware resides on RPU (Realtime CPUs) which interacts with | |
16 | * the FPGA program manager and the APUs. The RPU provides memory-mapped | |
17 | * interface (RPU if) which is used to communicate with APUs. | |
18 | * | |
19 | * The diagram below shows an overview of the CDX architecture: | |
20 | * | |
21 | * +--------------------------------------+ | |
22 | * | Application CPUs (APU) | | |
23 | * | | | |
24 | * | CDX device drivers| | |
25 | * | Linux OS | | | |
26 | * | CDX bus | | |
27 | * | | | | |
28 | * | CDX controller | | |
29 | * | | | | |
30 | * +-----------------------------|--------+ | |
31 | * | (discover, config, | |
32 | * | reset, rescan) | |
33 | * | | |
34 | * +------------------------| RPU if |----+ | |
35 | * | | | | |
36 | * | V | | |
37 | * | Realtime CPUs (RPU) | | |
38 | * | | | |
39 | * +--------------------------------------+ | |
40 | * | | |
41 | * +---------------------|----------------+ | |
42 | * | FPGA | | | |
43 | * | +-----------------------+ | | |
44 | * | | | | | | |
45 | * | +-------+ +-------+ +-------+ | | |
46 | * | | dev 1 | | dev 2 | | dev 3 | | | |
47 | * | +-------+ +-------+ +-------+ | | |
48 | * +--------------------------------------+ | |
49 | * | |
50 | * The RPU firmware extracts the device information from the loaded FPGA | |
51 | * image and implements a mechanism that allows the APU drivers to | |
52 | * enumerate such devices (device personality and resource details) via | |
53 | * a dedicated communication channel. RPU mediates operations such as | |
54 | * discover, reset and rescan of the FPGA devices for the APU. This is | |
55 | * done using memory mapped interface provided by the RPU to APU. | |
56 | */ | |
57 | ||
58 | #include <linux/init.h> | |
59 | #include <linux/kernel.h> | |
60 | #include <linux/of_device.h> | |
61 | #include <linux/slab.h> | |
62 | #include <linux/mm.h> | |
63 | #include <linux/xarray.h> | |
64 | #include <linux/cdx/cdx_bus.h> | |
b8c5ff76 NG |
65 | #include <linux/iommu.h> |
66 | #include <linux/dma-map-ops.h> | |
2959ab24 NG |
67 | #include "cdx.h" |
68 | ||
69 | /* Default DMA mask for devices on a CDX bus */ | |
70 | #define CDX_DEFAULT_DMA_MASK (~0ULL) | |
71 | #define MAX_CDX_CONTROLLERS 16 | |
72 | ||
73 | /* CDX controllers registered with the CDX bus */ | |
74 | static DEFINE_XARRAY_ALLOC(cdx_controllers); | |
75 | ||
48a6c7bc NG |
76 | /** |
77 | * cdx_dev_reset - Reset a CDX device | |
78 | * @dev: CDX device | |
79 | * | |
80 | * Return: -errno on failure, 0 on success. | |
81 | */ | |
82 | int cdx_dev_reset(struct device *dev) | |
83 | { | |
84 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
85 | struct cdx_controller *cdx = cdx_dev->cdx; | |
86 | struct cdx_device_config dev_config = {0}; | |
87 | struct cdx_driver *cdx_drv; | |
88 | int ret; | |
89 | ||
90 | cdx_drv = to_cdx_driver(dev->driver); | |
91 | /* Notify driver that device is being reset */ | |
92 | if (cdx_drv && cdx_drv->reset_prepare) | |
93 | cdx_drv->reset_prepare(cdx_dev); | |
94 | ||
95 | dev_config.type = CDX_DEV_RESET_CONF; | |
96 | ret = cdx->ops->dev_configure(cdx, cdx_dev->bus_num, | |
97 | cdx_dev->dev_num, &dev_config); | |
98 | if (ret) | |
99 | dev_err(dev, "cdx device reset failed\n"); | |
100 | ||
101 | /* Notify driver that device reset is complete */ | |
102 | if (cdx_drv && cdx_drv->reset_done) | |
103 | cdx_drv->reset_done(cdx_dev); | |
104 | ||
105 | return ret; | |
106 | } | |
107 | EXPORT_SYMBOL_GPL(cdx_dev_reset); | |
108 | ||
2959ab24 NG |
109 | /** |
110 | * cdx_unregister_device - Unregister a CDX device | |
111 | * @dev: CDX device | |
112 | * @data: This is always passed as NULL, and is not used in this API, | |
113 | * but is required here as the bus_for_each_dev() API expects | |
114 | * the passed function (cdx_unregister_device) to have this | |
115 | * as an argument. | |
116 | * | |
117 | * Return: 0 on success. | |
118 | */ | |
119 | static int cdx_unregister_device(struct device *dev, | |
120 | void *data) | |
121 | { | |
122 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
123 | ||
124 | kfree(cdx_dev->driver_override); | |
125 | cdx_dev->driver_override = NULL; | |
126 | /* | |
127 | * Do not free cdx_dev here as it would be freed in | |
128 | * cdx_device_release() called from within put_device(). | |
129 | */ | |
130 | device_del(&cdx_dev->dev); | |
131 | put_device(&cdx_dev->dev); | |
132 | ||
133 | return 0; | |
134 | } | |
135 | ||
136 | static void cdx_unregister_devices(struct bus_type *bus) | |
137 | { | |
138 | /* Reset all the devices attached to cdx bus */ | |
139 | bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device); | |
140 | } | |
141 | ||
142 | /** | |
143 | * cdx_match_one_device - Tell if a CDX device structure has a matching | |
144 | * CDX device id structure | |
145 | * @id: single CDX device id structure to match | |
146 | * @dev: the CDX device structure to match against | |
147 | * | |
148 | * Return: matching cdx_device_id structure or NULL if there is no match. | |
149 | */ | |
150 | static inline const struct cdx_device_id * | |
151 | cdx_match_one_device(const struct cdx_device_id *id, | |
152 | const struct cdx_device *dev) | |
153 | { | |
154 | /* Use vendor ID and device ID for matching */ | |
155 | if ((id->vendor == CDX_ANY_ID || id->vendor == dev->vendor) && | |
156 | (id->device == CDX_ANY_ID || id->device == dev->device)) | |
157 | return id; | |
158 | return NULL; | |
159 | } | |
160 | ||
161 | /** | |
162 | * cdx_match_id - See if a CDX device matches a given cdx_id table | |
163 | * @ids: array of CDX device ID structures to search in | |
164 | * @dev: the CDX device structure to match against. | |
165 | * | |
166 | * Used by a driver to check whether a CDX device is in its list of | |
167 | * supported devices. Returns the matching cdx_device_id structure or | |
168 | * NULL if there is no match. | |
169 | * | |
170 | * Return: matching cdx_device_id structure or NULL if there is no match. | |
171 | */ | |
172 | static inline const struct cdx_device_id * | |
173 | cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev) | |
174 | { | |
175 | if (ids) { | |
176 | while (ids->vendor || ids->device) { | |
177 | if (cdx_match_one_device(ids, dev)) | |
178 | return ids; | |
179 | ids++; | |
180 | } | |
181 | } | |
182 | return NULL; | |
183 | } | |
184 | ||
185 | /** | |
186 | * cdx_bus_match - device to driver matching callback | |
187 | * @dev: the cdx device to match against | |
188 | * @drv: the device driver to search for matching cdx device | |
189 | * structures | |
190 | * | |
191 | * Return: true on success, false otherwise. | |
192 | */ | |
193 | static int cdx_bus_match(struct device *dev, struct device_driver *drv) | |
194 | { | |
195 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
196 | struct cdx_driver *cdx_drv = to_cdx_driver(drv); | |
197 | const struct cdx_device_id *found_id = NULL; | |
198 | const struct cdx_device_id *ids; | |
199 | ||
200 | ids = cdx_drv->match_id_table; | |
201 | ||
202 | /* When driver_override is set, only bind to the matching driver */ | |
203 | if (cdx_dev->driver_override && strcmp(cdx_dev->driver_override, drv->name)) | |
204 | return false; | |
205 | ||
206 | found_id = cdx_match_id(ids, cdx_dev); | |
207 | if (!found_id) | |
208 | return false; | |
209 | ||
210 | do { | |
211 | /* | |
212 | * In case override_only was set, enforce driver_override | |
213 | * matching. | |
214 | */ | |
215 | if (!found_id->override_only) | |
216 | return true; | |
217 | if (cdx_dev->driver_override) | |
218 | return true; | |
219 | ||
220 | ids = found_id + 1; | |
221 | found_id = cdx_match_id(ids, cdx_dev); | |
222 | } while (found_id); | |
223 | ||
224 | return false; | |
225 | } | |
226 | ||
227 | static int cdx_probe(struct device *dev) | |
228 | { | |
229 | struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); | |
230 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
231 | int error; | |
232 | ||
233 | error = cdx_drv->probe(cdx_dev); | |
234 | if (error) { | |
235 | dev_err_probe(dev, error, "%s failed\n", __func__); | |
236 | return error; | |
237 | } | |
238 | ||
239 | return 0; | |
240 | } | |
241 | ||
242 | static void cdx_remove(struct device *dev) | |
243 | { | |
244 | struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); | |
245 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
246 | ||
247 | if (cdx_drv && cdx_drv->remove) | |
248 | cdx_drv->remove(cdx_dev); | |
249 | } | |
250 | ||
251 | static void cdx_shutdown(struct device *dev) | |
252 | { | |
253 | struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); | |
254 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
255 | ||
256 | if (cdx_drv && cdx_drv->shutdown) | |
257 | cdx_drv->shutdown(cdx_dev); | |
258 | } | |
259 | ||
260 | static int cdx_dma_configure(struct device *dev) | |
261 | { | |
b8c5ff76 | 262 | struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); |
2959ab24 NG |
263 | struct cdx_device *cdx_dev = to_cdx_device(dev); |
264 | u32 input_id = cdx_dev->req_id; | |
265 | int ret; | |
266 | ||
267 | ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id); | |
268 | if (ret && ret != -EPROBE_DEFER) { | |
269 | dev_err(dev, "of_dma_configure_id() failed\n"); | |
270 | return ret; | |
271 | } | |
272 | ||
b8c5ff76 NG |
273 | if (!ret && !cdx_drv->driver_managed_dma) { |
274 | ret = iommu_device_use_default_domain(dev); | |
275 | if (ret) | |
276 | arch_teardown_dma_ops(dev); | |
277 | } | |
278 | ||
2959ab24 NG |
279 | return 0; |
280 | } | |
281 | ||
b8c5ff76 NG |
282 | static void cdx_dma_cleanup(struct device *dev) |
283 | { | |
284 | struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver); | |
285 | ||
286 | if (!cdx_drv->driver_managed_dma) | |
287 | iommu_device_unuse_default_domain(dev); | |
288 | } | |
289 | ||
48a6c7bc NG |
290 | /* show configuration fields */ |
291 | #define cdx_config_attr(field, format_string) \ | |
292 | static ssize_t \ | |
293 | field##_show(struct device *dev, struct device_attribute *attr, char *buf) \ | |
294 | { \ | |
295 | struct cdx_device *cdx_dev = to_cdx_device(dev); \ | |
296 | return sysfs_emit(buf, format_string, cdx_dev->field); \ | |
297 | } \ | |
298 | static DEVICE_ATTR_RO(field) | |
299 | ||
300 | cdx_config_attr(vendor, "0x%04x\n"); | |
301 | cdx_config_attr(device, "0x%04x\n"); | |
302 | ||
303 | static ssize_t remove_store(struct device *dev, | |
304 | struct device_attribute *attr, | |
305 | const char *buf, size_t count) | |
306 | { | |
307 | bool val; | |
308 | ||
309 | if (kstrtobool(buf, &val) < 0) | |
310 | return -EINVAL; | |
311 | ||
312 | if (!val) | |
313 | return -EINVAL; | |
314 | ||
315 | if (device_remove_file_self(dev, attr)) { | |
316 | int ret; | |
317 | ||
318 | ret = cdx_unregister_device(dev, NULL); | |
319 | if (ret) | |
320 | return ret; | |
321 | } | |
322 | ||
323 | return count; | |
324 | } | |
325 | static DEVICE_ATTR_WO(remove); | |
326 | ||
327 | static ssize_t reset_store(struct device *dev, struct device_attribute *attr, | |
328 | const char *buf, size_t count) | |
329 | { | |
330 | bool val; | |
331 | int ret; | |
332 | ||
333 | if (kstrtobool(buf, &val) < 0) | |
334 | return -EINVAL; | |
335 | ||
336 | if (!val) | |
337 | return -EINVAL; | |
338 | ||
339 | ret = cdx_dev_reset(dev); | |
340 | if (ret) | |
341 | return ret; | |
342 | ||
343 | return count; | |
344 | } | |
345 | static DEVICE_ATTR_WO(reset); | |
346 | ||
347 | static ssize_t driver_override_store(struct device *dev, | |
348 | struct device_attribute *attr, | |
349 | const char *buf, size_t count) | |
350 | { | |
351 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
352 | int ret; | |
353 | ||
354 | if (WARN_ON(dev->bus != &cdx_bus_type)) | |
355 | return -EINVAL; | |
356 | ||
357 | ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count); | |
358 | if (ret) | |
359 | return ret; | |
360 | ||
361 | return count; | |
362 | } | |
363 | ||
364 | static ssize_t driver_override_show(struct device *dev, | |
365 | struct device_attribute *attr, char *buf) | |
366 | { | |
367 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
368 | ||
369 | return sysfs_emit(buf, "%s\n", cdx_dev->driver_override); | |
370 | } | |
371 | static DEVICE_ATTR_RW(driver_override); | |
372 | ||
373 | static struct attribute *cdx_dev_attrs[] = { | |
374 | &dev_attr_remove.attr, | |
375 | &dev_attr_reset.attr, | |
376 | &dev_attr_vendor.attr, | |
377 | &dev_attr_device.attr, | |
378 | &dev_attr_driver_override.attr, | |
379 | NULL, | |
380 | }; | |
381 | ATTRIBUTE_GROUPS(cdx_dev); | |
382 | ||
d06f5a3f | 383 | static ssize_t rescan_store(const struct bus_type *bus, |
2959ab24 NG |
384 | const char *buf, size_t count) |
385 | { | |
386 | struct cdx_controller *cdx; | |
387 | unsigned long index; | |
388 | bool val; | |
389 | ||
390 | if (kstrtobool(buf, &val) < 0) | |
391 | return -EINVAL; | |
392 | ||
393 | if (!val) | |
394 | return -EINVAL; | |
395 | ||
396 | /* Unregister all the devices on the bus */ | |
397 | cdx_unregister_devices(&cdx_bus_type); | |
398 | ||
399 | /* Rescan all the devices */ | |
400 | xa_for_each(&cdx_controllers, index, cdx) { | |
401 | int ret; | |
402 | ||
403 | ret = cdx->ops->scan(cdx); | |
404 | if (ret) | |
405 | dev_err(cdx->dev, "cdx bus scanning failed\n"); | |
406 | } | |
407 | ||
408 | return count; | |
409 | } | |
410 | static BUS_ATTR_WO(rescan); | |
411 | ||
412 | static struct attribute *cdx_bus_attrs[] = { | |
413 | &bus_attr_rescan.attr, | |
414 | NULL, | |
415 | }; | |
416 | ATTRIBUTE_GROUPS(cdx_bus); | |
417 | ||
418 | struct bus_type cdx_bus_type = { | |
419 | .name = "cdx", | |
420 | .match = cdx_bus_match, | |
421 | .probe = cdx_probe, | |
422 | .remove = cdx_remove, | |
423 | .shutdown = cdx_shutdown, | |
424 | .dma_configure = cdx_dma_configure, | |
b8c5ff76 | 425 | .dma_cleanup = cdx_dma_cleanup, |
2959ab24 | 426 | .bus_groups = cdx_bus_groups, |
48a6c7bc | 427 | .dev_groups = cdx_dev_groups, |
2959ab24 NG |
428 | }; |
429 | EXPORT_SYMBOL_GPL(cdx_bus_type); | |
430 | ||
431 | int __cdx_driver_register(struct cdx_driver *cdx_driver, | |
432 | struct module *owner) | |
433 | { | |
434 | int error; | |
435 | ||
436 | cdx_driver->driver.owner = owner; | |
437 | cdx_driver->driver.bus = &cdx_bus_type; | |
438 | ||
439 | error = driver_register(&cdx_driver->driver); | |
440 | if (error) { | |
441 | pr_err("driver_register() failed for %s: %d\n", | |
442 | cdx_driver->driver.name, error); | |
443 | return error; | |
444 | } | |
445 | ||
446 | return 0; | |
447 | } | |
448 | EXPORT_SYMBOL_GPL(__cdx_driver_register); | |
449 | ||
450 | void cdx_driver_unregister(struct cdx_driver *cdx_driver) | |
451 | { | |
452 | driver_unregister(&cdx_driver->driver); | |
453 | } | |
454 | EXPORT_SYMBOL_GPL(cdx_driver_unregister); | |
455 | ||
456 | static void cdx_device_release(struct device *dev) | |
457 | { | |
458 | struct cdx_device *cdx_dev = to_cdx_device(dev); | |
459 | ||
460 | kfree(cdx_dev); | |
461 | } | |
462 | ||
463 | int cdx_device_add(struct cdx_dev_params *dev_params) | |
464 | { | |
465 | struct cdx_controller *cdx = dev_params->cdx; | |
466 | struct device *parent = cdx->dev; | |
467 | struct cdx_device *cdx_dev; | |
468 | int ret; | |
469 | ||
470 | cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL); | |
471 | if (!cdx_dev) | |
472 | return -ENOMEM; | |
473 | ||
474 | /* Populate resource */ | |
475 | memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) * | |
476 | dev_params->res_count); | |
477 | cdx_dev->res_count = dev_params->res_count; | |
478 | ||
479 | /* Populate CDX dev params */ | |
480 | cdx_dev->req_id = dev_params->req_id; | |
481 | cdx_dev->vendor = dev_params->vendor; | |
482 | cdx_dev->device = dev_params->device; | |
483 | cdx_dev->bus_num = dev_params->bus_num; | |
484 | cdx_dev->dev_num = dev_params->dev_num; | |
485 | cdx_dev->cdx = dev_params->cdx; | |
486 | cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK; | |
487 | ||
488 | /* Initialize generic device */ | |
489 | device_initialize(&cdx_dev->dev); | |
490 | cdx_dev->dev.parent = parent; | |
491 | cdx_dev->dev.bus = &cdx_bus_type; | |
492 | cdx_dev->dev.dma_mask = &cdx_dev->dma_mask; | |
493 | cdx_dev->dev.release = cdx_device_release; | |
494 | ||
495 | /* Set Name */ | |
496 | dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", | |
497 | ((cdx->id << CDX_CONTROLLER_ID_SHIFT) | (cdx_dev->bus_num & CDX_BUS_NUM_MASK)), | |
498 | cdx_dev->dev_num); | |
499 | ||
500 | ret = device_add(&cdx_dev->dev); | |
501 | if (ret) { | |
502 | dev_err(&cdx_dev->dev, | |
503 | "cdx device add failed: %d", ret); | |
504 | goto fail; | |
505 | } | |
506 | ||
507 | return 0; | |
508 | fail: | |
509 | /* | |
510 | * Do not free cdx_dev here as it would be freed in | |
511 | * cdx_device_release() called from put_device(). | |
512 | */ | |
513 | put_device(&cdx_dev->dev); | |
514 | ||
515 | return ret; | |
516 | } | |
517 | EXPORT_SYMBOL_GPL(cdx_device_add); | |
518 | ||
519 | int cdx_register_controller(struct cdx_controller *cdx) | |
520 | { | |
521 | int ret; | |
522 | ||
523 | ret = xa_alloc(&cdx_controllers, &cdx->id, cdx, | |
524 | XA_LIMIT(0, MAX_CDX_CONTROLLERS - 1), GFP_KERNEL); | |
525 | if (ret) { | |
526 | dev_err(cdx->dev, | |
527 | "No free index available. Maximum controllers already registered\n"); | |
528 | cdx->id = (u8)MAX_CDX_CONTROLLERS; | |
529 | return ret; | |
530 | } | |
531 | ||
532 | /* Scan all the devices */ | |
533 | cdx->ops->scan(cdx); | |
534 | ||
535 | return 0; | |
536 | } | |
537 | EXPORT_SYMBOL_GPL(cdx_register_controller); | |
538 | ||
539 | void cdx_unregister_controller(struct cdx_controller *cdx) | |
540 | { | |
541 | if (cdx->id >= MAX_CDX_CONTROLLERS) | |
542 | return; | |
543 | ||
544 | device_for_each_child(cdx->dev, NULL, cdx_unregister_device); | |
545 | xa_erase(&cdx_controllers, cdx->id); | |
546 | } | |
547 | EXPORT_SYMBOL_GPL(cdx_unregister_controller); | |
548 | ||
549 | static int __init cdx_bus_init(void) | |
550 | { | |
551 | return bus_register(&cdx_bus_type); | |
552 | } | |
553 | postcore_initcall(cdx_bus_init); |