Commit | Line | Data |
---|---|---|
5b5c4e40 EP |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/types.h> | |
24 | #include <linux/kernel.h> | |
25 | #include <linux/pci.h> | |
26 | #include <linux/errno.h> | |
27 | #include <linux/acpi.h> | |
28 | #include <linux/hash.h> | |
29 | #include <linux/cpufreq.h> | |
f7c826ad | 30 | #include <linux/log2.h> |
5b5c4e40 EP |
31 | |
32 | #include "kfd_priv.h" | |
33 | #include "kfd_crat.h" | |
34 | #include "kfd_topology.h" | |
851a645e | 35 | #include "kfd_device_queue_manager.h" |
5b5c4e40 | 36 | |
4f449311 HK |
37 | /* topology_device_list - Master list of all topology devices */ |
38 | static struct list_head topology_device_list; | |
174de876 | 39 | struct kfd_system_properties sys_props; |
5b5c4e40 EP |
40 | |
41 | static DECLARE_RWSEM(topology_lock); | |
42 | ||
43 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id) | |
44 | { | |
45 | struct kfd_topology_device *top_dev; | |
46 | struct kfd_dev *device = NULL; | |
47 | ||
48 | down_read(&topology_lock); | |
49 | ||
50 | list_for_each_entry(top_dev, &topology_device_list, list) | |
51 | if (top_dev->gpu_id == gpu_id) { | |
52 | device = top_dev->gpu; | |
53 | break; | |
54 | } | |
55 | ||
56 | up_read(&topology_lock); | |
57 | ||
58 | return device; | |
59 | } | |
60 | ||
61 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev) | |
62 | { | |
63 | struct kfd_topology_device *top_dev; | |
64 | struct kfd_dev *device = NULL; | |
65 | ||
66 | down_read(&topology_lock); | |
67 | ||
68 | list_for_each_entry(top_dev, &topology_device_list, list) | |
69 | if (top_dev->gpu->pdev == pdev) { | |
70 | device = top_dev->gpu; | |
71 | break; | |
72 | } | |
73 | ||
74 | up_read(&topology_lock); | |
75 | ||
76 | return device; | |
77 | } | |
78 | ||
5b5c4e40 EP |
79 | static void kfd_release_topology_device(struct kfd_topology_device *dev) |
80 | { | |
81 | struct kfd_mem_properties *mem; | |
82 | struct kfd_cache_properties *cache; | |
83 | struct kfd_iolink_properties *iolink; | |
84 | ||
5b5c4e40 EP |
85 | list_del(&dev->list); |
86 | ||
87 | while (dev->mem_props.next != &dev->mem_props) { | |
88 | mem = container_of(dev->mem_props.next, | |
89 | struct kfd_mem_properties, list); | |
90 | list_del(&mem->list); | |
91 | kfree(mem); | |
92 | } | |
93 | ||
94 | while (dev->cache_props.next != &dev->cache_props) { | |
95 | cache = container_of(dev->cache_props.next, | |
96 | struct kfd_cache_properties, list); | |
97 | list_del(&cache->list); | |
98 | kfree(cache); | |
99 | } | |
100 | ||
101 | while (dev->io_link_props.next != &dev->io_link_props) { | |
102 | iolink = container_of(dev->io_link_props.next, | |
103 | struct kfd_iolink_properties, list); | |
104 | list_del(&iolink->list); | |
105 | kfree(iolink); | |
106 | } | |
107 | ||
108 | kfree(dev); | |
5b5c4e40 EP |
109 | } |
110 | ||
4f449311 | 111 | void kfd_release_topology_device_list(struct list_head *device_list) |
5b5c4e40 EP |
112 | { |
113 | struct kfd_topology_device *dev; | |
114 | ||
4f449311 HK |
115 | while (!list_empty(device_list)) { |
116 | dev = list_first_entry(device_list, | |
117 | struct kfd_topology_device, list); | |
5b5c4e40 | 118 | kfd_release_topology_device(dev); |
4f449311 | 119 | } |
5b5c4e40 EP |
120 | } |
121 | ||
4f449311 HK |
122 | static void kfd_release_live_view(void) |
123 | { | |
124 | kfd_release_topology_device_list(&topology_device_list); | |
5b5c4e40 EP |
125 | memset(&sys_props, 0, sizeof(sys_props)); |
126 | } | |
127 | ||
4f449311 HK |
128 | struct kfd_topology_device *kfd_create_topology_device( |
129 | struct list_head *device_list) | |
5b5c4e40 EP |
130 | { |
131 | struct kfd_topology_device *dev; | |
132 | ||
133 | dev = kfd_alloc_struct(dev); | |
4eacc26b | 134 | if (!dev) { |
5b5c4e40 | 135 | pr_err("No memory to allocate a topology device"); |
16b9201c | 136 | return NULL; |
5b5c4e40 EP |
137 | } |
138 | ||
139 | INIT_LIST_HEAD(&dev->mem_props); | |
140 | INIT_LIST_HEAD(&dev->cache_props); | |
141 | INIT_LIST_HEAD(&dev->io_link_props); | |
142 | ||
4f449311 | 143 | list_add_tail(&dev->list, device_list); |
5b5c4e40 EP |
144 | |
145 | return dev; | |
16b9201c | 146 | } |
5b5c4e40 | 147 | |
5b5c4e40 EP |
148 | |
149 | #define sysfs_show_gen_prop(buffer, fmt, ...) \ | |
150 | snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__) | |
151 | #define sysfs_show_32bit_prop(buffer, name, value) \ | |
152 | sysfs_show_gen_prop(buffer, "%s %u\n", name, value) | |
153 | #define sysfs_show_64bit_prop(buffer, name, value) \ | |
154 | sysfs_show_gen_prop(buffer, "%s %llu\n", name, value) | |
155 | #define sysfs_show_32bit_val(buffer, value) \ | |
156 | sysfs_show_gen_prop(buffer, "%u\n", value) | |
157 | #define sysfs_show_str_val(buffer, value) \ | |
158 | sysfs_show_gen_prop(buffer, "%s\n", value) | |
159 | ||
160 | static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, | |
161 | char *buffer) | |
162 | { | |
163 | ssize_t ret; | |
164 | ||
165 | /* Making sure that the buffer is an empty string */ | |
166 | buffer[0] = 0; | |
167 | ||
168 | if (attr == &sys_props.attr_genid) { | |
169 | ret = sysfs_show_32bit_val(buffer, sys_props.generation_count); | |
170 | } else if (attr == &sys_props.attr_props) { | |
171 | sysfs_show_64bit_prop(buffer, "platform_oem", | |
172 | sys_props.platform_oem); | |
173 | sysfs_show_64bit_prop(buffer, "platform_id", | |
174 | sys_props.platform_id); | |
175 | ret = sysfs_show_64bit_prop(buffer, "platform_rev", | |
176 | sys_props.platform_rev); | |
177 | } else { | |
178 | ret = -EINVAL; | |
179 | } | |
180 | ||
181 | return ret; | |
182 | } | |
183 | ||
5108d768 YZ |
184 | static void kfd_topology_kobj_release(struct kobject *kobj) |
185 | { | |
186 | kfree(kobj); | |
187 | } | |
188 | ||
5b5c4e40 EP |
189 | static const struct sysfs_ops sysprops_ops = { |
190 | .show = sysprops_show, | |
191 | }; | |
192 | ||
193 | static struct kobj_type sysprops_type = { | |
5108d768 | 194 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
195 | .sysfs_ops = &sysprops_ops, |
196 | }; | |
197 | ||
198 | static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, | |
199 | char *buffer) | |
200 | { | |
201 | ssize_t ret; | |
202 | struct kfd_iolink_properties *iolink; | |
203 | ||
204 | /* Making sure that the buffer is an empty string */ | |
205 | buffer[0] = 0; | |
206 | ||
207 | iolink = container_of(attr, struct kfd_iolink_properties, attr); | |
208 | sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type); | |
209 | sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj); | |
210 | sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min); | |
211 | sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from); | |
212 | sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to); | |
213 | sysfs_show_32bit_prop(buffer, "weight", iolink->weight); | |
214 | sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency); | |
215 | sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency); | |
216 | sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth); | |
217 | sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth); | |
218 | sysfs_show_32bit_prop(buffer, "recommended_transfer_size", | |
219 | iolink->rec_transfer_size); | |
220 | ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags); | |
221 | ||
222 | return ret; | |
223 | } | |
224 | ||
225 | static const struct sysfs_ops iolink_ops = { | |
226 | .show = iolink_show, | |
227 | }; | |
228 | ||
229 | static struct kobj_type iolink_type = { | |
5108d768 | 230 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
231 | .sysfs_ops = &iolink_ops, |
232 | }; | |
233 | ||
234 | static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, | |
235 | char *buffer) | |
236 | { | |
237 | ssize_t ret; | |
238 | struct kfd_mem_properties *mem; | |
239 | ||
240 | /* Making sure that the buffer is an empty string */ | |
241 | buffer[0] = 0; | |
242 | ||
243 | mem = container_of(attr, struct kfd_mem_properties, attr); | |
244 | sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type); | |
245 | sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes); | |
246 | sysfs_show_32bit_prop(buffer, "flags", mem->flags); | |
247 | sysfs_show_32bit_prop(buffer, "width", mem->width); | |
248 | ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max); | |
249 | ||
250 | return ret; | |
251 | } | |
252 | ||
253 | static const struct sysfs_ops mem_ops = { | |
254 | .show = mem_show, | |
255 | }; | |
256 | ||
257 | static struct kobj_type mem_type = { | |
5108d768 | 258 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
259 | .sysfs_ops = &mem_ops, |
260 | }; | |
261 | ||
262 | static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, | |
263 | char *buffer) | |
264 | { | |
265 | ssize_t ret; | |
bc0c75a3 | 266 | uint32_t i, j; |
5b5c4e40 EP |
267 | struct kfd_cache_properties *cache; |
268 | ||
269 | /* Making sure that the buffer is an empty string */ | |
270 | buffer[0] = 0; | |
271 | ||
272 | cache = container_of(attr, struct kfd_cache_properties, attr); | |
273 | sysfs_show_32bit_prop(buffer, "processor_id_low", | |
274 | cache->processor_id_low); | |
275 | sysfs_show_32bit_prop(buffer, "level", cache->cache_level); | |
276 | sysfs_show_32bit_prop(buffer, "size", cache->cache_size); | |
277 | sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size); | |
278 | sysfs_show_32bit_prop(buffer, "cache_lines_per_tag", | |
279 | cache->cachelines_per_tag); | |
280 | sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc); | |
281 | sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency); | |
282 | sysfs_show_32bit_prop(buffer, "type", cache->cache_type); | |
283 | snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer); | |
bc0c75a3 HK |
284 | for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++) |
285 | for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) { | |
286 | /* Check each bit */ | |
287 | if (cache->sibling_map[i] & (1 << j)) | |
288 | ret = snprintf(buffer, PAGE_SIZE, | |
289 | "%s%d%s", buffer, 1, ","); | |
290 | else | |
291 | ret = snprintf(buffer, PAGE_SIZE, | |
292 | "%s%d%s", buffer, 0, ","); | |
293 | } | |
294 | /* Replace the last "," with end of line */ | |
295 | *(buffer + strlen(buffer) - 1) = 0xA; | |
5b5c4e40 EP |
296 | return ret; |
297 | } | |
298 | ||
299 | static const struct sysfs_ops cache_ops = { | |
300 | .show = kfd_cache_show, | |
301 | }; | |
302 | ||
303 | static struct kobj_type cache_type = { | |
5108d768 | 304 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
305 | .sysfs_ops = &cache_ops, |
306 | }; | |
307 | ||
308 | static ssize_t node_show(struct kobject *kobj, struct attribute *attr, | |
309 | char *buffer) | |
310 | { | |
5b5c4e40 EP |
311 | struct kfd_topology_device *dev; |
312 | char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | |
313 | uint32_t i; | |
f7c826ad | 314 | uint32_t log_max_watch_addr; |
5b5c4e40 EP |
315 | |
316 | /* Making sure that the buffer is an empty string */ | |
317 | buffer[0] = 0; | |
318 | ||
319 | if (strcmp(attr->name, "gpu_id") == 0) { | |
320 | dev = container_of(attr, struct kfd_topology_device, | |
321 | attr_gpuid); | |
f7c826ad AS |
322 | return sysfs_show_32bit_val(buffer, dev->gpu_id); |
323 | } | |
324 | ||
325 | if (strcmp(attr->name, "name") == 0) { | |
5b5c4e40 EP |
326 | dev = container_of(attr, struct kfd_topology_device, |
327 | attr_name); | |
328 | for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) { | |
329 | public_name[i] = | |
330 | (char)dev->node_props.marketing_name[i]; | |
331 | if (dev->node_props.marketing_name[i] == 0) | |
332 | break; | |
333 | } | |
334 | public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0; | |
f7c826ad AS |
335 | return sysfs_show_str_val(buffer, public_name); |
336 | } | |
5b5c4e40 | 337 | |
f7c826ad AS |
338 | dev = container_of(attr, struct kfd_topology_device, |
339 | attr_props); | |
340 | sysfs_show_32bit_prop(buffer, "cpu_cores_count", | |
341 | dev->node_props.cpu_cores_count); | |
342 | sysfs_show_32bit_prop(buffer, "simd_count", | |
343 | dev->node_props.simd_count); | |
175b9263 FK |
344 | sysfs_show_32bit_prop(buffer, "mem_banks_count", |
345 | dev->node_props.mem_banks_count); | |
f7c826ad AS |
346 | sysfs_show_32bit_prop(buffer, "caches_count", |
347 | dev->node_props.caches_count); | |
348 | sysfs_show_32bit_prop(buffer, "io_links_count", | |
349 | dev->node_props.io_links_count); | |
350 | sysfs_show_32bit_prop(buffer, "cpu_core_id_base", | |
351 | dev->node_props.cpu_core_id_base); | |
352 | sysfs_show_32bit_prop(buffer, "simd_id_base", | |
353 | dev->node_props.simd_id_base); | |
f7c826ad AS |
354 | sysfs_show_32bit_prop(buffer, "max_waves_per_simd", |
355 | dev->node_props.max_waves_per_simd); | |
356 | sysfs_show_32bit_prop(buffer, "lds_size_in_kb", | |
357 | dev->node_props.lds_size_in_kb); | |
358 | sysfs_show_32bit_prop(buffer, "gds_size_in_kb", | |
359 | dev->node_props.gds_size_in_kb); | |
360 | sysfs_show_32bit_prop(buffer, "wave_front_size", | |
361 | dev->node_props.wave_front_size); | |
362 | sysfs_show_32bit_prop(buffer, "array_count", | |
363 | dev->node_props.array_count); | |
364 | sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine", | |
365 | dev->node_props.simd_arrays_per_engine); | |
366 | sysfs_show_32bit_prop(buffer, "cu_per_simd_array", | |
367 | dev->node_props.cu_per_simd_array); | |
368 | sysfs_show_32bit_prop(buffer, "simd_per_cu", | |
369 | dev->node_props.simd_per_cu); | |
370 | sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu", | |
371 | dev->node_props.max_slots_scratch_cu); | |
372 | sysfs_show_32bit_prop(buffer, "vendor_id", | |
373 | dev->node_props.vendor_id); | |
374 | sysfs_show_32bit_prop(buffer, "device_id", | |
375 | dev->node_props.device_id); | |
376 | sysfs_show_32bit_prop(buffer, "location_id", | |
377 | dev->node_props.location_id); | |
378 | ||
379 | if (dev->gpu) { | |
380 | log_max_watch_addr = | |
381 | __ilog2_u32(dev->gpu->device_info->num_of_watch_points); | |
382 | ||
383 | if (log_max_watch_addr) { | |
384 | dev->node_props.capability |= | |
385 | HSA_CAP_WATCH_POINTS_SUPPORTED; | |
386 | ||
387 | dev->node_props.capability |= | |
388 | ((log_max_watch_addr << | |
389 | HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & | |
390 | HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); | |
5b5c4e40 EP |
391 | } |
392 | ||
f7c826ad | 393 | sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute", |
cea405b1 | 394 | dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz( |
f7c826ad | 395 | dev->gpu->kgd)); |
42e08c78 | 396 | |
f7c826ad | 397 | sysfs_show_64bit_prop(buffer, "local_mem_size", |
42e08c78 | 398 | (unsigned long long int) 0); |
f7c826ad AS |
399 | |
400 | sysfs_show_32bit_prop(buffer, "fw_version", | |
cea405b1 | 401 | dev->gpu->kfd2kgd->get_fw_version( |
f7c826ad AS |
402 | dev->gpu->kgd, |
403 | KGD_ENGINE_MEC1)); | |
826f5de8 AS |
404 | sysfs_show_32bit_prop(buffer, "capability", |
405 | dev->node_props.capability); | |
5b5c4e40 EP |
406 | } |
407 | ||
f7c826ad AS |
408 | return sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute", |
409 | cpufreq_quick_get_max(0)/1000); | |
5b5c4e40 EP |
410 | } |
411 | ||
412 | static const struct sysfs_ops node_ops = { | |
413 | .show = node_show, | |
414 | }; | |
415 | ||
416 | static struct kobj_type node_type = { | |
5108d768 | 417 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
418 | .sysfs_ops = &node_ops, |
419 | }; | |
420 | ||
421 | static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) | |
422 | { | |
423 | sysfs_remove_file(kobj, attr); | |
424 | kobject_del(kobj); | |
425 | kobject_put(kobj); | |
426 | } | |
427 | ||
428 | static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) | |
429 | { | |
430 | struct kfd_iolink_properties *iolink; | |
431 | struct kfd_cache_properties *cache; | |
432 | struct kfd_mem_properties *mem; | |
433 | ||
5b5c4e40 EP |
434 | if (dev->kobj_iolink) { |
435 | list_for_each_entry(iolink, &dev->io_link_props, list) | |
436 | if (iolink->kobj) { | |
437 | kfd_remove_sysfs_file(iolink->kobj, | |
438 | &iolink->attr); | |
16b9201c | 439 | iolink->kobj = NULL; |
5b5c4e40 EP |
440 | } |
441 | kobject_del(dev->kobj_iolink); | |
442 | kobject_put(dev->kobj_iolink); | |
16b9201c | 443 | dev->kobj_iolink = NULL; |
5b5c4e40 EP |
444 | } |
445 | ||
446 | if (dev->kobj_cache) { | |
447 | list_for_each_entry(cache, &dev->cache_props, list) | |
448 | if (cache->kobj) { | |
449 | kfd_remove_sysfs_file(cache->kobj, | |
450 | &cache->attr); | |
16b9201c | 451 | cache->kobj = NULL; |
5b5c4e40 EP |
452 | } |
453 | kobject_del(dev->kobj_cache); | |
454 | kobject_put(dev->kobj_cache); | |
16b9201c | 455 | dev->kobj_cache = NULL; |
5b5c4e40 EP |
456 | } |
457 | ||
458 | if (dev->kobj_mem) { | |
459 | list_for_each_entry(mem, &dev->mem_props, list) | |
460 | if (mem->kobj) { | |
461 | kfd_remove_sysfs_file(mem->kobj, &mem->attr); | |
16b9201c | 462 | mem->kobj = NULL; |
5b5c4e40 EP |
463 | } |
464 | kobject_del(dev->kobj_mem); | |
465 | kobject_put(dev->kobj_mem); | |
16b9201c | 466 | dev->kobj_mem = NULL; |
5b5c4e40 EP |
467 | } |
468 | ||
469 | if (dev->kobj_node) { | |
470 | sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); | |
471 | sysfs_remove_file(dev->kobj_node, &dev->attr_name); | |
472 | sysfs_remove_file(dev->kobj_node, &dev->attr_props); | |
473 | kobject_del(dev->kobj_node); | |
474 | kobject_put(dev->kobj_node); | |
16b9201c | 475 | dev->kobj_node = NULL; |
5b5c4e40 EP |
476 | } |
477 | } | |
478 | ||
479 | static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, | |
480 | uint32_t id) | |
481 | { | |
482 | struct kfd_iolink_properties *iolink; | |
483 | struct kfd_cache_properties *cache; | |
484 | struct kfd_mem_properties *mem; | |
485 | int ret; | |
486 | uint32_t i; | |
487 | ||
32fa8219 FK |
488 | if (WARN_ON(dev->kobj_node)) |
489 | return -EEXIST; | |
490 | ||
5b5c4e40 EP |
491 | /* |
492 | * Creating the sysfs folders | |
493 | */ | |
5b5c4e40 EP |
494 | dev->kobj_node = kfd_alloc_struct(dev->kobj_node); |
495 | if (!dev->kobj_node) | |
496 | return -ENOMEM; | |
497 | ||
498 | ret = kobject_init_and_add(dev->kobj_node, &node_type, | |
499 | sys_props.kobj_nodes, "%d", id); | |
500 | if (ret < 0) | |
501 | return ret; | |
502 | ||
503 | dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); | |
504 | if (!dev->kobj_mem) | |
505 | return -ENOMEM; | |
506 | ||
507 | dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); | |
508 | if (!dev->kobj_cache) | |
509 | return -ENOMEM; | |
510 | ||
511 | dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); | |
512 | if (!dev->kobj_iolink) | |
513 | return -ENOMEM; | |
514 | ||
515 | /* | |
516 | * Creating sysfs files for node properties | |
517 | */ | |
518 | dev->attr_gpuid.name = "gpu_id"; | |
519 | dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; | |
520 | sysfs_attr_init(&dev->attr_gpuid); | |
521 | dev->attr_name.name = "name"; | |
522 | dev->attr_name.mode = KFD_SYSFS_FILE_MODE; | |
523 | sysfs_attr_init(&dev->attr_name); | |
524 | dev->attr_props.name = "properties"; | |
525 | dev->attr_props.mode = KFD_SYSFS_FILE_MODE; | |
526 | sysfs_attr_init(&dev->attr_props); | |
527 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); | |
528 | if (ret < 0) | |
529 | return ret; | |
530 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); | |
531 | if (ret < 0) | |
532 | return ret; | |
533 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); | |
534 | if (ret < 0) | |
535 | return ret; | |
536 | ||
537 | i = 0; | |
538 | list_for_each_entry(mem, &dev->mem_props, list) { | |
539 | mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
540 | if (!mem->kobj) | |
541 | return -ENOMEM; | |
542 | ret = kobject_init_and_add(mem->kobj, &mem_type, | |
543 | dev->kobj_mem, "%d", i); | |
544 | if (ret < 0) | |
545 | return ret; | |
546 | ||
547 | mem->attr.name = "properties"; | |
548 | mem->attr.mode = KFD_SYSFS_FILE_MODE; | |
549 | sysfs_attr_init(&mem->attr); | |
550 | ret = sysfs_create_file(mem->kobj, &mem->attr); | |
551 | if (ret < 0) | |
552 | return ret; | |
553 | i++; | |
554 | } | |
555 | ||
556 | i = 0; | |
557 | list_for_each_entry(cache, &dev->cache_props, list) { | |
558 | cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
559 | if (!cache->kobj) | |
560 | return -ENOMEM; | |
561 | ret = kobject_init_and_add(cache->kobj, &cache_type, | |
562 | dev->kobj_cache, "%d", i); | |
563 | if (ret < 0) | |
564 | return ret; | |
565 | ||
566 | cache->attr.name = "properties"; | |
567 | cache->attr.mode = KFD_SYSFS_FILE_MODE; | |
568 | sysfs_attr_init(&cache->attr); | |
569 | ret = sysfs_create_file(cache->kobj, &cache->attr); | |
570 | if (ret < 0) | |
571 | return ret; | |
572 | i++; | |
573 | } | |
574 | ||
575 | i = 0; | |
576 | list_for_each_entry(iolink, &dev->io_link_props, list) { | |
577 | iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
578 | if (!iolink->kobj) | |
579 | return -ENOMEM; | |
580 | ret = kobject_init_and_add(iolink->kobj, &iolink_type, | |
581 | dev->kobj_iolink, "%d", i); | |
582 | if (ret < 0) | |
583 | return ret; | |
584 | ||
585 | iolink->attr.name = "properties"; | |
586 | iolink->attr.mode = KFD_SYSFS_FILE_MODE; | |
587 | sysfs_attr_init(&iolink->attr); | |
588 | ret = sysfs_create_file(iolink->kobj, &iolink->attr); | |
589 | if (ret < 0) | |
590 | return ret; | |
591 | i++; | |
592 | } | |
593 | ||
594 | return 0; | |
595 | } | |
596 | ||
597 | static int kfd_build_sysfs_node_tree(void) | |
598 | { | |
599 | struct kfd_topology_device *dev; | |
600 | int ret; | |
601 | uint32_t i = 0; | |
602 | ||
603 | list_for_each_entry(dev, &topology_device_list, list) { | |
8dfead6c | 604 | ret = kfd_build_sysfs_node_entry(dev, i); |
5b5c4e40 EP |
605 | if (ret < 0) |
606 | return ret; | |
607 | i++; | |
608 | } | |
609 | ||
610 | return 0; | |
611 | } | |
612 | ||
613 | static void kfd_remove_sysfs_node_tree(void) | |
614 | { | |
615 | struct kfd_topology_device *dev; | |
616 | ||
617 | list_for_each_entry(dev, &topology_device_list, list) | |
618 | kfd_remove_sysfs_node_entry(dev); | |
619 | } | |
620 | ||
621 | static int kfd_topology_update_sysfs(void) | |
622 | { | |
623 | int ret; | |
624 | ||
625 | pr_info("Creating topology SYSFS entries\n"); | |
4eacc26b | 626 | if (!sys_props.kobj_topology) { |
5b5c4e40 EP |
627 | sys_props.kobj_topology = |
628 | kfd_alloc_struct(sys_props.kobj_topology); | |
629 | if (!sys_props.kobj_topology) | |
630 | return -ENOMEM; | |
631 | ||
632 | ret = kobject_init_and_add(sys_props.kobj_topology, | |
633 | &sysprops_type, &kfd_device->kobj, | |
634 | "topology"); | |
635 | if (ret < 0) | |
636 | return ret; | |
637 | ||
638 | sys_props.kobj_nodes = kobject_create_and_add("nodes", | |
639 | sys_props.kobj_topology); | |
640 | if (!sys_props.kobj_nodes) | |
641 | return -ENOMEM; | |
642 | ||
643 | sys_props.attr_genid.name = "generation_id"; | |
644 | sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; | |
645 | sysfs_attr_init(&sys_props.attr_genid); | |
646 | ret = sysfs_create_file(sys_props.kobj_topology, | |
647 | &sys_props.attr_genid); | |
648 | if (ret < 0) | |
649 | return ret; | |
650 | ||
651 | sys_props.attr_props.name = "system_properties"; | |
652 | sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; | |
653 | sysfs_attr_init(&sys_props.attr_props); | |
654 | ret = sysfs_create_file(sys_props.kobj_topology, | |
655 | &sys_props.attr_props); | |
656 | if (ret < 0) | |
657 | return ret; | |
658 | } | |
659 | ||
660 | kfd_remove_sysfs_node_tree(); | |
661 | ||
662 | return kfd_build_sysfs_node_tree(); | |
663 | } | |
664 | ||
665 | static void kfd_topology_release_sysfs(void) | |
666 | { | |
667 | kfd_remove_sysfs_node_tree(); | |
668 | if (sys_props.kobj_topology) { | |
669 | sysfs_remove_file(sys_props.kobj_topology, | |
670 | &sys_props.attr_genid); | |
671 | sysfs_remove_file(sys_props.kobj_topology, | |
672 | &sys_props.attr_props); | |
673 | if (sys_props.kobj_nodes) { | |
674 | kobject_del(sys_props.kobj_nodes); | |
675 | kobject_put(sys_props.kobj_nodes); | |
16b9201c | 676 | sys_props.kobj_nodes = NULL; |
5b5c4e40 EP |
677 | } |
678 | kobject_del(sys_props.kobj_topology); | |
679 | kobject_put(sys_props.kobj_topology); | |
16b9201c | 680 | sys_props.kobj_topology = NULL; |
5b5c4e40 EP |
681 | } |
682 | } | |
683 | ||
4f449311 HK |
684 | /* Called with write topology_lock acquired */ |
685 | static void kfd_topology_update_device_list(struct list_head *temp_list, | |
686 | struct list_head *master_list) | |
687 | { | |
688 | while (!list_empty(temp_list)) { | |
689 | list_move_tail(temp_list->next, master_list); | |
690 | sys_props.num_devices++; | |
691 | } | |
692 | } | |
693 | ||
5b5c4e40 EP |
694 | int kfd_topology_init(void) |
695 | { | |
16b9201c | 696 | void *crat_image = NULL; |
5b5c4e40 EP |
697 | size_t image_size = 0; |
698 | int ret; | |
4f449311 | 699 | struct list_head temp_topology_device_list; |
5b5c4e40 | 700 | |
4f449311 HK |
701 | /* topology_device_list - Master list of all topology devices |
702 | * temp_topology_device_list - temporary list created while parsing CRAT | |
703 | * or VCRAT. Once parsing is complete the contents of list is moved to | |
704 | * topology_device_list | |
5b5c4e40 | 705 | */ |
4f449311 HK |
706 | |
707 | /* Initialize the head for the both the lists */ | |
5b5c4e40 | 708 | INIT_LIST_HEAD(&topology_device_list); |
4f449311 | 709 | INIT_LIST_HEAD(&temp_topology_device_list); |
5b5c4e40 | 710 | init_rwsem(&topology_lock); |
5b5c4e40 EP |
711 | |
712 | memset(&sys_props, 0, sizeof(sys_props)); | |
713 | ||
714 | /* | |
715 | * Get the CRAT image from the ACPI | |
716 | */ | |
8e05247d HK |
717 | ret = kfd_create_crat_image_acpi(&crat_image, &image_size); |
718 | if (!ret) { | |
4f449311 HK |
719 | ret = kfd_parse_crat_table(crat_image, |
720 | &temp_topology_device_list, 0); | |
8e05247d | 721 | if (ret) |
5b5c4e40 | 722 | goto err; |
5b5c4e40 | 723 | } else if (ret == -ENODATA) { |
8e05247d | 724 | /* TODO: Create fake CRAT table */ |
5b5c4e40 | 725 | ret = 0; |
8e05247d | 726 | goto err; |
5b5c4e40 EP |
727 | } else { |
728 | pr_err("Couldn't get CRAT table size from ACPI\n"); | |
8e05247d | 729 | goto err; |
5b5c4e40 EP |
730 | } |
731 | ||
8e05247d | 732 | down_write(&topology_lock); |
4f449311 HK |
733 | kfd_topology_update_device_list(&temp_topology_device_list, |
734 | &topology_device_list); | |
8e05247d HK |
735 | ret = kfd_topology_update_sysfs(); |
736 | up_write(&topology_lock); | |
737 | ||
4f449311 HK |
738 | if (!ret) { |
739 | sys_props.generation_count++; | |
8e05247d | 740 | pr_info("Finished initializing topology\n"); |
4f449311 | 741 | } else |
8e05247d HK |
742 | pr_err("Failed to update topology in sysfs ret=%d\n", ret); |
743 | ||
5b5c4e40 | 744 | err: |
8e05247d | 745 | kfd_destroy_crat_image(crat_image); |
5b5c4e40 EP |
746 | return ret; |
747 | } | |
748 | ||
749 | void kfd_topology_shutdown(void) | |
750 | { | |
4f449311 | 751 | down_write(&topology_lock); |
5b5c4e40 EP |
752 | kfd_topology_release_sysfs(); |
753 | kfd_release_live_view(); | |
4f449311 | 754 | up_write(&topology_lock); |
5b5c4e40 EP |
755 | } |
756 | ||
757 | static void kfd_debug_print_topology(void) | |
758 | { | |
759 | struct kfd_topology_device *dev; | |
760 | uint32_t i = 0; | |
761 | ||
762 | pr_info("DEBUG PRINT OF TOPOLOGY:"); | |
763 | list_for_each_entry(dev, &topology_device_list, list) { | |
764 | pr_info("Node: %d\n", i); | |
765 | pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no")); | |
766 | pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count); | |
8e05247d | 767 | pr_info("\tSIMD count: %d\n", dev->node_props.simd_count); |
5b5c4e40 EP |
768 | i++; |
769 | } | |
770 | } | |
771 | ||
772 | static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu) | |
773 | { | |
774 | uint32_t hashout; | |
775 | uint32_t buf[7]; | |
585f0e6c | 776 | uint64_t local_mem_size; |
5b5c4e40 | 777 | int i; |
0504cccf | 778 | struct kfd_local_mem_info local_mem_info; |
5b5c4e40 EP |
779 | |
780 | if (!gpu) | |
781 | return 0; | |
782 | ||
0504cccf HK |
783 | gpu->kfd2kgd->get_local_mem_info(gpu->kgd, &local_mem_info); |
784 | ||
785 | local_mem_size = local_mem_info.local_mem_size_private + | |
786 | local_mem_info.local_mem_size_public; | |
585f0e6c | 787 | |
5b5c4e40 EP |
788 | buf[0] = gpu->pdev->devfn; |
789 | buf[1] = gpu->pdev->subsystem_vendor; | |
790 | buf[2] = gpu->pdev->subsystem_device; | |
791 | buf[3] = gpu->pdev->device; | |
792 | buf[4] = gpu->pdev->bus->number; | |
585f0e6c EC |
793 | buf[5] = lower_32_bits(local_mem_size); |
794 | buf[6] = upper_32_bits(local_mem_size); | |
5b5c4e40 EP |
795 | |
796 | for (i = 0, hashout = 0; i < 7; i++) | |
797 | hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); | |
798 | ||
799 | return hashout; | |
800 | } | |
801 | ||
802 | static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) | |
803 | { | |
804 | struct kfd_topology_device *dev; | |
16b9201c | 805 | struct kfd_topology_device *out_dev = NULL; |
5b5c4e40 | 806 | |
5b5c4e40 | 807 | list_for_each_entry(dev, &topology_device_list, list) |
4eacc26b | 808 | if (!dev->gpu && (dev->node_props.simd_count > 0)) { |
5b5c4e40 EP |
809 | dev->gpu = gpu; |
810 | out_dev = dev; | |
811 | break; | |
812 | } | |
813 | ||
814 | return out_dev; | |
815 | } | |
816 | ||
817 | static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) | |
818 | { | |
819 | /* | |
820 | * TODO: Generate an event for thunk about the arrival/removal | |
821 | * of the GPU | |
822 | */ | |
823 | } | |
824 | ||
825 | int kfd_topology_add_device(struct kfd_dev *gpu) | |
826 | { | |
827 | uint32_t gpu_id; | |
828 | struct kfd_topology_device *dev; | |
f7ce2fad | 829 | struct kfd_cu_info cu_info; |
4f449311 HK |
830 | int res = 0; |
831 | struct list_head temp_topology_device_list; | |
832 | ||
833 | INIT_LIST_HEAD(&temp_topology_device_list); | |
5b5c4e40 | 834 | |
5b5c4e40 EP |
835 | gpu_id = kfd_generate_gpu_id(gpu); |
836 | ||
79775b62 | 837 | pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id); |
5b5c4e40 | 838 | |
5b5c4e40 EP |
839 | /* |
840 | * Try to assign the GPU to existing topology device (generated from | |
841 | * CRAT table | |
842 | */ | |
843 | dev = kfd_assign_gpu(gpu); | |
844 | if (!dev) { | |
845 | pr_info("GPU was not found in the current topology. Extending.\n"); | |
846 | kfd_debug_print_topology(); | |
4f449311 | 847 | dev = kfd_create_topology_device(&temp_topology_device_list); |
5b5c4e40 EP |
848 | if (!dev) { |
849 | res = -ENOMEM; | |
850 | goto err; | |
851 | } | |
4f449311 | 852 | |
5b5c4e40 EP |
853 | dev->gpu = gpu; |
854 | ||
855 | /* | |
856 | * TODO: Make a call to retrieve topology information from the | |
857 | * GPU vBIOS | |
858 | */ | |
859 | ||
4f449311 HK |
860 | down_write(&topology_lock); |
861 | kfd_topology_update_device_list(&temp_topology_device_list, | |
862 | &topology_device_list); | |
863 | ||
8eabaf54 KR |
864 | /* Update the SYSFS tree, since we added another topology |
865 | * device | |
5b5c4e40 EP |
866 | */ |
867 | if (kfd_topology_update_sysfs() < 0) | |
868 | kfd_topology_release_sysfs(); | |
869 | ||
4f449311 HK |
870 | up_write(&topology_lock); |
871 | ||
5b5c4e40 EP |
872 | } |
873 | ||
874 | dev->gpu_id = gpu_id; | |
875 | gpu->id = gpu_id; | |
f7ce2fad FC |
876 | dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info); |
877 | dev->node_props.simd_count = dev->node_props.simd_per_cu * | |
878 | cu_info.cu_active_number; | |
5b5c4e40 EP |
879 | dev->node_props.vendor_id = gpu->pdev->vendor; |
880 | dev->node_props.device_id = gpu->pdev->device; | |
d63f0ba2 HK |
881 | dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number, |
882 | gpu->pdev->devfn); | |
5b5c4e40 EP |
883 | /* |
884 | * TODO: Retrieve max engine clock values from KGD | |
885 | */ | |
886 | ||
7639a8c4 BG |
887 | if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) { |
888 | dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE; | |
42aa8793 | 889 | pr_debug("Adding doorbell packet type capability\n"); |
7639a8c4 BG |
890 | } |
891 | ||
4f449311 | 892 | if (!res) |
5b5c4e40 | 893 | kfd_notify_gpu_change(gpu_id, 1); |
4f449311 | 894 | err: |
5b5c4e40 EP |
895 | return res; |
896 | } | |
897 | ||
898 | int kfd_topology_remove_device(struct kfd_dev *gpu) | |
899 | { | |
4f449311 | 900 | struct kfd_topology_device *dev, *tmp; |
5b5c4e40 EP |
901 | uint32_t gpu_id; |
902 | int res = -ENODEV; | |
903 | ||
5b5c4e40 EP |
904 | down_write(&topology_lock); |
905 | ||
4f449311 | 906 | list_for_each_entry_safe(dev, tmp, &topology_device_list, list) |
5b5c4e40 EP |
907 | if (dev->gpu == gpu) { |
908 | gpu_id = dev->gpu_id; | |
909 | kfd_remove_sysfs_node_entry(dev); | |
910 | kfd_release_topology_device(dev); | |
4f449311 | 911 | sys_props.num_devices--; |
5b5c4e40 EP |
912 | res = 0; |
913 | if (kfd_topology_update_sysfs() < 0) | |
914 | kfd_topology_release_sysfs(); | |
915 | break; | |
916 | } | |
917 | ||
918 | up_write(&topology_lock); | |
919 | ||
174de876 | 920 | if (!res) |
5b5c4e40 EP |
921 | kfd_notify_gpu_change(gpu_id, 0); |
922 | ||
923 | return res; | |
924 | } | |
925 | ||
6d82eb0e HK |
926 | /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD |
927 | * topology. If GPU device is found @idx, then valid kfd_dev pointer is | |
928 | * returned through @kdev | |
929 | * Return - 0: On success (@kdev will be NULL for non GPU nodes) | |
930 | * -1: If end of list | |
5b5c4e40 | 931 | */ |
6d82eb0e | 932 | int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev) |
5b5c4e40 EP |
933 | { |
934 | ||
935 | struct kfd_topology_device *top_dev; | |
5b5c4e40 EP |
936 | uint8_t device_idx = 0; |
937 | ||
6d82eb0e | 938 | *kdev = NULL; |
5b5c4e40 EP |
939 | down_read(&topology_lock); |
940 | ||
941 | list_for_each_entry(top_dev, &topology_device_list, list) { | |
942 | if (device_idx == idx) { | |
6d82eb0e HK |
943 | *kdev = top_dev->gpu; |
944 | up_read(&topology_lock); | |
945 | return 0; | |
5b5c4e40 EP |
946 | } |
947 | ||
948 | device_idx++; | |
949 | } | |
950 | ||
951 | up_read(&topology_lock); | |
952 | ||
6d82eb0e | 953 | return -1; |
5b5c4e40 EP |
954 | |
955 | } | |
851a645e FK |
956 | |
957 | #if defined(CONFIG_DEBUG_FS) | |
958 | ||
959 | int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) | |
960 | { | |
961 | struct kfd_topology_device *dev; | |
962 | unsigned int i = 0; | |
963 | int r = 0; | |
964 | ||
965 | down_read(&topology_lock); | |
966 | ||
967 | list_for_each_entry(dev, &topology_device_list, list) { | |
968 | if (!dev->gpu) { | |
969 | i++; | |
970 | continue; | |
971 | } | |
972 | ||
973 | seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); | |
974 | r = dqm_debugfs_hqds(m, dev->gpu->dqm); | |
975 | if (r) | |
976 | break; | |
977 | } | |
978 | ||
979 | up_read(&topology_lock); | |
980 | ||
981 | return r; | |
982 | } | |
983 | ||
984 | int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) | |
985 | { | |
986 | struct kfd_topology_device *dev; | |
987 | unsigned int i = 0; | |
988 | int r = 0; | |
989 | ||
990 | down_read(&topology_lock); | |
991 | ||
992 | list_for_each_entry(dev, &topology_device_list, list) { | |
993 | if (!dev->gpu) { | |
994 | i++; | |
995 | continue; | |
996 | } | |
997 | ||
998 | seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); | |
999 | r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets); | |
1000 | if (r) | |
1001 | break; | |
1002 | } | |
1003 | ||
1004 | up_read(&topology_lock); | |
1005 | ||
1006 | return r; | |
1007 | } | |
1008 | ||
1009 | #endif |