Commit | Line | Data |
---|---|---|
5b5c4e40 EP |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/types.h> | |
24 | #include <linux/kernel.h> | |
25 | #include <linux/pci.h> | |
26 | #include <linux/errno.h> | |
27 | #include <linux/acpi.h> | |
28 | #include <linux/hash.h> | |
29 | #include <linux/cpufreq.h> | |
f7c826ad | 30 | #include <linux/log2.h> |
5b5c4e40 EP |
31 | |
32 | #include "kfd_priv.h" | |
33 | #include "kfd_crat.h" | |
34 | #include "kfd_topology.h" | |
851a645e | 35 | #include "kfd_device_queue_manager.h" |
5b5c4e40 | 36 | |
4f449311 HK |
37 | /* topology_device_list - Master list of all topology devices */ |
38 | static struct list_head topology_device_list; | |
174de876 | 39 | struct kfd_system_properties sys_props; |
5b5c4e40 EP |
40 | |
41 | static DECLARE_RWSEM(topology_lock); | |
42 | ||
43 | struct kfd_dev *kfd_device_by_id(uint32_t gpu_id) | |
44 | { | |
45 | struct kfd_topology_device *top_dev; | |
46 | struct kfd_dev *device = NULL; | |
47 | ||
48 | down_read(&topology_lock); | |
49 | ||
50 | list_for_each_entry(top_dev, &topology_device_list, list) | |
51 | if (top_dev->gpu_id == gpu_id) { | |
52 | device = top_dev->gpu; | |
53 | break; | |
54 | } | |
55 | ||
56 | up_read(&topology_lock); | |
57 | ||
58 | return device; | |
59 | } | |
60 | ||
61 | struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev) | |
62 | { | |
63 | struct kfd_topology_device *top_dev; | |
64 | struct kfd_dev *device = NULL; | |
65 | ||
66 | down_read(&topology_lock); | |
67 | ||
68 | list_for_each_entry(top_dev, &topology_device_list, list) | |
69 | if (top_dev->gpu->pdev == pdev) { | |
70 | device = top_dev->gpu; | |
71 | break; | |
72 | } | |
73 | ||
74 | up_read(&topology_lock); | |
75 | ||
76 | return device; | |
77 | } | |
78 | ||
5b5c4e40 EP |
79 | static void kfd_release_topology_device(struct kfd_topology_device *dev) |
80 | { | |
81 | struct kfd_mem_properties *mem; | |
82 | struct kfd_cache_properties *cache; | |
83 | struct kfd_iolink_properties *iolink; | |
84 | ||
5b5c4e40 EP |
85 | list_del(&dev->list); |
86 | ||
87 | while (dev->mem_props.next != &dev->mem_props) { | |
88 | mem = container_of(dev->mem_props.next, | |
89 | struct kfd_mem_properties, list); | |
90 | list_del(&mem->list); | |
91 | kfree(mem); | |
92 | } | |
93 | ||
94 | while (dev->cache_props.next != &dev->cache_props) { | |
95 | cache = container_of(dev->cache_props.next, | |
96 | struct kfd_cache_properties, list); | |
97 | list_del(&cache->list); | |
98 | kfree(cache); | |
99 | } | |
100 | ||
101 | while (dev->io_link_props.next != &dev->io_link_props) { | |
102 | iolink = container_of(dev->io_link_props.next, | |
103 | struct kfd_iolink_properties, list); | |
104 | list_del(&iolink->list); | |
105 | kfree(iolink); | |
106 | } | |
107 | ||
108 | kfree(dev); | |
5b5c4e40 EP |
109 | } |
110 | ||
4f449311 | 111 | void kfd_release_topology_device_list(struct list_head *device_list) |
5b5c4e40 EP |
112 | { |
113 | struct kfd_topology_device *dev; | |
114 | ||
4f449311 HK |
115 | while (!list_empty(device_list)) { |
116 | dev = list_first_entry(device_list, | |
117 | struct kfd_topology_device, list); | |
5b5c4e40 | 118 | kfd_release_topology_device(dev); |
4f449311 | 119 | } |
5b5c4e40 EP |
120 | } |
121 | ||
4f449311 HK |
122 | static void kfd_release_live_view(void) |
123 | { | |
124 | kfd_release_topology_device_list(&topology_device_list); | |
5b5c4e40 EP |
125 | memset(&sys_props, 0, sizeof(sys_props)); |
126 | } | |
127 | ||
4f449311 HK |
128 | struct kfd_topology_device *kfd_create_topology_device( |
129 | struct list_head *device_list) | |
5b5c4e40 EP |
130 | { |
131 | struct kfd_topology_device *dev; | |
132 | ||
133 | dev = kfd_alloc_struct(dev); | |
4eacc26b | 134 | if (!dev) { |
5b5c4e40 | 135 | pr_err("No memory to allocate a topology device"); |
16b9201c | 136 | return NULL; |
5b5c4e40 EP |
137 | } |
138 | ||
139 | INIT_LIST_HEAD(&dev->mem_props); | |
140 | INIT_LIST_HEAD(&dev->cache_props); | |
141 | INIT_LIST_HEAD(&dev->io_link_props); | |
142 | ||
4f449311 | 143 | list_add_tail(&dev->list, device_list); |
5b5c4e40 EP |
144 | |
145 | return dev; | |
16b9201c | 146 | } |
5b5c4e40 | 147 | |
5b5c4e40 EP |
148 | |
149 | #define sysfs_show_gen_prop(buffer, fmt, ...) \ | |
150 | snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__) | |
151 | #define sysfs_show_32bit_prop(buffer, name, value) \ | |
152 | sysfs_show_gen_prop(buffer, "%s %u\n", name, value) | |
153 | #define sysfs_show_64bit_prop(buffer, name, value) \ | |
154 | sysfs_show_gen_prop(buffer, "%s %llu\n", name, value) | |
155 | #define sysfs_show_32bit_val(buffer, value) \ | |
156 | sysfs_show_gen_prop(buffer, "%u\n", value) | |
157 | #define sysfs_show_str_val(buffer, value) \ | |
158 | sysfs_show_gen_prop(buffer, "%s\n", value) | |
159 | ||
160 | static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr, | |
161 | char *buffer) | |
162 | { | |
163 | ssize_t ret; | |
164 | ||
165 | /* Making sure that the buffer is an empty string */ | |
166 | buffer[0] = 0; | |
167 | ||
168 | if (attr == &sys_props.attr_genid) { | |
169 | ret = sysfs_show_32bit_val(buffer, sys_props.generation_count); | |
170 | } else if (attr == &sys_props.attr_props) { | |
171 | sysfs_show_64bit_prop(buffer, "platform_oem", | |
172 | sys_props.platform_oem); | |
173 | sysfs_show_64bit_prop(buffer, "platform_id", | |
174 | sys_props.platform_id); | |
175 | ret = sysfs_show_64bit_prop(buffer, "platform_rev", | |
176 | sys_props.platform_rev); | |
177 | } else { | |
178 | ret = -EINVAL; | |
179 | } | |
180 | ||
181 | return ret; | |
182 | } | |
183 | ||
5108d768 YZ |
184 | static void kfd_topology_kobj_release(struct kobject *kobj) |
185 | { | |
186 | kfree(kobj); | |
187 | } | |
188 | ||
5b5c4e40 EP |
189 | static const struct sysfs_ops sysprops_ops = { |
190 | .show = sysprops_show, | |
191 | }; | |
192 | ||
193 | static struct kobj_type sysprops_type = { | |
5108d768 | 194 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
195 | .sysfs_ops = &sysprops_ops, |
196 | }; | |
197 | ||
198 | static ssize_t iolink_show(struct kobject *kobj, struct attribute *attr, | |
199 | char *buffer) | |
200 | { | |
201 | ssize_t ret; | |
202 | struct kfd_iolink_properties *iolink; | |
203 | ||
204 | /* Making sure that the buffer is an empty string */ | |
205 | buffer[0] = 0; | |
206 | ||
207 | iolink = container_of(attr, struct kfd_iolink_properties, attr); | |
208 | sysfs_show_32bit_prop(buffer, "type", iolink->iolink_type); | |
209 | sysfs_show_32bit_prop(buffer, "version_major", iolink->ver_maj); | |
210 | sysfs_show_32bit_prop(buffer, "version_minor", iolink->ver_min); | |
211 | sysfs_show_32bit_prop(buffer, "node_from", iolink->node_from); | |
212 | sysfs_show_32bit_prop(buffer, "node_to", iolink->node_to); | |
213 | sysfs_show_32bit_prop(buffer, "weight", iolink->weight); | |
214 | sysfs_show_32bit_prop(buffer, "min_latency", iolink->min_latency); | |
215 | sysfs_show_32bit_prop(buffer, "max_latency", iolink->max_latency); | |
216 | sysfs_show_32bit_prop(buffer, "min_bandwidth", iolink->min_bandwidth); | |
217 | sysfs_show_32bit_prop(buffer, "max_bandwidth", iolink->max_bandwidth); | |
218 | sysfs_show_32bit_prop(buffer, "recommended_transfer_size", | |
219 | iolink->rec_transfer_size); | |
220 | ret = sysfs_show_32bit_prop(buffer, "flags", iolink->flags); | |
221 | ||
222 | return ret; | |
223 | } | |
224 | ||
225 | static const struct sysfs_ops iolink_ops = { | |
226 | .show = iolink_show, | |
227 | }; | |
228 | ||
229 | static struct kobj_type iolink_type = { | |
5108d768 | 230 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
231 | .sysfs_ops = &iolink_ops, |
232 | }; | |
233 | ||
234 | static ssize_t mem_show(struct kobject *kobj, struct attribute *attr, | |
235 | char *buffer) | |
236 | { | |
237 | ssize_t ret; | |
238 | struct kfd_mem_properties *mem; | |
239 | ||
240 | /* Making sure that the buffer is an empty string */ | |
241 | buffer[0] = 0; | |
242 | ||
243 | mem = container_of(attr, struct kfd_mem_properties, attr); | |
244 | sysfs_show_32bit_prop(buffer, "heap_type", mem->heap_type); | |
245 | sysfs_show_64bit_prop(buffer, "size_in_bytes", mem->size_in_bytes); | |
246 | sysfs_show_32bit_prop(buffer, "flags", mem->flags); | |
247 | sysfs_show_32bit_prop(buffer, "width", mem->width); | |
248 | ret = sysfs_show_32bit_prop(buffer, "mem_clk_max", mem->mem_clk_max); | |
249 | ||
250 | return ret; | |
251 | } | |
252 | ||
253 | static const struct sysfs_ops mem_ops = { | |
254 | .show = mem_show, | |
255 | }; | |
256 | ||
257 | static struct kobj_type mem_type = { | |
5108d768 | 258 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
259 | .sysfs_ops = &mem_ops, |
260 | }; | |
261 | ||
262 | static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, | |
263 | char *buffer) | |
264 | { | |
265 | ssize_t ret; | |
266 | uint32_t i; | |
267 | struct kfd_cache_properties *cache; | |
268 | ||
269 | /* Making sure that the buffer is an empty string */ | |
270 | buffer[0] = 0; | |
271 | ||
272 | cache = container_of(attr, struct kfd_cache_properties, attr); | |
273 | sysfs_show_32bit_prop(buffer, "processor_id_low", | |
274 | cache->processor_id_low); | |
275 | sysfs_show_32bit_prop(buffer, "level", cache->cache_level); | |
276 | sysfs_show_32bit_prop(buffer, "size", cache->cache_size); | |
277 | sysfs_show_32bit_prop(buffer, "cache_line_size", cache->cacheline_size); | |
278 | sysfs_show_32bit_prop(buffer, "cache_lines_per_tag", | |
279 | cache->cachelines_per_tag); | |
280 | sysfs_show_32bit_prop(buffer, "association", cache->cache_assoc); | |
281 | sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency); | |
282 | sysfs_show_32bit_prop(buffer, "type", cache->cache_type); | |
283 | snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer); | |
284 | for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++) | |
285 | ret = snprintf(buffer, PAGE_SIZE, "%s%d%s", | |
286 | buffer, cache->sibling_map[i], | |
287 | (i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ? | |
288 | "\n" : ","); | |
289 | ||
290 | return ret; | |
291 | } | |
292 | ||
293 | static const struct sysfs_ops cache_ops = { | |
294 | .show = kfd_cache_show, | |
295 | }; | |
296 | ||
297 | static struct kobj_type cache_type = { | |
5108d768 | 298 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
299 | .sysfs_ops = &cache_ops, |
300 | }; | |
301 | ||
302 | static ssize_t node_show(struct kobject *kobj, struct attribute *attr, | |
303 | char *buffer) | |
304 | { | |
5b5c4e40 EP |
305 | struct kfd_topology_device *dev; |
306 | char public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE]; | |
307 | uint32_t i; | |
f7c826ad | 308 | uint32_t log_max_watch_addr; |
5b5c4e40 EP |
309 | |
310 | /* Making sure that the buffer is an empty string */ | |
311 | buffer[0] = 0; | |
312 | ||
313 | if (strcmp(attr->name, "gpu_id") == 0) { | |
314 | dev = container_of(attr, struct kfd_topology_device, | |
315 | attr_gpuid); | |
f7c826ad AS |
316 | return sysfs_show_32bit_val(buffer, dev->gpu_id); |
317 | } | |
318 | ||
319 | if (strcmp(attr->name, "name") == 0) { | |
5b5c4e40 EP |
320 | dev = container_of(attr, struct kfd_topology_device, |
321 | attr_name); | |
322 | for (i = 0; i < KFD_TOPOLOGY_PUBLIC_NAME_SIZE; i++) { | |
323 | public_name[i] = | |
324 | (char)dev->node_props.marketing_name[i]; | |
325 | if (dev->node_props.marketing_name[i] == 0) | |
326 | break; | |
327 | } | |
328 | public_name[KFD_TOPOLOGY_PUBLIC_NAME_SIZE-1] = 0x0; | |
f7c826ad AS |
329 | return sysfs_show_str_val(buffer, public_name); |
330 | } | |
5b5c4e40 | 331 | |
f7c826ad AS |
332 | dev = container_of(attr, struct kfd_topology_device, |
333 | attr_props); | |
334 | sysfs_show_32bit_prop(buffer, "cpu_cores_count", | |
335 | dev->node_props.cpu_cores_count); | |
336 | sysfs_show_32bit_prop(buffer, "simd_count", | |
337 | dev->node_props.simd_count); | |
338 | ||
339 | if (dev->mem_bank_count < dev->node_props.mem_banks_count) { | |
79775b62 | 340 | pr_info_once("mem_banks_count truncated from %d to %d\n", |
f7c826ad AS |
341 | dev->node_props.mem_banks_count, |
342 | dev->mem_bank_count); | |
343 | sysfs_show_32bit_prop(buffer, "mem_banks_count", | |
344 | dev->mem_bank_count); | |
345 | } else { | |
346 | sysfs_show_32bit_prop(buffer, "mem_banks_count", | |
347 | dev->node_props.mem_banks_count); | |
348 | } | |
f1386fbc | 349 | |
f7c826ad AS |
350 | sysfs_show_32bit_prop(buffer, "caches_count", |
351 | dev->node_props.caches_count); | |
352 | sysfs_show_32bit_prop(buffer, "io_links_count", | |
353 | dev->node_props.io_links_count); | |
354 | sysfs_show_32bit_prop(buffer, "cpu_core_id_base", | |
355 | dev->node_props.cpu_core_id_base); | |
356 | sysfs_show_32bit_prop(buffer, "simd_id_base", | |
357 | dev->node_props.simd_id_base); | |
f7c826ad AS |
358 | sysfs_show_32bit_prop(buffer, "max_waves_per_simd", |
359 | dev->node_props.max_waves_per_simd); | |
360 | sysfs_show_32bit_prop(buffer, "lds_size_in_kb", | |
361 | dev->node_props.lds_size_in_kb); | |
362 | sysfs_show_32bit_prop(buffer, "gds_size_in_kb", | |
363 | dev->node_props.gds_size_in_kb); | |
364 | sysfs_show_32bit_prop(buffer, "wave_front_size", | |
365 | dev->node_props.wave_front_size); | |
366 | sysfs_show_32bit_prop(buffer, "array_count", | |
367 | dev->node_props.array_count); | |
368 | sysfs_show_32bit_prop(buffer, "simd_arrays_per_engine", | |
369 | dev->node_props.simd_arrays_per_engine); | |
370 | sysfs_show_32bit_prop(buffer, "cu_per_simd_array", | |
371 | dev->node_props.cu_per_simd_array); | |
372 | sysfs_show_32bit_prop(buffer, "simd_per_cu", | |
373 | dev->node_props.simd_per_cu); | |
374 | sysfs_show_32bit_prop(buffer, "max_slots_scratch_cu", | |
375 | dev->node_props.max_slots_scratch_cu); | |
376 | sysfs_show_32bit_prop(buffer, "vendor_id", | |
377 | dev->node_props.vendor_id); | |
378 | sysfs_show_32bit_prop(buffer, "device_id", | |
379 | dev->node_props.device_id); | |
380 | sysfs_show_32bit_prop(buffer, "location_id", | |
381 | dev->node_props.location_id); | |
382 | ||
383 | if (dev->gpu) { | |
384 | log_max_watch_addr = | |
385 | __ilog2_u32(dev->gpu->device_info->num_of_watch_points); | |
386 | ||
387 | if (log_max_watch_addr) { | |
388 | dev->node_props.capability |= | |
389 | HSA_CAP_WATCH_POINTS_SUPPORTED; | |
390 | ||
391 | dev->node_props.capability |= | |
392 | ((log_max_watch_addr << | |
393 | HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT) & | |
394 | HSA_CAP_WATCH_POINTS_TOTALBITS_MASK); | |
5b5c4e40 EP |
395 | } |
396 | ||
f7c826ad | 397 | sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute", |
cea405b1 | 398 | dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz( |
f7c826ad | 399 | dev->gpu->kgd)); |
42e08c78 | 400 | |
f7c826ad | 401 | sysfs_show_64bit_prop(buffer, "local_mem_size", |
42e08c78 | 402 | (unsigned long long int) 0); |
f7c826ad AS |
403 | |
404 | sysfs_show_32bit_prop(buffer, "fw_version", | |
cea405b1 | 405 | dev->gpu->kfd2kgd->get_fw_version( |
f7c826ad AS |
406 | dev->gpu->kgd, |
407 | KGD_ENGINE_MEC1)); | |
826f5de8 AS |
408 | sysfs_show_32bit_prop(buffer, "capability", |
409 | dev->node_props.capability); | |
5b5c4e40 EP |
410 | } |
411 | ||
f7c826ad AS |
412 | return sysfs_show_32bit_prop(buffer, "max_engine_clk_ccompute", |
413 | cpufreq_quick_get_max(0)/1000); | |
5b5c4e40 EP |
414 | } |
415 | ||
416 | static const struct sysfs_ops node_ops = { | |
417 | .show = node_show, | |
418 | }; | |
419 | ||
420 | static struct kobj_type node_type = { | |
5108d768 | 421 | .release = kfd_topology_kobj_release, |
5b5c4e40 EP |
422 | .sysfs_ops = &node_ops, |
423 | }; | |
424 | ||
425 | static void kfd_remove_sysfs_file(struct kobject *kobj, struct attribute *attr) | |
426 | { | |
427 | sysfs_remove_file(kobj, attr); | |
428 | kobject_del(kobj); | |
429 | kobject_put(kobj); | |
430 | } | |
431 | ||
432 | static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev) | |
433 | { | |
434 | struct kfd_iolink_properties *iolink; | |
435 | struct kfd_cache_properties *cache; | |
436 | struct kfd_mem_properties *mem; | |
437 | ||
5b5c4e40 EP |
438 | if (dev->kobj_iolink) { |
439 | list_for_each_entry(iolink, &dev->io_link_props, list) | |
440 | if (iolink->kobj) { | |
441 | kfd_remove_sysfs_file(iolink->kobj, | |
442 | &iolink->attr); | |
16b9201c | 443 | iolink->kobj = NULL; |
5b5c4e40 EP |
444 | } |
445 | kobject_del(dev->kobj_iolink); | |
446 | kobject_put(dev->kobj_iolink); | |
16b9201c | 447 | dev->kobj_iolink = NULL; |
5b5c4e40 EP |
448 | } |
449 | ||
450 | if (dev->kobj_cache) { | |
451 | list_for_each_entry(cache, &dev->cache_props, list) | |
452 | if (cache->kobj) { | |
453 | kfd_remove_sysfs_file(cache->kobj, | |
454 | &cache->attr); | |
16b9201c | 455 | cache->kobj = NULL; |
5b5c4e40 EP |
456 | } |
457 | kobject_del(dev->kobj_cache); | |
458 | kobject_put(dev->kobj_cache); | |
16b9201c | 459 | dev->kobj_cache = NULL; |
5b5c4e40 EP |
460 | } |
461 | ||
462 | if (dev->kobj_mem) { | |
463 | list_for_each_entry(mem, &dev->mem_props, list) | |
464 | if (mem->kobj) { | |
465 | kfd_remove_sysfs_file(mem->kobj, &mem->attr); | |
16b9201c | 466 | mem->kobj = NULL; |
5b5c4e40 EP |
467 | } |
468 | kobject_del(dev->kobj_mem); | |
469 | kobject_put(dev->kobj_mem); | |
16b9201c | 470 | dev->kobj_mem = NULL; |
5b5c4e40 EP |
471 | } |
472 | ||
473 | if (dev->kobj_node) { | |
474 | sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid); | |
475 | sysfs_remove_file(dev->kobj_node, &dev->attr_name); | |
476 | sysfs_remove_file(dev->kobj_node, &dev->attr_props); | |
477 | kobject_del(dev->kobj_node); | |
478 | kobject_put(dev->kobj_node); | |
16b9201c | 479 | dev->kobj_node = NULL; |
5b5c4e40 EP |
480 | } |
481 | } | |
482 | ||
483 | static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev, | |
484 | uint32_t id) | |
485 | { | |
486 | struct kfd_iolink_properties *iolink; | |
487 | struct kfd_cache_properties *cache; | |
488 | struct kfd_mem_properties *mem; | |
489 | int ret; | |
490 | uint32_t i; | |
491 | ||
32fa8219 FK |
492 | if (WARN_ON(dev->kobj_node)) |
493 | return -EEXIST; | |
494 | ||
5b5c4e40 EP |
495 | /* |
496 | * Creating the sysfs folders | |
497 | */ | |
5b5c4e40 EP |
498 | dev->kobj_node = kfd_alloc_struct(dev->kobj_node); |
499 | if (!dev->kobj_node) | |
500 | return -ENOMEM; | |
501 | ||
502 | ret = kobject_init_and_add(dev->kobj_node, &node_type, | |
503 | sys_props.kobj_nodes, "%d", id); | |
504 | if (ret < 0) | |
505 | return ret; | |
506 | ||
507 | dev->kobj_mem = kobject_create_and_add("mem_banks", dev->kobj_node); | |
508 | if (!dev->kobj_mem) | |
509 | return -ENOMEM; | |
510 | ||
511 | dev->kobj_cache = kobject_create_and_add("caches", dev->kobj_node); | |
512 | if (!dev->kobj_cache) | |
513 | return -ENOMEM; | |
514 | ||
515 | dev->kobj_iolink = kobject_create_and_add("io_links", dev->kobj_node); | |
516 | if (!dev->kobj_iolink) | |
517 | return -ENOMEM; | |
518 | ||
519 | /* | |
520 | * Creating sysfs files for node properties | |
521 | */ | |
522 | dev->attr_gpuid.name = "gpu_id"; | |
523 | dev->attr_gpuid.mode = KFD_SYSFS_FILE_MODE; | |
524 | sysfs_attr_init(&dev->attr_gpuid); | |
525 | dev->attr_name.name = "name"; | |
526 | dev->attr_name.mode = KFD_SYSFS_FILE_MODE; | |
527 | sysfs_attr_init(&dev->attr_name); | |
528 | dev->attr_props.name = "properties"; | |
529 | dev->attr_props.mode = KFD_SYSFS_FILE_MODE; | |
530 | sysfs_attr_init(&dev->attr_props); | |
531 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_gpuid); | |
532 | if (ret < 0) | |
533 | return ret; | |
534 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_name); | |
535 | if (ret < 0) | |
536 | return ret; | |
537 | ret = sysfs_create_file(dev->kobj_node, &dev->attr_props); | |
538 | if (ret < 0) | |
539 | return ret; | |
540 | ||
541 | i = 0; | |
542 | list_for_each_entry(mem, &dev->mem_props, list) { | |
543 | mem->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
544 | if (!mem->kobj) | |
545 | return -ENOMEM; | |
546 | ret = kobject_init_and_add(mem->kobj, &mem_type, | |
547 | dev->kobj_mem, "%d", i); | |
548 | if (ret < 0) | |
549 | return ret; | |
550 | ||
551 | mem->attr.name = "properties"; | |
552 | mem->attr.mode = KFD_SYSFS_FILE_MODE; | |
553 | sysfs_attr_init(&mem->attr); | |
554 | ret = sysfs_create_file(mem->kobj, &mem->attr); | |
555 | if (ret < 0) | |
556 | return ret; | |
557 | i++; | |
558 | } | |
559 | ||
560 | i = 0; | |
561 | list_for_each_entry(cache, &dev->cache_props, list) { | |
562 | cache->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
563 | if (!cache->kobj) | |
564 | return -ENOMEM; | |
565 | ret = kobject_init_and_add(cache->kobj, &cache_type, | |
566 | dev->kobj_cache, "%d", i); | |
567 | if (ret < 0) | |
568 | return ret; | |
569 | ||
570 | cache->attr.name = "properties"; | |
571 | cache->attr.mode = KFD_SYSFS_FILE_MODE; | |
572 | sysfs_attr_init(&cache->attr); | |
573 | ret = sysfs_create_file(cache->kobj, &cache->attr); | |
574 | if (ret < 0) | |
575 | return ret; | |
576 | i++; | |
577 | } | |
578 | ||
579 | i = 0; | |
580 | list_for_each_entry(iolink, &dev->io_link_props, list) { | |
581 | iolink->kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); | |
582 | if (!iolink->kobj) | |
583 | return -ENOMEM; | |
584 | ret = kobject_init_and_add(iolink->kobj, &iolink_type, | |
585 | dev->kobj_iolink, "%d", i); | |
586 | if (ret < 0) | |
587 | return ret; | |
588 | ||
589 | iolink->attr.name = "properties"; | |
590 | iolink->attr.mode = KFD_SYSFS_FILE_MODE; | |
591 | sysfs_attr_init(&iolink->attr); | |
592 | ret = sysfs_create_file(iolink->kobj, &iolink->attr); | |
593 | if (ret < 0) | |
594 | return ret; | |
595 | i++; | |
596 | } | |
597 | ||
598 | return 0; | |
599 | } | |
600 | ||
601 | static int kfd_build_sysfs_node_tree(void) | |
602 | { | |
603 | struct kfd_topology_device *dev; | |
604 | int ret; | |
605 | uint32_t i = 0; | |
606 | ||
607 | list_for_each_entry(dev, &topology_device_list, list) { | |
8dfead6c | 608 | ret = kfd_build_sysfs_node_entry(dev, i); |
5b5c4e40 EP |
609 | if (ret < 0) |
610 | return ret; | |
611 | i++; | |
612 | } | |
613 | ||
614 | return 0; | |
615 | } | |
616 | ||
617 | static void kfd_remove_sysfs_node_tree(void) | |
618 | { | |
619 | struct kfd_topology_device *dev; | |
620 | ||
621 | list_for_each_entry(dev, &topology_device_list, list) | |
622 | kfd_remove_sysfs_node_entry(dev); | |
623 | } | |
624 | ||
625 | static int kfd_topology_update_sysfs(void) | |
626 | { | |
627 | int ret; | |
628 | ||
629 | pr_info("Creating topology SYSFS entries\n"); | |
4eacc26b | 630 | if (!sys_props.kobj_topology) { |
5b5c4e40 EP |
631 | sys_props.kobj_topology = |
632 | kfd_alloc_struct(sys_props.kobj_topology); | |
633 | if (!sys_props.kobj_topology) | |
634 | return -ENOMEM; | |
635 | ||
636 | ret = kobject_init_and_add(sys_props.kobj_topology, | |
637 | &sysprops_type, &kfd_device->kobj, | |
638 | "topology"); | |
639 | if (ret < 0) | |
640 | return ret; | |
641 | ||
642 | sys_props.kobj_nodes = kobject_create_and_add("nodes", | |
643 | sys_props.kobj_topology); | |
644 | if (!sys_props.kobj_nodes) | |
645 | return -ENOMEM; | |
646 | ||
647 | sys_props.attr_genid.name = "generation_id"; | |
648 | sys_props.attr_genid.mode = KFD_SYSFS_FILE_MODE; | |
649 | sysfs_attr_init(&sys_props.attr_genid); | |
650 | ret = sysfs_create_file(sys_props.kobj_topology, | |
651 | &sys_props.attr_genid); | |
652 | if (ret < 0) | |
653 | return ret; | |
654 | ||
655 | sys_props.attr_props.name = "system_properties"; | |
656 | sys_props.attr_props.mode = KFD_SYSFS_FILE_MODE; | |
657 | sysfs_attr_init(&sys_props.attr_props); | |
658 | ret = sysfs_create_file(sys_props.kobj_topology, | |
659 | &sys_props.attr_props); | |
660 | if (ret < 0) | |
661 | return ret; | |
662 | } | |
663 | ||
664 | kfd_remove_sysfs_node_tree(); | |
665 | ||
666 | return kfd_build_sysfs_node_tree(); | |
667 | } | |
668 | ||
669 | static void kfd_topology_release_sysfs(void) | |
670 | { | |
671 | kfd_remove_sysfs_node_tree(); | |
672 | if (sys_props.kobj_topology) { | |
673 | sysfs_remove_file(sys_props.kobj_topology, | |
674 | &sys_props.attr_genid); | |
675 | sysfs_remove_file(sys_props.kobj_topology, | |
676 | &sys_props.attr_props); | |
677 | if (sys_props.kobj_nodes) { | |
678 | kobject_del(sys_props.kobj_nodes); | |
679 | kobject_put(sys_props.kobj_nodes); | |
16b9201c | 680 | sys_props.kobj_nodes = NULL; |
5b5c4e40 EP |
681 | } |
682 | kobject_del(sys_props.kobj_topology); | |
683 | kobject_put(sys_props.kobj_topology); | |
16b9201c | 684 | sys_props.kobj_topology = NULL; |
5b5c4e40 EP |
685 | } |
686 | } | |
687 | ||
4f449311 HK |
688 | /* Called with write topology_lock acquired */ |
689 | static void kfd_topology_update_device_list(struct list_head *temp_list, | |
690 | struct list_head *master_list) | |
691 | { | |
692 | while (!list_empty(temp_list)) { | |
693 | list_move_tail(temp_list->next, master_list); | |
694 | sys_props.num_devices++; | |
695 | } | |
696 | } | |
697 | ||
5b5c4e40 EP |
698 | int kfd_topology_init(void) |
699 | { | |
16b9201c | 700 | void *crat_image = NULL; |
5b5c4e40 EP |
701 | size_t image_size = 0; |
702 | int ret; | |
4f449311 | 703 | struct list_head temp_topology_device_list; |
5b5c4e40 | 704 | |
4f449311 HK |
705 | /* topology_device_list - Master list of all topology devices |
706 | * temp_topology_device_list - temporary list created while parsing CRAT | |
707 | * or VCRAT. Once parsing is complete the contents of list is moved to | |
708 | * topology_device_list | |
5b5c4e40 | 709 | */ |
4f449311 HK |
710 | |
711 | /* Initialize the head for the both the lists */ | |
5b5c4e40 | 712 | INIT_LIST_HEAD(&topology_device_list); |
4f449311 | 713 | INIT_LIST_HEAD(&temp_topology_device_list); |
5b5c4e40 | 714 | init_rwsem(&topology_lock); |
5b5c4e40 EP |
715 | |
716 | memset(&sys_props, 0, sizeof(sys_props)); | |
717 | ||
718 | /* | |
719 | * Get the CRAT image from the ACPI | |
720 | */ | |
8e05247d HK |
721 | ret = kfd_create_crat_image_acpi(&crat_image, &image_size); |
722 | if (!ret) { | |
4f449311 HK |
723 | ret = kfd_parse_crat_table(crat_image, |
724 | &temp_topology_device_list, 0); | |
8e05247d | 725 | if (ret) |
5b5c4e40 | 726 | goto err; |
5b5c4e40 | 727 | } else if (ret == -ENODATA) { |
8e05247d | 728 | /* TODO: Create fake CRAT table */ |
5b5c4e40 | 729 | ret = 0; |
8e05247d | 730 | goto err; |
5b5c4e40 EP |
731 | } else { |
732 | pr_err("Couldn't get CRAT table size from ACPI\n"); | |
8e05247d | 733 | goto err; |
5b5c4e40 EP |
734 | } |
735 | ||
8e05247d | 736 | down_write(&topology_lock); |
4f449311 HK |
737 | kfd_topology_update_device_list(&temp_topology_device_list, |
738 | &topology_device_list); | |
8e05247d HK |
739 | ret = kfd_topology_update_sysfs(); |
740 | up_write(&topology_lock); | |
741 | ||
4f449311 HK |
742 | if (!ret) { |
743 | sys_props.generation_count++; | |
8e05247d | 744 | pr_info("Finished initializing topology\n"); |
4f449311 | 745 | } else |
8e05247d HK |
746 | pr_err("Failed to update topology in sysfs ret=%d\n", ret); |
747 | ||
5b5c4e40 | 748 | err: |
8e05247d | 749 | kfd_destroy_crat_image(crat_image); |
5b5c4e40 EP |
750 | return ret; |
751 | } | |
752 | ||
753 | void kfd_topology_shutdown(void) | |
754 | { | |
4f449311 | 755 | down_write(&topology_lock); |
5b5c4e40 EP |
756 | kfd_topology_release_sysfs(); |
757 | kfd_release_live_view(); | |
4f449311 | 758 | up_write(&topology_lock); |
5b5c4e40 EP |
759 | } |
760 | ||
761 | static void kfd_debug_print_topology(void) | |
762 | { | |
763 | struct kfd_topology_device *dev; | |
764 | uint32_t i = 0; | |
765 | ||
766 | pr_info("DEBUG PRINT OF TOPOLOGY:"); | |
767 | list_for_each_entry(dev, &topology_device_list, list) { | |
768 | pr_info("Node: %d\n", i); | |
769 | pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no")); | |
770 | pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count); | |
8e05247d | 771 | pr_info("\tSIMD count: %d\n", dev->node_props.simd_count); |
5b5c4e40 EP |
772 | i++; |
773 | } | |
774 | } | |
775 | ||
776 | static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu) | |
777 | { | |
778 | uint32_t hashout; | |
779 | uint32_t buf[7]; | |
585f0e6c | 780 | uint64_t local_mem_size; |
5b5c4e40 | 781 | int i; |
0504cccf | 782 | struct kfd_local_mem_info local_mem_info; |
5b5c4e40 EP |
783 | |
784 | if (!gpu) | |
785 | return 0; | |
786 | ||
0504cccf HK |
787 | gpu->kfd2kgd->get_local_mem_info(gpu->kgd, &local_mem_info); |
788 | ||
789 | local_mem_size = local_mem_info.local_mem_size_private + | |
790 | local_mem_info.local_mem_size_public; | |
585f0e6c | 791 | |
5b5c4e40 EP |
792 | buf[0] = gpu->pdev->devfn; |
793 | buf[1] = gpu->pdev->subsystem_vendor; | |
794 | buf[2] = gpu->pdev->subsystem_device; | |
795 | buf[3] = gpu->pdev->device; | |
796 | buf[4] = gpu->pdev->bus->number; | |
585f0e6c EC |
797 | buf[5] = lower_32_bits(local_mem_size); |
798 | buf[6] = upper_32_bits(local_mem_size); | |
5b5c4e40 EP |
799 | |
800 | for (i = 0, hashout = 0; i < 7; i++) | |
801 | hashout ^= hash_32(buf[i], KFD_GPU_ID_HASH_WIDTH); | |
802 | ||
803 | return hashout; | |
804 | } | |
805 | ||
806 | static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) | |
807 | { | |
808 | struct kfd_topology_device *dev; | |
16b9201c | 809 | struct kfd_topology_device *out_dev = NULL; |
5b5c4e40 | 810 | |
5b5c4e40 | 811 | list_for_each_entry(dev, &topology_device_list, list) |
4eacc26b | 812 | if (!dev->gpu && (dev->node_props.simd_count > 0)) { |
5b5c4e40 EP |
813 | dev->gpu = gpu; |
814 | out_dev = dev; | |
815 | break; | |
816 | } | |
817 | ||
818 | return out_dev; | |
819 | } | |
820 | ||
821 | static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival) | |
822 | { | |
823 | /* | |
824 | * TODO: Generate an event for thunk about the arrival/removal | |
825 | * of the GPU | |
826 | */ | |
827 | } | |
828 | ||
829 | int kfd_topology_add_device(struct kfd_dev *gpu) | |
830 | { | |
831 | uint32_t gpu_id; | |
832 | struct kfd_topology_device *dev; | |
f7ce2fad | 833 | struct kfd_cu_info cu_info; |
4f449311 HK |
834 | int res = 0; |
835 | struct list_head temp_topology_device_list; | |
836 | ||
837 | INIT_LIST_HEAD(&temp_topology_device_list); | |
5b5c4e40 | 838 | |
5b5c4e40 EP |
839 | gpu_id = kfd_generate_gpu_id(gpu); |
840 | ||
79775b62 | 841 | pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id); |
5b5c4e40 | 842 | |
5b5c4e40 EP |
843 | /* |
844 | * Try to assign the GPU to existing topology device (generated from | |
845 | * CRAT table | |
846 | */ | |
847 | dev = kfd_assign_gpu(gpu); | |
848 | if (!dev) { | |
849 | pr_info("GPU was not found in the current topology. Extending.\n"); | |
850 | kfd_debug_print_topology(); | |
4f449311 | 851 | dev = kfd_create_topology_device(&temp_topology_device_list); |
5b5c4e40 EP |
852 | if (!dev) { |
853 | res = -ENOMEM; | |
854 | goto err; | |
855 | } | |
4f449311 | 856 | |
5b5c4e40 EP |
857 | dev->gpu = gpu; |
858 | ||
859 | /* | |
860 | * TODO: Make a call to retrieve topology information from the | |
861 | * GPU vBIOS | |
862 | */ | |
863 | ||
4f449311 HK |
864 | down_write(&topology_lock); |
865 | kfd_topology_update_device_list(&temp_topology_device_list, | |
866 | &topology_device_list); | |
867 | ||
8eabaf54 KR |
868 | /* Update the SYSFS tree, since we added another topology |
869 | * device | |
5b5c4e40 EP |
870 | */ |
871 | if (kfd_topology_update_sysfs() < 0) | |
872 | kfd_topology_release_sysfs(); | |
873 | ||
4f449311 HK |
874 | up_write(&topology_lock); |
875 | ||
5b5c4e40 EP |
876 | } |
877 | ||
878 | dev->gpu_id = gpu_id; | |
879 | gpu->id = gpu_id; | |
f7ce2fad FC |
880 | dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info); |
881 | dev->node_props.simd_count = dev->node_props.simd_per_cu * | |
882 | cu_info.cu_active_number; | |
5b5c4e40 EP |
883 | dev->node_props.vendor_id = gpu->pdev->vendor; |
884 | dev->node_props.device_id = gpu->pdev->device; | |
d63f0ba2 HK |
885 | dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number, |
886 | gpu->pdev->devfn); | |
5b5c4e40 EP |
887 | /* |
888 | * TODO: Retrieve max engine clock values from KGD | |
889 | */ | |
890 | ||
7639a8c4 BG |
891 | if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) { |
892 | dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE; | |
42aa8793 | 893 | pr_debug("Adding doorbell packet type capability\n"); |
7639a8c4 BG |
894 | } |
895 | ||
4f449311 | 896 | if (!res) |
5b5c4e40 | 897 | kfd_notify_gpu_change(gpu_id, 1); |
4f449311 | 898 | err: |
5b5c4e40 EP |
899 | return res; |
900 | } | |
901 | ||
902 | int kfd_topology_remove_device(struct kfd_dev *gpu) | |
903 | { | |
4f449311 | 904 | struct kfd_topology_device *dev, *tmp; |
5b5c4e40 EP |
905 | uint32_t gpu_id; |
906 | int res = -ENODEV; | |
907 | ||
5b5c4e40 EP |
908 | down_write(&topology_lock); |
909 | ||
4f449311 | 910 | list_for_each_entry_safe(dev, tmp, &topology_device_list, list) |
5b5c4e40 EP |
911 | if (dev->gpu == gpu) { |
912 | gpu_id = dev->gpu_id; | |
913 | kfd_remove_sysfs_node_entry(dev); | |
914 | kfd_release_topology_device(dev); | |
4f449311 | 915 | sys_props.num_devices--; |
5b5c4e40 EP |
916 | res = 0; |
917 | if (kfd_topology_update_sysfs() < 0) | |
918 | kfd_topology_release_sysfs(); | |
919 | break; | |
920 | } | |
921 | ||
922 | up_write(&topology_lock); | |
923 | ||
174de876 | 924 | if (!res) |
5b5c4e40 EP |
925 | kfd_notify_gpu_change(gpu_id, 0); |
926 | ||
927 | return res; | |
928 | } | |
929 | ||
6d82eb0e HK |
930 | /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD |
931 | * topology. If GPU device is found @idx, then valid kfd_dev pointer is | |
932 | * returned through @kdev | |
933 | * Return - 0: On success (@kdev will be NULL for non GPU nodes) | |
934 | * -1: If end of list | |
5b5c4e40 | 935 | */ |
6d82eb0e | 936 | int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev) |
5b5c4e40 EP |
937 | { |
938 | ||
939 | struct kfd_topology_device *top_dev; | |
5b5c4e40 EP |
940 | uint8_t device_idx = 0; |
941 | ||
6d82eb0e | 942 | *kdev = NULL; |
5b5c4e40 EP |
943 | down_read(&topology_lock); |
944 | ||
945 | list_for_each_entry(top_dev, &topology_device_list, list) { | |
946 | if (device_idx == idx) { | |
6d82eb0e HK |
947 | *kdev = top_dev->gpu; |
948 | up_read(&topology_lock); | |
949 | return 0; | |
5b5c4e40 EP |
950 | } |
951 | ||
952 | device_idx++; | |
953 | } | |
954 | ||
955 | up_read(&topology_lock); | |
956 | ||
6d82eb0e | 957 | return -1; |
5b5c4e40 EP |
958 | |
959 | } | |
851a645e FK |
960 | |
961 | #if defined(CONFIG_DEBUG_FS) | |
962 | ||
963 | int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data) | |
964 | { | |
965 | struct kfd_topology_device *dev; | |
966 | unsigned int i = 0; | |
967 | int r = 0; | |
968 | ||
969 | down_read(&topology_lock); | |
970 | ||
971 | list_for_each_entry(dev, &topology_device_list, list) { | |
972 | if (!dev->gpu) { | |
973 | i++; | |
974 | continue; | |
975 | } | |
976 | ||
977 | seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); | |
978 | r = dqm_debugfs_hqds(m, dev->gpu->dqm); | |
979 | if (r) | |
980 | break; | |
981 | } | |
982 | ||
983 | up_read(&topology_lock); | |
984 | ||
985 | return r; | |
986 | } | |
987 | ||
988 | int kfd_debugfs_rls_by_device(struct seq_file *m, void *data) | |
989 | { | |
990 | struct kfd_topology_device *dev; | |
991 | unsigned int i = 0; | |
992 | int r = 0; | |
993 | ||
994 | down_read(&topology_lock); | |
995 | ||
996 | list_for_each_entry(dev, &topology_device_list, list) { | |
997 | if (!dev->gpu) { | |
998 | i++; | |
999 | continue; | |
1000 | } | |
1001 | ||
1002 | seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id); | |
1003 | r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets); | |
1004 | if (r) | |
1005 | break; | |
1006 | } | |
1007 | ||
1008 | up_read(&topology_lock); | |
1009 | ||
1010 | return r; | |
1011 | } | |
1012 | ||
1013 | #endif |