Commit | Line | Data |
---|---|---|
ca120a79 GH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * SiFive composable cache controller Driver | |
4 | * | |
5 | * Copyright (C) 2018-2022 SiFive, Inc. | |
6 | * | |
7 | */ | |
696ab9bd BD |
8 | |
9 | #define pr_fmt(fmt) "CCACHE: " fmt | |
10 | ||
0d5701dc | 11 | #include <linux/align.h> |
ca120a79 GH |
12 | #include <linux/debugfs.h> |
13 | #include <linux/interrupt.h> | |
14 | #include <linux/of_irq.h> | |
15 | #include <linux/of_address.h> | |
16 | #include <linux/device.h> | |
afc7a583 | 17 | #include <linux/bitfield.h> |
c90847bc SH |
18 | #include <linux/platform_device.h> |
19 | #include <linux/property.h> | |
0d5701dc | 20 | #include <asm/cacheflush.h> |
ca120a79 | 21 | #include <asm/cacheinfo.h> |
0d5701dc | 22 | #include <asm/dma-noncoherent.h> |
ca120a79 GH |
23 | #include <soc/sifive/sifive_ccache.h> |
24 | ||
25 | #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 | |
26 | #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 | |
27 | #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 | |
28 | ||
29 | #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 | |
30 | #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 | |
31 | #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 | |
32 | ||
33 | #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 | |
34 | #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 | |
35 | #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 | |
36 | ||
37 | #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 | |
38 | #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 | |
39 | #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 | |
40 | ||
41 | #define SIFIVE_CCACHE_CONFIG 0x00 | |
afc7a583 ZL |
42 | #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) |
43 | #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) | |
44 | #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) | |
45 | #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) | |
46 | ||
0d5701dc ERB |
47 | #define SIFIVE_CCACHE_FLUSH64 0x200 |
48 | #define SIFIVE_CCACHE_FLUSH32 0x240 | |
49 | ||
ca120a79 GH |
50 | #define SIFIVE_CCACHE_WAYENABLE 0x08 |
51 | #define SIFIVE_CCACHE_ECCINJECTERR 0x40 | |
52 | ||
53 | #define SIFIVE_CCACHE_MAX_ECCINTR 4 | |
0d5701dc | 54 | #define SIFIVE_CCACHE_LINE_SIZE 64 |
ca120a79 GH |
55 | |
56 | static void __iomem *ccache_base; | |
57 | static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; | |
58 | static struct riscv_cacheinfo_ops ccache_cache_ops; | |
95f196f3 | 59 | static int level; |
ca120a79 GH |
60 | |
61 | enum { | |
62 | DIR_CORR = 0, | |
63 | DATA_CORR, | |
64 | DATA_UNCORR, | |
65 | DIR_UNCORR, | |
66 | }; | |
67 | ||
0d5701dc ERB |
68 | enum { |
69 | QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), | |
70 | QUIRK_BROKEN_DATA_UNCORR = BIT(1), | |
71 | }; | |
72 | ||
ca120a79 GH |
73 | #ifdef CONFIG_DEBUG_FS |
74 | static struct dentry *sifive_test; | |
75 | ||
76 | static ssize_t ccache_write(struct file *file, const char __user *data, | |
77 | size_t count, loff_t *ppos) | |
78 | { | |
79 | unsigned int val; | |
80 | ||
81 | if (kstrtouint_from_user(data, count, 0, &val)) | |
82 | return -EINVAL; | |
83 | if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) | |
84 | writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); | |
85 | else | |
86 | return -EINVAL; | |
87 | return count; | |
88 | } | |
89 | ||
90 | static const struct file_operations ccache_fops = { | |
91 | .owner = THIS_MODULE, | |
92 | .open = simple_open, | |
93 | .write = ccache_write | |
94 | }; | |
95 | ||
96 | static void setup_sifive_debug(void) | |
97 | { | |
98 | sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); | |
99 | ||
100 | debugfs_create_file("sifive_debug_inject_error", 0200, | |
101 | sifive_test, NULL, &ccache_fops); | |
102 | } | |
103 | #endif | |
104 | ||
105 | static void ccache_config_read(void) | |
106 | { | |
3fb787e5 BD |
107 | u32 cfg; |
108 | ||
109 | cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); | |
afc7a583 ZL |
110 | pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", |
111 | FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), | |
112 | FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), | |
113 | BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), | |
114 | BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); | |
3fb787e5 BD |
115 | |
116 | cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); | |
696ab9bd | 117 | pr_info("Index of the largest way enabled: %u\n", cfg); |
ca120a79 GH |
118 | } |
119 | ||
120 | static const struct of_device_id sifive_ccache_ids[] = { | |
121 | { .compatible = "sifive,fu540-c000-ccache" }, | |
122 | { .compatible = "sifive,fu740-c000-ccache" }, | |
0d5701dc ERB |
123 | { .compatible = "starfive,jh7100-ccache", |
124 | .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, | |
ca120a79 GH |
125 | { .compatible = "sifive,ccache0" }, |
126 | { /* end of table */ } | |
127 | }; | |
128 | ||
129 | static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); | |
130 | ||
131 | int register_sifive_ccache_error_notifier(struct notifier_block *nb) | |
132 | { | |
133 | return atomic_notifier_chain_register(&ccache_err_chain, nb); | |
134 | } | |
135 | EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); | |
136 | ||
137 | int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) | |
138 | { | |
139 | return atomic_notifier_chain_unregister(&ccache_err_chain, nb); | |
140 | } | |
141 | EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); | |
142 | ||
0d5701dc ERB |
143 | #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
144 | static void ccache_flush_range(phys_addr_t start, size_t len) | |
145 | { | |
146 | phys_addr_t end = start + len; | |
147 | phys_addr_t line; | |
148 | ||
149 | if (!len) | |
150 | return; | |
151 | ||
152 | mb(); | |
153 | for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; | |
154 | line += SIFIVE_CCACHE_LINE_SIZE) { | |
155 | #ifdef CONFIG_32BIT | |
156 | writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); | |
157 | #else | |
158 | writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64); | |
159 | #endif | |
160 | mb(); | |
161 | } | |
162 | } | |
163 | ||
164 | static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { | |
165 | .wback = &ccache_flush_range, | |
166 | .inv = &ccache_flush_range, | |
167 | .wback_inv = &ccache_flush_range, | |
168 | }; | |
169 | #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ | |
170 | ||
ca120a79 GH |
171 | static int ccache_largest_wayenabled(void) |
172 | { | |
173 | return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; | |
174 | } | |
175 | ||
176 | static ssize_t number_of_ways_enabled_show(struct device *dev, | |
177 | struct device_attribute *attr, | |
178 | char *buf) | |
179 | { | |
180 | return sprintf(buf, "%u\n", ccache_largest_wayenabled()); | |
181 | } | |
182 | ||
183 | static DEVICE_ATTR_RO(number_of_ways_enabled); | |
184 | ||
185 | static struct attribute *priv_attrs[] = { | |
186 | &dev_attr_number_of_ways_enabled.attr, | |
187 | NULL, | |
188 | }; | |
189 | ||
190 | static const struct attribute_group priv_attr_group = { | |
191 | .attrs = priv_attrs, | |
192 | }; | |
193 | ||
194 | static const struct attribute_group *ccache_get_priv_group(struct cacheinfo | |
195 | *this_leaf) | |
196 | { | |
197 | /* We want to use private group for composable cache only */ | |
95f196f3 | 198 | if (this_leaf->level == level) |
ca120a79 GH |
199 | return &priv_attr_group; |
200 | else | |
201 | return NULL; | |
202 | } | |
203 | ||
204 | static irqreturn_t ccache_int_handler(int irq, void *device) | |
205 | { | |
206 | unsigned int add_h, add_l; | |
207 | ||
208 | if (irq == g_irq[DIR_CORR]) { | |
209 | add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); | |
210 | add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); | |
696ab9bd | 211 | pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); |
ca120a79 GH |
212 | /* Reading this register clears the DirError interrupt sig */ |
213 | readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); | |
214 | atomic_notifier_call_chain(&ccache_err_chain, | |
215 | SIFIVE_CCACHE_ERR_TYPE_CE, | |
216 | "DirECCFix"); | |
217 | } | |
218 | if (irq == g_irq[DIR_UNCORR]) { | |
219 | add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); | |
220 | add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); | |
221 | /* Reading this register clears the DirFail interrupt sig */ | |
222 | readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); | |
223 | atomic_notifier_call_chain(&ccache_err_chain, | |
224 | SIFIVE_CCACHE_ERR_TYPE_UE, | |
225 | "DirECCFail"); | |
226 | panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); | |
227 | } | |
228 | if (irq == g_irq[DATA_CORR]) { | |
229 | add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); | |
230 | add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); | |
696ab9bd | 231 | pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); |
ca120a79 GH |
232 | /* Reading this register clears the DataError interrupt sig */ |
233 | readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); | |
234 | atomic_notifier_call_chain(&ccache_err_chain, | |
235 | SIFIVE_CCACHE_ERR_TYPE_CE, | |
236 | "DatECCFix"); | |
237 | } | |
238 | if (irq == g_irq[DATA_UNCORR]) { | |
239 | add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); | |
240 | add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); | |
696ab9bd | 241 | pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); |
ca120a79 GH |
242 | /* Reading this register clears the DataFail interrupt sig */ |
243 | readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); | |
244 | atomic_notifier_call_chain(&ccache_err_chain, | |
245 | SIFIVE_CCACHE_ERR_TYPE_UE, | |
246 | "DatECCFail"); | |
247 | } | |
248 | ||
249 | return IRQ_HANDLED; | |
250 | } | |
251 | ||
c90847bc SH |
252 | static int sifive_ccache_probe(struct platform_device *pdev) |
253 | { | |
254 | struct device *dev = &pdev->dev; | |
255 | unsigned long quirks; | |
256 | int intr_num, rc; | |
257 | ||
258 | quirks = (unsigned long)device_get_match_data(dev); | |
259 | ||
260 | intr_num = platform_irq_count(pdev); | |
261 | if (!intr_num) | |
262 | return dev_err_probe(dev, -ENODEV, "No interrupts property\n"); | |
263 | ||
264 | for (int i = 0; i < intr_num; i++) { | |
265 | if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) | |
266 | continue; | |
267 | ||
268 | g_irq[i] = platform_get_irq(pdev, i); | |
269 | if (g_irq[i] < 0) | |
270 | return g_irq[i]; | |
271 | ||
272 | rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL); | |
273 | if (rc) | |
274 | return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]); | |
275 | } | |
276 | ||
277 | return 0; | |
278 | } | |
279 | ||
280 | static struct platform_driver sifive_ccache_driver = { | |
281 | .probe = sifive_ccache_probe, | |
282 | .driver = { | |
283 | .name = "sifive_ccache", | |
284 | .of_match_table = sifive_ccache_ids, | |
285 | }, | |
286 | }; | |
287 | ||
ca120a79 GH |
288 | static int __init sifive_ccache_init(void) |
289 | { | |
290 | struct device_node *np; | |
291 | struct resource res; | |
0d5701dc ERB |
292 | const struct of_device_id *match; |
293 | unsigned long quirks; | |
c90847bc | 294 | int rc; |
ca120a79 | 295 | |
0d5701dc | 296 | np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); |
ca120a79 GH |
297 | if (!np) |
298 | return -ENODEV; | |
299 | ||
0d5701dc ERB |
300 | quirks = (uintptr_t)match->data; |
301 | ||
8fbf94fe YY |
302 | if (of_address_to_resource(np, 0, &res)) { |
303 | rc = -ENODEV; | |
304 | goto err_node_put; | |
305 | } | |
ca120a79 GH |
306 | |
307 | ccache_base = ioremap(res.start, resource_size(&res)); | |
8fbf94fe YY |
308 | if (!ccache_base) { |
309 | rc = -ENOMEM; | |
310 | goto err_node_put; | |
311 | } | |
ca120a79 | 312 | |
73e770f0 YY |
313 | if (of_property_read_u32(np, "cache-level", &level)) { |
314 | rc = -ENOENT; | |
315 | goto err_unmap; | |
316 | } | |
95f196f3 | 317 | |
0d5701dc ERB |
318 | #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
319 | if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { | |
320 | riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; | |
321 | riscv_noncoherent_supported(); | |
322 | riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); | |
323 | } | |
324 | #endif | |
325 | ||
ca120a79 GH |
326 | ccache_config_read(); |
327 | ||
328 | ccache_cache_ops.get_priv_group = ccache_get_priv_group; | |
329 | riscv_set_cacheinfo_ops(&ccache_cache_ops); | |
330 | ||
331 | #ifdef CONFIG_DEBUG_FS | |
332 | setup_sifive_debug(); | |
333 | #endif | |
c90847bc SH |
334 | |
335 | rc = platform_driver_register(&sifive_ccache_driver); | |
336 | if (rc) | |
337 | goto err_unmap; | |
338 | ||
339 | of_node_put(np); | |
340 | ||
ca120a79 | 341 | return 0; |
73e770f0 YY |
342 | |
343 | err_unmap: | |
344 | iounmap(ccache_base); | |
8fbf94fe YY |
345 | err_node_put: |
346 | of_node_put(np); | |
73e770f0 | 347 | return rc; |
ca120a79 GH |
348 | } |
349 | ||
0d5701dc | 350 | arch_initcall(sifive_ccache_init); |