| 1 |
// SPDX-License-Identifier: GPL-2.0 |
| 2 |
/* |
| 3 |
* SiFive composable cache controller Driver |
| 4 |
* |
| 5 |
* Copyright (C) 2018-2022 SiFive, Inc. |
| 6 |
* |
| 7 |
*/ |
| 8 |
|
| 9 |
#define pr_fmt(fmt) "CCACHE: " fmt |
| 10 |
|
| 11 |
#include <linux/align.h> |
| 12 |
#include <linux/debugfs.h> |
| 13 |
#include <linux/interrupt.h> |
| 14 |
#include <linux/of_irq.h> |
| 15 |
#include <linux/of_address.h> |
| 16 |
#include <linux/device.h> |
| 17 |
#include <linux/bitfield.h> |
| 18 |
#include <linux/platform_device.h> |
| 19 |
#include <linux/property.h> |
| 20 |
#include <asm/cacheflush.h> |
| 21 |
#include <asm/cacheinfo.h> |
| 22 |
#include <asm/dma-noncoherent.h> |
| 23 |
#include <soc/sifive/sifive_ccache.h> |
| 24 |
|
| 25 |
#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 |
| 26 |
#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 |
| 27 |
#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 |
| 28 |
|
| 29 |
#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 |
| 30 |
#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 |
| 31 |
#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 |
| 32 |
|
| 33 |
#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 |
| 34 |
#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 |
| 35 |
#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 |
| 36 |
|
| 37 |
#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 |
| 38 |
#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 |
| 39 |
#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 |
| 40 |
|
| 41 |
#define SIFIVE_CCACHE_CONFIG 0x00 |
| 42 |
#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) |
| 43 |
#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) |
| 44 |
#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) |
| 45 |
#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) |
| 46 |
|
| 47 |
#define SIFIVE_CCACHE_FLUSH64 0x200 |
| 48 |
#define SIFIVE_CCACHE_FLUSH32 0x240 |
| 49 |
|
| 50 |
#define SIFIVE_CCACHE_WAYENABLE 0x08 |
| 51 |
#define SIFIVE_CCACHE_ECCINJECTERR 0x40 |
| 52 |
|
| 53 |
#define SIFIVE_CCACHE_MAX_ECCINTR 4 |
| 54 |
#define SIFIVE_CCACHE_LINE_SIZE 64 |
| 55 |
|
| 56 |
static void __iomem *ccache_base; |
| 57 |
static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; |
| 58 |
static struct riscv_cacheinfo_ops ccache_cache_ops; |
| 59 |
static int level; |
| 60 |
|
| 61 |
enum { |
| 62 |
DIR_CORR = 0, |
| 63 |
DATA_CORR, |
| 64 |
DATA_UNCORR, |
| 65 |
DIR_UNCORR, |
| 66 |
}; |
| 67 |
|
| 68 |
enum { |
| 69 |
QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), |
| 70 |
QUIRK_BROKEN_DATA_UNCORR = BIT(1), |
| 71 |
}; |
| 72 |
|
| 73 |
#ifdef CONFIG_DEBUG_FS |
| 74 |
static struct dentry *sifive_test; |
| 75 |
|
| 76 |
static ssize_t ccache_write(struct file *file, const char __user *data, |
| 77 |
size_t count, loff_t *ppos) |
| 78 |
{ |
| 79 |
unsigned int val; |
| 80 |
|
| 81 |
if (kstrtouint_from_user(data, count, 0, &val)) |
| 82 |
return -EINVAL; |
| 83 |
if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) |
| 84 |
writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); |
| 85 |
else |
| 86 |
return -EINVAL; |
| 87 |
return count; |
| 88 |
} |
| 89 |
|
| 90 |
static const struct file_operations ccache_fops = { |
| 91 |
.owner = THIS_MODULE, |
| 92 |
.open = simple_open, |
| 93 |
.write = ccache_write |
| 94 |
}; |
| 95 |
|
| 96 |
static void setup_sifive_debug(void) |
| 97 |
{ |
| 98 |
sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); |
| 99 |
|
| 100 |
debugfs_create_file("sifive_debug_inject_error", 0200, |
| 101 |
sifive_test, NULL, &ccache_fops); |
| 102 |
} |
| 103 |
#endif |
| 104 |
|
| 105 |
static void ccache_config_read(void) |
| 106 |
{ |
| 107 |
u32 cfg; |
| 108 |
|
| 109 |
cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); |
| 110 |
pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", |
| 111 |
FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), |
| 112 |
FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), |
| 113 |
BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), |
| 114 |
BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); |
| 115 |
|
| 116 |
cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); |
| 117 |
pr_info("Index of the largest way enabled: %u\n", cfg); |
| 118 |
} |
| 119 |
|
| 120 |
static const struct of_device_id sifive_ccache_ids[] = { |
| 121 |
{ .compatible = "eswin,eic7700-l3-cache", |
| 122 |
.data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) }, |
| 123 |
{ .compatible = "sifive,fu540-c000-ccache" }, |
| 124 |
{ .compatible = "sifive,fu740-c000-ccache" }, |
| 125 |
{ .compatible = "starfive,jh7100-ccache", |
| 126 |
.data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, |
| 127 |
{ .compatible = "sifive,ccache0" }, |
| 128 |
{ /* end of table */ } |
| 129 |
}; |
| 130 |
|
| 131 |
static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); |
| 132 |
|
| 133 |
int register_sifive_ccache_error_notifier(struct notifier_block *nb) |
| 134 |
{ |
| 135 |
return atomic_notifier_chain_register(&ccache_err_chain, nb); |
| 136 |
} |
| 137 |
EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); |
| 138 |
|
| 139 |
int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) |
| 140 |
{ |
| 141 |
return atomic_notifier_chain_unregister(&ccache_err_chain, nb); |
| 142 |
} |
| 143 |
EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); |
| 144 |
|
| 145 |
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
| 146 |
static void ccache_flush_range(phys_addr_t start, size_t len) |
| 147 |
{ |
| 148 |
phys_addr_t end = start + len; |
| 149 |
phys_addr_t line; |
| 150 |
|
| 151 |
if (!len) |
| 152 |
return; |
| 153 |
|
| 154 |
mb(); /* complete earlier memory accesses before the cache flush */ |
| 155 |
for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; |
| 156 |
line += SIFIVE_CCACHE_LINE_SIZE) { |
| 157 |
#ifdef CONFIG_32BIT |
| 158 |
writel_relaxed(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); |
| 159 |
#else |
| 160 |
writeq_relaxed(line, ccache_base + SIFIVE_CCACHE_FLUSH64); |
| 161 |
#endif |
| 162 |
} |
| 163 |
mb(); /* issue later memory accesses after the cache flush */ |
| 164 |
} |
| 165 |
|
| 166 |
static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { |
| 167 |
.wback = &ccache_flush_range, |
| 168 |
.inv = &ccache_flush_range, |
| 169 |
.wback_inv = &ccache_flush_range, |
| 170 |
}; |
| 171 |
#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ |
| 172 |
|
| 173 |
static int ccache_largest_wayenabled(void) |
| 174 |
{ |
| 175 |
return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; |
| 176 |
} |
| 177 |
|
| 178 |
static ssize_t number_of_ways_enabled_show(struct device *dev, |
| 179 |
struct device_attribute *attr, |
| 180 |
char *buf) |
| 181 |
{ |
| 182 |
return sprintf(buf, "%u\n", ccache_largest_wayenabled()); |
| 183 |
} |
| 184 |
|
| 185 |
static DEVICE_ATTR_RO(number_of_ways_enabled); |
| 186 |
|
| 187 |
static struct attribute *priv_attrs[] = { |
| 188 |
&dev_attr_number_of_ways_enabled.attr, |
| 189 |
NULL, |
| 190 |
}; |
| 191 |
|
| 192 |
static const struct attribute_group priv_attr_group = { |
| 193 |
.attrs = priv_attrs, |
| 194 |
}; |
| 195 |
|
| 196 |
static const struct attribute_group *ccache_get_priv_group(struct cacheinfo |
| 197 |
*this_leaf) |
| 198 |
{ |
| 199 |
/* We want to use private group for composable cache only */ |
| 200 |
if (this_leaf->level == level) |
| 201 |
return &priv_attr_group; |
| 202 |
else |
| 203 |
return NULL; |
| 204 |
} |
| 205 |
|
| 206 |
static irqreturn_t ccache_int_handler(int irq, void *device) |
| 207 |
{ |
| 208 |
unsigned int add_h, add_l; |
| 209 |
|
| 210 |
if (irq == g_irq[DIR_CORR]) { |
| 211 |
add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); |
| 212 |
add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); |
| 213 |
pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); |
| 214 |
/* Reading this register clears the DirError interrupt sig */ |
| 215 |
readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); |
| 216 |
atomic_notifier_call_chain(&ccache_err_chain, |
| 217 |
SIFIVE_CCACHE_ERR_TYPE_CE, |
| 218 |
"DirECCFix"); |
| 219 |
} |
| 220 |
if (irq == g_irq[DIR_UNCORR]) { |
| 221 |
add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); |
| 222 |
add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); |
| 223 |
/* Reading this register clears the DirFail interrupt sig */ |
| 224 |
readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); |
| 225 |
atomic_notifier_call_chain(&ccache_err_chain, |
| 226 |
SIFIVE_CCACHE_ERR_TYPE_UE, |
| 227 |
"DirECCFail"); |
| 228 |
panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); |
| 229 |
} |
| 230 |
if (irq == g_irq[DATA_CORR]) { |
| 231 |
add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); |
| 232 |
add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); |
| 233 |
pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); |
| 234 |
/* Reading this register clears the DataError interrupt sig */ |
| 235 |
readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); |
| 236 |
atomic_notifier_call_chain(&ccache_err_chain, |
| 237 |
SIFIVE_CCACHE_ERR_TYPE_CE, |
| 238 |
"DatECCFix"); |
| 239 |
} |
| 240 |
if (irq == g_irq[DATA_UNCORR]) { |
| 241 |
add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); |
| 242 |
add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); |
| 243 |
pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); |
| 244 |
/* Reading this register clears the DataFail interrupt sig */ |
| 245 |
readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); |
| 246 |
atomic_notifier_call_chain(&ccache_err_chain, |
| 247 |
SIFIVE_CCACHE_ERR_TYPE_UE, |
| 248 |
"DatECCFail"); |
| 249 |
} |
| 250 |
|
| 251 |
return IRQ_HANDLED; |
| 252 |
} |
| 253 |
|
| 254 |
static int sifive_ccache_probe(struct platform_device *pdev) |
| 255 |
{ |
| 256 |
struct device *dev = &pdev->dev; |
| 257 |
unsigned long quirks; |
| 258 |
int intr_num, rc; |
| 259 |
|
| 260 |
quirks = (unsigned long)device_get_match_data(dev); |
| 261 |
|
| 262 |
intr_num = platform_irq_count(pdev); |
| 263 |
if (!intr_num) |
| 264 |
return dev_err_probe(dev, -ENODEV, "No interrupts property\n"); |
| 265 |
|
| 266 |
for (int i = 0; i < intr_num; i++) { |
| 267 |
if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) |
| 268 |
continue; |
| 269 |
|
| 270 |
g_irq[i] = platform_get_irq(pdev, i); |
| 271 |
if (g_irq[i] < 0) |
| 272 |
return g_irq[i]; |
| 273 |
|
| 274 |
rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL); |
| 275 |
if (rc) |
| 276 |
return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]); |
| 277 |
} |
| 278 |
|
| 279 |
return 0; |
| 280 |
} |
| 281 |
|
| 282 |
static struct platform_driver sifive_ccache_driver = { |
| 283 |
.probe = sifive_ccache_probe, |
| 284 |
.driver = { |
| 285 |
.name = "sifive_ccache", |
| 286 |
.of_match_table = sifive_ccache_ids, |
| 287 |
}, |
| 288 |
}; |
| 289 |
|
| 290 |
static int __init sifive_ccache_init(void) |
| 291 |
{ |
| 292 |
struct device_node *np; |
| 293 |
struct resource res; |
| 294 |
const struct of_device_id *match; |
| 295 |
unsigned long quirks __maybe_unused; |
| 296 |
int rc; |
| 297 |
|
| 298 |
np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); |
| 299 |
if (!np) |
| 300 |
return -ENODEV; |
| 301 |
|
| 302 |
quirks = (uintptr_t)match->data; |
| 303 |
|
| 304 |
if (of_address_to_resource(np, 0, &res)) { |
| 305 |
rc = -ENODEV; |
| 306 |
goto err_node_put; |
| 307 |
} |
| 308 |
|
| 309 |
ccache_base = ioremap(res.start, resource_size(&res)); |
| 310 |
if (!ccache_base) { |
| 311 |
rc = -ENOMEM; |
| 312 |
goto err_node_put; |
| 313 |
} |
| 314 |
|
| 315 |
if (of_property_read_u32(np, "cache-level", &level)) { |
| 316 |
rc = -ENOENT; |
| 317 |
goto err_unmap; |
| 318 |
} |
| 319 |
|
| 320 |
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
| 321 |
if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { |
| 322 |
riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; |
| 323 |
riscv_noncoherent_supported(); |
| 324 |
riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); |
| 325 |
} |
| 326 |
#endif |
| 327 |
|
| 328 |
ccache_config_read(); |
| 329 |
|
| 330 |
ccache_cache_ops.get_priv_group = ccache_get_priv_group; |
| 331 |
riscv_set_cacheinfo_ops(&ccache_cache_ops); |
| 332 |
|
| 333 |
#ifdef CONFIG_DEBUG_FS |
| 334 |
setup_sifive_debug(); |
| 335 |
#endif |
| 336 |
|
| 337 |
rc = platform_driver_register(&sifive_ccache_driver); |
| 338 |
if (rc) |
| 339 |
goto err_unmap; |
| 340 |
|
| 341 |
of_node_put(np); |
| 342 |
|
| 343 |
return 0; |
| 344 |
|
| 345 |
err_unmap: |
| 346 |
iounmap(ccache_base); |
| 347 |
err_node_put: |
| 348 |
of_node_put(np); |
| 349 |
return rc; |
| 350 |
} |
| 351 |
|
| 352 |
arch_initcall(sifive_ccache_init); |