123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * SBI initialilization and all extension implementation.
- *
- * Copyright (c) 2020 Western Digital Corporation or its affiliates.
- */
- #include <linux/init.h>
- #include <linux/pm.h>
- #include <asm/sbi.h>
- #include <asm/smp.h>
- /* default SBI version is 0.1 */
- unsigned long sbi_spec_version = SBI_SPEC_VERSION_DEFAULT;
- EXPORT_SYMBOL(sbi_spec_version);
- static void (*__sbi_set_timer)(uint64_t stime);
- static int (*__sbi_send_ipi)(const unsigned long *hart_mask);
- static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask,
- unsigned long start, unsigned long size,
- unsigned long arg4, unsigned long arg5);
- struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
- unsigned long arg1, unsigned long arg2,
- unsigned long arg3, unsigned long arg4,
- unsigned long arg5)
- {
- struct sbiret ret;
- register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0);
- register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1);
- register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2);
- register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3);
- register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4);
- register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5);
- register uintptr_t a6 asm ("a6") = (uintptr_t)(fid);
- register uintptr_t a7 asm ("a7") = (uintptr_t)(ext);
- asm volatile ("ecall"
- : "+r" (a0), "+r" (a1)
- : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7)
- : "memory");
- ret.error = a0;
- ret.value = a1;
- return ret;
- }
- EXPORT_SYMBOL(sbi_ecall);
- int sbi_err_map_linux_errno(int err)
- {
- switch (err) {
- case SBI_SUCCESS:
- return 0;
- case SBI_ERR_DENIED:
- return -EPERM;
- case SBI_ERR_INVALID_PARAM:
- return -EINVAL;
- case SBI_ERR_INVALID_ADDRESS:
- return -EFAULT;
- case SBI_ERR_NOT_SUPPORTED:
- case SBI_ERR_FAILURE:
- default:
- return -ENOTSUPP;
- };
- }
- EXPORT_SYMBOL(sbi_err_map_linux_errno);
- #ifdef CONFIG_RISCV_SBI_V01
- /**
- * sbi_console_putchar() - Writes given character to the console device.
- * @ch: The data to be written to the console.
- *
- * Return: None
- */
- void sbi_console_putchar(int ch)
- {
- sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0);
- }
- EXPORT_SYMBOL(sbi_console_putchar);
- /**
- * sbi_console_getchar() - Reads a byte from console device.
- *
- * Returns the value read from console.
- */
- int sbi_console_getchar(void)
- {
- struct sbiret ret;
- ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0);
- return ret.error;
- }
- EXPORT_SYMBOL(sbi_console_getchar);
- /**
- * sbi_shutdown() - Remove all the harts from executing supervisor code.
- *
- * Return: None
- */
- extern void khv_shutdown(void);
- void sbi_shutdown(void)
- {
- #ifdef CONFIG_VIRTIO_KHV_MMIO
- khv_shutdown();
- #endif
- sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0);
- }
- EXPORT_SYMBOL(sbi_shutdown);
- /**
- * sbi_clear_ipi() - Clear any pending IPIs for the calling hart.
- *
- * Return: None
- */
- void sbi_clear_ipi(void)
- {
- sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0);
- }
- EXPORT_SYMBOL(sbi_clear_ipi);
- /**
- * sbi_set_timer_v01() - Program the timer for next timer event.
- * @stime_value: The value after which next timer event should fire.
- *
- * Return: None
- */
- static void __sbi_set_timer_v01(uint64_t stime_value)
- {
- #if __riscv_xlen == 32
- sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value,
- stime_value >> 32, 0, 0, 0, 0);
- #else
- sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0);
- #endif
- }
- static int __sbi_send_ipi_v01(const unsigned long *hart_mask)
- {
- sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask,
- 0, 0, 0, 0, 0);
- return 0;
- }
- static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask,
- unsigned long start, unsigned long size,
- unsigned long arg4, unsigned long arg5)
- {
- int result = 0;
- /* v0.2 function IDs are equivalent to v0.1 extension IDs */
- switch (fid) {
- case SBI_EXT_RFENCE_REMOTE_FENCE_I:
- sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0,
- (unsigned long)hart_mask, 0, 0, 0, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
- sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0,
- (unsigned long)hart_mask, start, size,
- 0, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
- sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0,
- (unsigned long)hart_mask, start, size,
- arg4, 0, 0);
- break;
- default:
- pr_err("SBI call [%d]not supported in SBI v0.1\n", fid);
- result = -EINVAL;
- }
- return result;
- }
- static void sbi_set_power_off(void)
- {
- pm_power_off = sbi_shutdown;
- }
- #else
- static void __sbi_set_timer_v01(uint64_t stime_value)
- {
- pr_warn("Timer extension is not available in SBI v%lu.%lu\n",
- sbi_major_version(), sbi_minor_version());
- }
- static int __sbi_send_ipi_v01(const unsigned long *hart_mask)
- {
- pr_warn("IPI extension is not available in SBI v%lu.%lu\n",
- sbi_major_version(), sbi_minor_version());
- return 0;
- }
- static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask,
- unsigned long start, unsigned long size,
- unsigned long arg4, unsigned long arg5)
- {
- pr_warn("remote fence extension is not available in SBI v%lu.%lu\n",
- sbi_major_version(), sbi_minor_version());
- return 0;
- }
- static void sbi_set_power_off(void) {}
- #endif /* CONFIG_RISCV_SBI_V01 */
- static void __sbi_set_timer_v02(uint64_t stime_value)
- {
- #if __riscv_xlen == 32
- sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value,
- stime_value >> 32, 0, 0, 0, 0);
- #else
- sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0,
- 0, 0, 0, 0);
- #endif
- }
- static int __sbi_send_ipi_v02(const unsigned long *hart_mask)
- {
- unsigned long hartid, hmask_val, hbase;
- struct cpumask tmask;
- struct sbiret ret = {0};
- int result;
- if (!hart_mask || !(*hart_mask)) {
- riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask);
- hart_mask = cpumask_bits(&tmask);
- }
- hmask_val = 0;
- hbase = 0;
- for_each_set_bit(hartid, hart_mask, NR_CPUS) {
- if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) {
- ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI,
- hmask_val, hbase, 0, 0, 0, 0);
- if (ret.error)
- goto ecall_failed;
- hmask_val = 0;
- hbase = 0;
- }
- if (!hmask_val)
- hbase = hartid;
- hmask_val |= 1UL << (hartid - hbase);
- }
- if (hmask_val) {
- ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI,
- hmask_val, hbase, 0, 0, 0, 0);
- if (ret.error)
- goto ecall_failed;
- }
- return 0;
- ecall_failed:
- result = sbi_err_map_linux_errno(ret.error);
- pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
- __func__, hbase, hmask_val, result);
- return result;
- }
- static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val,
- unsigned long hbase, unsigned long start,
- unsigned long size, unsigned long arg4,
- unsigned long arg5)
- {
- struct sbiret ret = {0};
- int ext = SBI_EXT_RFENCE;
- int result = 0;
- switch (fid) {
- case SBI_EXT_RFENCE_REMOTE_FENCE_I:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, arg4, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, arg4, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, 0, 0);
- break;
- case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID:
- ret = sbi_ecall(ext, fid, hmask_val, hbase, start,
- size, arg4, 0);
- break;
- default:
- pr_err("unknown function ID [%lu] for SBI extension [%d]\n",
- fid, ext);
- result = -EINVAL;
- }
- if (ret.error) {
- result = sbi_err_map_linux_errno(ret.error);
- pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
- __func__, hbase, hmask_val, result);
- }
- return result;
- }
- static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask,
- unsigned long start, unsigned long size,
- unsigned long arg4, unsigned long arg5)
- {
- unsigned long hmask_val, hartid, hbase;
- struct cpumask tmask;
- int result;
- if (!hart_mask || !(*hart_mask)) {
- riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask);
- hart_mask = cpumask_bits(&tmask);
- }
- hmask_val = 0;
- hbase = 0;
- for_each_set_bit(hartid, hart_mask, NR_CPUS) {
- if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) {
- result = __sbi_rfence_v02_call(fid, hmask_val, hbase,
- start, size, arg4, arg5);
- if (result)
- return result;
- hmask_val = 0;
- hbase = 0;
- }
- if (!hmask_val)
- hbase = hartid;
- hmask_val |= 1UL << (hartid - hbase);
- }
- if (hmask_val) {
- result = __sbi_rfence_v02_call(fid, hmask_val, hbase,
- start, size, arg4, arg5);
- if (result)
- return result;
- }
- return 0;
- }
- /**
- * sbi_set_timer() - Program the timer for next timer event.
- * @stime_value: The value after which next timer event should fire.
- *
- * Return: None
- */
- void sbi_set_timer(uint64_t stime_value)
- {
- __sbi_set_timer(stime_value);
- }
- /**
- * sbi_send_ipi() - Send an IPI to any hart.
- * @hart_mask: A cpu mask containing all the target harts.
- *
- * Return: None
- */
- void sbi_send_ipi(const unsigned long *hart_mask)
- {
- __sbi_send_ipi(hart_mask);
- }
- EXPORT_SYMBOL(sbi_send_ipi);
- /**
- * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts.
- * @hart_mask: A cpu mask containing all the target harts.
- *
- * Return: None
- */
- void sbi_remote_fence_i(const unsigned long *hart_mask)
- {
- __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I,
- hart_mask, 0, 0, 0, 0);
- }
- EXPORT_SYMBOL(sbi_remote_fence_i);
- /**
- * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote
- * harts for the specified virtual address range.
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the virtual address
- * @size: Total size of the virtual address range.
- *
- * Return: None
- */
- void sbi_remote_sfence_vma(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size)
- {
- __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA,
- hart_mask, start, size, 0, 0);
- }
- EXPORT_SYMBOL(sbi_remote_sfence_vma);
- /**
- * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given
- * remote harts for a virtual address range belonging to a specific ASID.
- *
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the virtual address
- * @size: Total size of the virtual address range.
- * @asid: The value of address space identifier (ASID).
- *
- * Return: None
- */
- void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size,
- unsigned long asid)
- {
- __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID,
- hart_mask, start, size, asid, 0);
- }
- EXPORT_SYMBOL(sbi_remote_sfence_vma_asid);
- /**
- * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote
- * harts for the specified guest physical address range.
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the guest physical address
- * @size: Total size of the guest physical address range.
- *
- * Return: None
- */
- int sbi_remote_hfence_gvma(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size)
- {
- return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA,
- hart_mask, start, size, 0, 0);
- }
- EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma);
- /**
- * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given
- * remote harts for a guest physical address range belonging to a specific VMID.
- *
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the guest physical address
- * @size: Total size of the guest physical address range.
- * @vmid: The value of guest ID (VMID).
- *
- * Return: 0 if success, Error otherwise.
- */
- int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size,
- unsigned long vmid)
- {
- return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID,
- hart_mask, start, size, vmid, 0);
- }
- EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid);
- /**
- * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote
- * harts for the current guest virtual address range.
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the current guest virtual address
- * @size: Total size of the current guest virtual address range.
- *
- * Return: None
- */
- int sbi_remote_hfence_vvma(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size)
- {
- return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA,
- hart_mask, start, size, 0, 0);
- }
- EXPORT_SYMBOL(sbi_remote_hfence_vvma);
- /**
- * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given
- * remote harts for current guest virtual address range belonging to a specific
- * ASID.
- *
- * @hart_mask: A cpu mask containing all the target harts.
- * @start: Start of the current guest virtual address
- * @size: Total size of the current guest virtual address range.
- * @asid: The value of address space identifier (ASID).
- *
- * Return: None
- */
- int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask,
- unsigned long start,
- unsigned long size,
- unsigned long asid)
- {
- return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID,
- hart_mask, start, size, asid, 0);
- }
- EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid);
- /**
- * sbi_probe_extension() - Check if an SBI extension ID is supported or not.
- * @extid: The extension ID to be probed.
- *
- * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise.
- */
- int sbi_probe_extension(int extid)
- {
- struct sbiret ret;
- ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid,
- 0, 0, 0, 0, 0);
- if (!ret.error)
- if (ret.value)
- return ret.value;
- return -ENOTSUPP;
- }
- EXPORT_SYMBOL(sbi_probe_extension);
- static long __sbi_base_ecall(int fid)
- {
- struct sbiret ret;
- ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0);
- if (!ret.error)
- return ret.value;
- else
- return sbi_err_map_linux_errno(ret.error);
- }
- static inline long sbi_get_spec_version(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION);
- }
- static inline long sbi_get_firmware_id(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID);
- }
- static inline long sbi_get_firmware_version(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION);
- }
- long sbi_get_mvendorid(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID);
- }
- long sbi_get_marchid(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID);
- }
- long sbi_get_mimpid(void)
- {
- return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID);
- }
- static void sbi_send_cpumask_ipi(const struct cpumask *target)
- {
- struct cpumask hartid_mask;
- riscv_cpuid_to_hartid_mask(target, &hartid_mask);
- sbi_send_ipi(cpumask_bits(&hartid_mask));
- }
- static struct riscv_ipi_ops sbi_ipi_ops = {
- .ipi_inject = sbi_send_cpumask_ipi
- };
- int __init sbi_init(void)
- {
- int ret;
- sbi_set_power_off();
- ret = sbi_get_spec_version();
- if (ret > 0)
- sbi_spec_version = ret;
- pr_info("SBI specification v%lu.%lu detected\n",
- sbi_major_version(), sbi_minor_version());
- if (!sbi_spec_is_0_1()) {
- pr_info("SBI implementation ID=0x%lx Version=0x%lx\n",
- sbi_get_firmware_id(), sbi_get_firmware_version());
- if (sbi_probe_extension(SBI_EXT_TIME) > 0) {
- __sbi_set_timer = __sbi_set_timer_v02;
- pr_info("SBI v0.2 TIME extension detected\n");
- } else {
- __sbi_set_timer = __sbi_set_timer_v01;
- }
- if (sbi_probe_extension(SBI_EXT_IPI) > 0) {
- __sbi_send_ipi = __sbi_send_ipi_v02;
- pr_info("SBI v0.2 IPI extension detected\n");
- } else {
- __sbi_send_ipi = __sbi_send_ipi_v01;
- }
- if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) {
- __sbi_rfence = __sbi_rfence_v02;
- pr_info("SBI v0.2 RFENCE extension detected\n");
- } else {
- __sbi_rfence = __sbi_rfence_v01;
- }
- } else {
- __sbi_set_timer = __sbi_set_timer_v01;
- __sbi_send_ipi = __sbi_send_ipi_v01;
- __sbi_rfence = __sbi_rfence_v01;
- }
- riscv_set_ipi_ops(&sbi_ipi_ops);
- return 0;
- }
|