From bc96ebb74c0afdb56438135498e81b87d8f7922f Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Sun, 10 Oct 2021 00:33:52 -0700 Subject: [PATCH] os: add FlushDataCache --- .../ncm/ncm_content_manager_impl.hpp | 2 +- .../include/stratosphere/os.hpp | 1 + .../include/stratosphere/os/os_cache.hpp | 24 +++++++++ .../source/os/impl/os_cache_impl.hpp | 23 ++++++++ .../os/impl/os_cache_impl.os.horizon.hpp | 54 +++++++++++++++++++ .../libstratosphere/source/os/os_cache.cpp | 29 ++++++++++ stratosphere/spl/source/spl_api_impl.cpp | 36 ++++++------- 7 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_cache.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_cache_impl.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_cache_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/source/os/os_cache.cpp diff --git a/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_manager_impl.hpp b/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_manager_impl.hpp index ed5f69e14..01026705d 100644 --- a/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_manager_impl.hpp +++ b/libraries/libstratosphere/include/stratosphere/ncm/ncm_content_manager_impl.hpp @@ -37,7 +37,7 @@ namespace ams::ncm { size_t m_peak_total_alloc_size; size_t m_peak_alloc_size; public: - explicit ContentMetaMemoryResource(void *heap, size_t heap_size) : m_allocator(heap, heap_size), m_peak_alloc_size(0), m_peak_total_alloc_size(0) { /* ... */ } + explicit ContentMetaMemoryResource(void *heap, size_t heap_size) : m_allocator(heap, heap_size), m_peak_total_alloc_size(0), m_peak_alloc_size(0) { /* ... */ } mem::StandardAllocator *GetAllocator() { return std::addressof(m_allocator); } size_t GetPeakTotalAllocationSize() const { return m_peak_total_alloc_size; } diff --git a/libraries/libstratosphere/include/stratosphere/os.hpp b/libraries/libstratosphere/include/stratosphere/os.hpp index c1e05ba46..42ab77ea8 100644 --- a/libraries/libstratosphere/include/stratosphere/os.hpp +++ b/libraries/libstratosphere/include/stratosphere/os.hpp @@ -53,3 +53,4 @@ #include #include #include +#include diff --git a/libraries/libstratosphere/include/stratosphere/os/os_cache.hpp b/libraries/libstratosphere/include/stratosphere/os/os_cache.hpp new file mode 100644 index 000000000..67e4b0464 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_cache.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os { + + void FlushDataCache(const void *addr, size_t size); + void FlushEntireDataCache(); + +} diff --git a/libraries/libstratosphere/source/os/impl/os_cache_impl.hpp b/libraries/libstratosphere/source/os/impl/os_cache_impl.hpp new file mode 100644 index 000000000..7b2830a3e --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_cache_impl.hpp @@ -0,0 +1,23 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +#ifdef ATMOSPHERE_OS_HORIZON + #include "os_cache_impl.os.horizon.hpp" +#else + #error "Unknown OS for ThreadManagerImpl" +#endif diff --git a/libraries/libstratosphere/source/os/impl/os_cache_impl.os.horizon.hpp b/libraries/libstratosphere/source/os/impl/os_cache_impl.os.horizon.hpp new file mode 100644 index 000000000..46562b203 --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_cache_impl.os.horizon.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os::impl { + + inline void FlushDataCacheImpl(const void *addr, size_t size) { + #if defined(ATMOSPHERE_ARCH_ARM64) + { + /* Declare helper variables. */ + uintptr_t cache_type_register = 0; + uintptr_t cache_line_size = 0; + const uintptr_t end_addr = reinterpret_cast(addr) + size; + + /* Get the cache type register. */ + __asm__ __volatile__("mrs %[cache_type_register], ctr_el0" : [cache_type_register]"=r"(cache_type_register)); + + /* Calculate cache line size. */ + cache_line_size = 4 << ((cache_type_register >> 16) & 0xF); + + /* Iterate, flushing cache lines. */ + for (uintptr_t cur = reinterpret_cast(addr) & ~(cache_line_size - 1); cur < end_addr; cur += cache_line_size) { + __asm__ __volatile__ ("dc civac, %[cur]" :: [cur]"r"(cur)); + } + + /* Insert a memory barrier, now that memory has been flushed. */ + __asm__ __volatile__("dsb sy" ::: "memory"); + } + #else + const auto result = svc::FlushProcessDataCache(svc::PseudoHandle::CurrentProcess, reinterpret_cast(addr), size); + R_ASSERT(result); + AMS_UNUSED(result); + #endif + } + + inline void FlushEntireDataCacheImpl() { + svc::FlushEntireDataCache(); + } + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/os/os_cache.cpp b/libraries/libstratosphere/source/os/os_cache.cpp new file mode 100644 index 000000000..65a27c226 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_cache.cpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/os_cache_impl.hpp" + +namespace ams::os { + + void FlushDataCache(const void *addr, size_t size) { + return impl::FlushDataCacheImpl(addr, size); + } + + void FlushEntireDataCache() { + return impl::FlushEntireDataCacheImpl(); + } + +} diff --git a/stratosphere/spl/source/spl_api_impl.cpp b/stratosphere/spl/source/spl_api_impl.cpp index ad4b4bd5b..c7892db4e 100644 --- a/stratosphere/spl/source/spl_api_impl.cpp +++ b/stratosphere/spl/source/spl_api_impl.cpp @@ -372,7 +372,7 @@ namespace ams::spl::impl { std::memcpy(layout->in_block, src, sizeof(layout->in_block)); - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); { std::scoped_lock lk(g_async_op_lock); smc::AsyncOperationKey op_key; @@ -390,7 +390,7 @@ namespace ams::spl::impl { return res; } } - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); std::memcpy(dst, layout->out_block, sizeof(layout->out_block)); return smc::Result::Success; @@ -407,7 +407,7 @@ namespace ams::spl::impl { R_UNLESS(src_size <= sizeof(DecryptAndStoreDeviceUniqueKeyLayout), spl::ResultInvalidSize()); std::memcpy(layout, src, src_size); - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); smc::Result smc_res; if (hos::GetVersion() >= hos::Version_5_0_0) { smc_res = smc::DecryptDeviceUniqueData(layout->data, src_size, access_key, key_source, static_cast(option)); @@ -438,7 +438,7 @@ namespace ams::spl::impl { std::memcpy(layout->mod + mod_ofs, mod, mod_size); /* Do exp mod operation. */ - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); { std::scoped_lock lk(g_async_op_lock); smc::AsyncOperationKey op_key; @@ -452,7 +452,7 @@ namespace ams::spl::impl { return smc::ConvertResult(res); } } - armDCacheFlush(g_work_buffer, sizeof(out_size)); + os::FlushDataCache(g_work_buffer, sizeof(out_size)); std::memcpy(out, g_work_buffer, out_size); return ResultSuccess(); @@ -478,7 +478,7 @@ namespace ams::spl::impl { std::memcpy(layout->mod + mod_ofs, mod, mod_size); /* Do exp mod operation. */ - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); { std::scoped_lock lk(g_async_op_lock); smc::AsyncOperationKey op_key; @@ -492,7 +492,7 @@ namespace ams::spl::impl { return smc::ConvertResult(res); } } - armDCacheFlush(g_work_buffer, sizeof(*out_access_key)); + os::FlushDataCache(g_work_buffer, sizeof(*out_access_key)); std::memcpy(out_access_key, g_work_buffer, sizeof(*out_access_key)); return ResultSuccess(); @@ -566,7 +566,7 @@ namespace ams::spl::impl { std::memcpy(layout->mod + mod_ofs, mod, mod_size); /* Do exp mod operation. */ - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); { std::scoped_lock lk(g_async_op_lock); smc::AsyncOperationKey op_key; @@ -580,7 +580,7 @@ namespace ams::spl::impl { return smc::ConvertResult(res); } } - armDCacheFlush(g_work_buffer, sizeof(out_size)); + os::FlushDataCache(g_work_buffer, sizeof(out_size)); std::memcpy(out, g_work_buffer, out_size); return ResultSuccess(); @@ -701,9 +701,9 @@ namespace ams::spl::impl { crypt_ctx->out.address = dst_se_addr; crypt_ctx->out.size = dst_size; - armDCacheFlush(crypt_ctx, sizeof(*crypt_ctx)); - armDCacheFlush(const_cast(src), src_size); - armDCacheFlush(dst, dst_size); + os::FlushDataCache(crypt_ctx, sizeof(*crypt_ctx)); + os::FlushDataCache(const_cast(src), src_size); + os::FlushDataCache(dst, dst_size); { std::scoped_lock lk(g_async_op_lock); smc::AsyncOperationKey op_key; @@ -720,7 +720,7 @@ namespace ams::spl::impl { return smc::ConvertResult(res); } } - armDCacheFlush(dst, dst_size); + os::FlushDataCache(dst, dst_size); return ResultSuccess(); } @@ -783,7 +783,7 @@ namespace ams::spl::impl { R_UNLESS(src_size <= sizeof(DecryptDeviceUniqueDataLayout), spl::ResultInvalidSize()); std::memcpy(layout->data, src, src_size); - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); smc::Result smc_res; size_t copy_size = 0; @@ -795,7 +795,7 @@ namespace ams::spl::impl { copy_size = std::min(dst_size, copy_size); } - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); if (smc_res == smc::Result::Success) { std::memcpy(dst, layout->data, copy_size); } @@ -827,7 +827,7 @@ namespace ams::spl::impl { std::memcpy(layout, src, src_size); - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); return smc::ConvertResult(smc::LoadEsDeviceKey(layout->data, src_size, access_key, key_source, option)); } } @@ -915,12 +915,12 @@ namespace ams::spl::impl { layout->access_key_enc = access_key_enc; layout->source_enc = source_enc; - armDCacheFlush(layout, sizeof(*layout)); + os::FlushDataCache(layout, sizeof(*layout)); smc::Result smc_res = smc::ReencryptDeviceUniqueData(layout->data, src_size, layout->access_key_dec, layout->source_dec, layout->access_key_enc, layout->source_enc, option); if (smc_res == smc::Result::Success) { size_t copy_size = std::min(dst_size, src_size); - armDCacheFlush(layout, copy_size); + os::FlushDataCache(layout, copy_size); std::memcpy(dst, layout->data, copy_size); }