diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index 279eb2a0a..66be8d6d8 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -27,6 +27,9 @@ #include #include +/* Tracing functionality. */ +#include + /* Core pre-initialization includes. */ #include #include diff --git a/libraries/libmesosphere/include/mesosphere/kern_common.hpp b/libraries/libmesosphere/include/mesosphere/kern_common.hpp index e27cfc96e..b1123b397 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_common.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_common.hpp @@ -37,4 +37,6 @@ namespace ams::kern { #define MESOSPHERE_ENABLE_DEBUG_PRINT #endif +#define MESOSPHERE_BUILD_FOR_TRACING + #include diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index d70485b62..6f0d6c51f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -100,8 +100,6 @@ namespace ams::kern { KMemoryRegionType_DramSystemNonSecurePool = 0xDA6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, KMemoryRegionType_DramSystemPool = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected, - - KMemoryRegionType_DramKernel = 0xE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected, KMemoryRegionType_DramKernelCode = 0xCE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected, KMemoryRegionType_DramKernelSlab = 0x14E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected, @@ -526,6 +524,10 @@ namespace ams::kern { return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelTemp); } + static NOINLINE KMemoryRegion &GetKernelTraceBufferRegion() { + return *GetVirtualLinearMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualKernelTraceBuffer); + } + static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address)); } @@ -730,6 +732,10 @@ namespace ams::kern { return GetVirtualLinearExtents(GetLinearRegionPhysicalExtents()); } + static NOINLINE auto GetMainMemoryPhysicalExtents() { + return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); + } + static NOINLINE auto GetCarveoutRegionExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected); } @@ -777,6 +783,10 @@ namespace ams::kern { static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool); } + + static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() { + return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer); + } }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp new file mode 100644 index 000000000..ae4bd65b3 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::kern { + + #if defined(MESOSPHERE_BUILD_FOR_TRACING) + constexpr inline bool IsKTraceEnabled = true; + #else + constexpr inline bool IsKTraceEnabled = false; + #endif + + constexpr inline size_t KTraceBufferSize = IsKTraceEnabled ? 16_MB : 0; + + static_assert(IsKTraceEnabled || !IsKTraceEnabled); + + class KTrace { + private: + static bool s_is_active; + public: + static void Initialize(KVirtualAddress address, size_t size); + static void Start(); + static void Stop(); + + static ALWAYS_INLINE bool IsActive() { return s_is_active; } + }; + +} + +#define MESOSPHERE_KTRACE_RESUME() \ + ({ \ + if constexpr (::ams::kern::IsKTraceEnabled) { \ + ::ams::kern::KTrace::Start(); \ + } \ + }) + +#define MESOSPHERE_KTRACE_PAUSE() \ + ({ \ + if constexpr (::ams::kern::IsKTraceEnabled) { \ + ::ams::kern::KTrace::Stop(); \ + } \ + }) diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp index 3c73bc298..bf98bdd1d 100644 --- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp +++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp @@ -338,39 +338,52 @@ namespace ams::kern::board::nintendo::nx { } size_t KSystemControl::Init::GetApplicationPoolSize() { - switch (GetMemoryArrangeForInit()) { - case smc::MemoryArrangement_4GB: - default: - return 3285_MB; - case smc::MemoryArrangement_4GBForAppletDev: - return 2048_MB; - case smc::MemoryArrangement_4GBForSystemDev: - return 3285_MB; - case smc::MemoryArrangement_6GB: - return 4916_MB; - case smc::MemoryArrangement_6GBForAppletDev: - return 3285_MB; - case smc::MemoryArrangement_8GB: - return 4916_MB; - } + /* Get the base pool size. */ + const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t { + switch (GetMemoryArrangeForInit()) { + case smc::MemoryArrangement_4GB: + default: + return 3285_MB; + case smc::MemoryArrangement_4GBForAppletDev: + return 2048_MB; + case smc::MemoryArrangement_4GBForSystemDev: + return 3285_MB; + case smc::MemoryArrangement_6GB: + return 4916_MB; + case smc::MemoryArrangement_6GBForAppletDev: + return 3285_MB; + case smc::MemoryArrangement_8GB: + return 4916_MB; + } + }(); + + /* Return (possibly) adjusted size. */ + return base_pool_size; } size_t KSystemControl::Init::GetAppletPoolSize() { - switch (GetMemoryArrangeForInit()) { - case smc::MemoryArrangement_4GB: - default: - return 507_MB; - case smc::MemoryArrangement_4GBForAppletDev: - return 1554_MB; - case smc::MemoryArrangement_4GBForSystemDev: - return 448_MB; - case smc::MemoryArrangement_6GB: - return 562_MB; - case smc::MemoryArrangement_6GBForAppletDev: - return 2193_MB; - case smc::MemoryArrangement_8GB: - return 2193_MB; - } + /* Get the base pool size. */ + const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t { + switch (GetMemoryArrangeForInit()) { + case smc::MemoryArrangement_4GB: + default: + return 507_MB; + case smc::MemoryArrangement_4GBForAppletDev: + return 1554_MB; + case smc::MemoryArrangement_4GBForSystemDev: + return 448_MB; + case smc::MemoryArrangement_6GB: + return 562_MB; + case smc::MemoryArrangement_6GBForAppletDev: + return 2193_MB; + case smc::MemoryArrangement_8GB: + return 2193_MB; + } + }(); + + /* Return (possibly) adjusted size. */ + constexpr size_t ExtraSystemMemoryForAtmosphere = 33_MB; + return base_pool_size - ExtraSystemMemoryForAtmosphere - KTraceBufferSize; } size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() { @@ -461,6 +474,12 @@ namespace ams::kern::board::nintendo::nx { g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption); MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null); } + + /* Initialize KTrace. */ + if constexpr (IsKTraceEnabled) { + const auto &ktrace = KMemoryLayout::GetKernelTraceBufferRegion(); + KTrace::Initialize(ktrace.GetAddress(), ktrace.GetSize()); + } } u32 KSystemControl::GetInitialProcessBinaryPool() { diff --git a/libraries/libmesosphere/source/kern_k_memory_layout.cpp b/libraries/libmesosphere/source/kern_k_memory_layout.cpp index 6ea5a2c43..3f281d059 100644 --- a/libraries/libmesosphere/source/kern_k_memory_layout.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_layout.cpp @@ -227,7 +227,9 @@ namespace ams::kern { void SetupPoolPartitionMemoryRegions() { /* Start by identifying the extents of the DRAM memory region. */ - const auto dram_extents = KMemoryLayout::GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); + const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents(); + + const uintptr_t pool_end = dram_extents.GetEndAddress() - KTraceBufferSize; /* Get Application and Applet pool sizes. */ const size_t application_pool_size = KSystemControl::Init::GetApplicationPoolSize(); @@ -242,7 +244,7 @@ namespace ams::kern { const uintptr_t pool_partitions_start = KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_DramPoolPartition)->GetAddress(); /* Decide on starting addresses for our pools. */ - const uintptr_t application_pool_start = dram_extents.GetEndAddress() - application_pool_size; + const uintptr_t application_pool_start = pool_end - application_pool_size; const uintptr_t applet_pool_start = application_pool_start - applet_pool_size; const uintptr_t unsafe_system_pool_start = std::min(kernel_dram_start + CarveoutSizeMax, util::AlignDown(applet_pool_start - unsafe_system_pool_min_size, CarveoutAlignment)); const size_t unsafe_system_pool_size = applet_pool_start - unsafe_system_pool_start; diff --git a/libraries/libmesosphere/source/kern_k_trace.cpp b/libraries/libmesosphere/source/kern_k_trace.cpp new file mode 100644 index 000000000..15ab7f75a --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_trace.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + /* Static initializations. */ + constinit bool KTrace::s_is_active = false; + + namespace { + + constinit KSpinLock g_ktrace_lock; + constinit KVirtualAddress g_ktrace_buffer_address = Null; + constinit size_t g_ktrace_buffer_size = 0; + + struct KTraceHeader { + u32 magic; + u32 offset; + u32 index; + u32 count; + + static constexpr u32 Magic = util::FourCC<'K','T','R','0'>::Code; + }; + static_assert(util::is_pod::value); + + struct KTraceRecord { + u8 core_id; + u8 type; + u16 process_id; + u32 thread_id; + u64 tick; + u64 data[6]; + }; + static_assert(util::is_pod::value); + static_assert(sizeof(KTraceRecord) == 0x40); + + } + + void KTrace::Initialize(KVirtualAddress address, size_t size) { + /* Only perform tracing when on development hardware. */ + if (KTargetSystem::IsDebugMode()) { + const size_t offset = util::AlignUp(sizeof(KTraceHeader), sizeof(KTraceRecord)); + if (offset < size) { + /* Clear the trace buffer. */ + std::memset(GetVoidPointer(address), 0, size); + + /* Initialize the KTrace header. */ + KTraceHeader *header = GetPointer(address); + header->magic = KTraceHeader::Magic; + header->offset = offset; + header->index = 0; + header->count = (size - offset) / sizeof(KTraceRecord); + + /* Set the global data. */ + g_ktrace_buffer_address = address; + g_ktrace_buffer_size = size; + } + } + } + + void KTrace::Start() { + if (g_ktrace_buffer_address != Null) { + /* Get exclusive access to the trace buffer. */ + KScopedInterruptDisable di; + KScopedSpinLock lk(g_ktrace_lock); + + /* Reset the header. */ + KTraceHeader *header = GetPointer(g_ktrace_buffer_address); + header->index = 0; + + /* Reset the records. */ + KTraceRecord *records = GetPointer(g_ktrace_buffer_address + header->offset); + std::memset(records, 0, sizeof(*records) * header->count); + + /* Note that we're active. */ + s_is_active = true; + } + } + + void KTrace::Stop() { + if (g_ktrace_buffer_address != Null) { + /* Get exclusive access to the trace buffer. */ + KScopedInterruptDisable di; + KScopedSpinLock lk(g_ktrace_lock); + + /* Note that we're paused. */ + s_is_active = false; + } + } + +} diff --git a/libraries/libmesosphere/source/kern_kernel.cpp b/libraries/libmesosphere/source/kern_kernel.cpp index f9aa6f747..94171c726 100644 --- a/libraries/libmesosphere/source/kern_kernel.cpp +++ b/libraries/libmesosphere/source/kern_kernel.cpp @@ -157,6 +157,10 @@ namespace ams::kern { PrintMemoryRegion(" SystemUnsafe", KMemoryLayout::GetKernelSystemNonSecurePoolRegionPhysicalExtents()); PrintMemoryRegion(" Applet", KMemoryLayout::GetKernelAppletPoolRegionPhysicalExtents()); PrintMemoryRegion(" Application", KMemoryLayout::GetKernelApplicationPoolRegionPhysicalExtents()); + if constexpr (IsKTraceEnabled) { + MESOSPHERE_LOG(" Debug\n"); + PrintMemoryRegion(" Trace Buffer", KMemoryLayout::GetKernelTraceBufferRegionPhysicalExtents()); + } MESOSPHERE_LOG("\n"); } diff --git a/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp b/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp index b8ba28c6f..177f4a50d 100644 --- a/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp +++ b/libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp @@ -39,12 +39,12 @@ namespace ams::kern::svc { switch (kern_trace_state) { case ams::svc::KernelTraceState_Enabled: { - /* TODO: MESOSPHERE_KTRACE_RESUME(); */ + MESOSPHERE_KTRACE_RESUME(); } break; case ams::svc::KernelTraceState_Disabled: { - /* TODO: MESOSPHERE_KTRACE_PAUSE(); */ + MESOSPHERE_KTRACE_PAUSE(); } break; default: diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index 995dad19d..e578a591a 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -183,6 +183,11 @@ namespace ams::kern::init { MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug)); ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator); } + constexpr size_t MiscUnknownDebugRegionSize = PageSize; + constexpr size_t MiscUnknownDebugRegionAlign = PageSize; + const KVirtualAddress misc_unk_debug_virt_addr = KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(MiscUnknownDebugRegionSize, MiscUnknownDebugRegionAlign, KMemoryRegionType_KernelMisc, PageSize); + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug)); + ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator); /* Setup board-specific device physical regions. */ SetupDevicePhysicalMemoryRegions(); @@ -241,6 +246,14 @@ namespace ams::kern::init { /* Insert a physical region for the kernel page table heap region */ MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_end_phys_addr), page_table_heap_size, KMemoryRegionType_DramKernelPtHeap)); + /* Insert a physical region for the kernel trace buffer. */ + static_assert(!IsKTraceEnabled || KTraceBufferSize > 0); + if constexpr (IsKTraceEnabled) { + const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents(); + const KPhysicalAddress ktrace_buffer_phys_addr = dram_extents.GetEndAddress() - KTraceBufferSize; + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(ktrace_buffer_phys_addr), KTraceBufferSize, KMemoryRegionType_KernelTraceBuffer)); + } + /* All DRAM regions that we haven't tagged by this point will be mapped under the linear mapping. Tag them. */ for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) { if (region.GetType() == KMemoryRegionType_Dram) { @@ -258,17 +271,37 @@ namespace ams::kern::init { const uintptr_t linear_region_phys_to_virt_diff = GetInteger(linear_region_start) - GetInteger(aligned_linear_phys_start); /* Map and create regions for all the linearly-mapped data. */ - for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) { - if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) { - continue; + { + uintptr_t cur_phys_addr = 0; + uintptr_t cur_size = 0; + for (auto ®ion : KMemoryLayout::GetPhysicalMemoryRegionTree()) { + if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) { + continue; + } + + if (cur_phys_addr == 0) { + cur_phys_addr = region.GetAddress(); + cur_size = region.GetSize(); + } else if (cur_phys_addr + cur_size == region.GetAddress()) { + cur_size += region.GetSize(); + } else { + const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff; + ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator); + cur_phys_addr = region.GetAddress(); + cur_size = region.GetSize(); + } + + const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff; + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType()))); + region.SetPairAddress(region_virt_addr); + KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress()); } - const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff; - ttbr1_table.Map(region_virt_addr, region.GetSize(), region.GetAddress(), KernelRwDataAttribute, g_initial_page_allocator); - - MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType()))); - region.SetPairAddress(region_virt_addr); - KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress()); + /* Map the last block, which we may have skipped. */ + if (cur_phys_addr != 0) { + const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff; + ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator); + } } /* Create regions for and map all core-specific stacks. */