From 3bce00817098833b2c9fb9d74e812a0b3f92ad33 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 1 Dec 2020 04:57:09 -0800 Subject: [PATCH] kern: implement kmemoryblock/kmemoryinfo merge disable --- .../mesosphere/kern_k_memory_block.hpp | 222 +++++++++++++++--- .../kern_k_memory_block_manager.hpp | 6 +- .../source/kern_k_memory_block_manager.cpp | 137 ++++------- .../source/kern_k_page_table_base.cpp | 91 +++---- 4 files changed, 286 insertions(+), 170 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp index 08817d6c1..18e030681 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block.hpp @@ -176,15 +176,31 @@ namespace ams::kern { KMemoryAttribute_SetMask = KMemoryAttribute_Uncached, }; + enum KMemoryBlockDisableMergeAttribute : u8 { + KMemoryBlockDisableMergeAttribute_None = 0, + KMemoryBlockDisableMergeAttribute_Normal = (1u << 0), + KMemoryBlockDisableMergeAttribute_DeviceLeft = (1u << 1), + KMemoryBlockDisableMergeAttribute_IpcLeft = (1u << 2), + KMemoryBlockDisableMergeAttribute_Locked = (1u << 3), + KMemoryBlockDisableMergeAttribute_DeviceRight = (1u << 4), + + KMemoryBlockDisableMergeAttribute_AllLeft = KMemoryBlockDisableMergeAttribute_Normal | KMemoryBlockDisableMergeAttribute_DeviceLeft | KMemoryBlockDisableMergeAttribute_IpcLeft | KMemoryBlockDisableMergeAttribute_Locked, + KMemoryBlockDisableMergeAttribute_AllRight = KMemoryBlockDisableMergeAttribute_DeviceRight, + }; + struct KMemoryInfo { uintptr_t address; size_t size; KMemoryState state; + u16 device_disable_merge_left_count; + u16 device_disable_merge_right_count; + u16 ipc_lock_count; + u16 device_use_count; + u16 ipc_disable_merge_count; KMemoryPermission perm; KMemoryAttribute attribute; KMemoryPermission original_perm; - u16 ipc_lock_count; - u16 device_use_count; + KMemoryBlockDisableMergeAttribute disable_merge_attribute; constexpr ams::svc::MemoryInfo GetSvcMemoryInfo() const { return { @@ -223,6 +239,10 @@ namespace ams::kern { return this->ipc_lock_count; } + constexpr u16 GetIpcDisableMergeCount() const { + return this->ipc_disable_merge_count; + } + constexpr KMemoryState GetState() const { return this->state; } @@ -238,18 +258,26 @@ namespace ams::kern { constexpr KMemoryAttribute GetAttribute() const { return this->attribute; } + + constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const { + return this->disable_merge_attribute; + } }; class KMemoryBlock : public util::IntrusiveRedBlackTreeBaseNode { private: + u16 device_disable_merge_left_count; + u16 device_disable_merge_right_count; KProcessAddress address; size_t num_pages; KMemoryState memory_state; u16 ipc_lock_count; u16 device_use_count; + u16 ipc_disable_merge_count; KMemoryPermission perm; KMemoryPermission original_perm; KMemoryAttribute attribute; + KMemoryBlockDisableMergeAttribute disable_merge_attribute; public: static constexpr ALWAYS_INLINE int Compare(const KMemoryBlock &lhs, const KMemoryBlock &rhs) { if (lhs.GetAddress() < rhs.GetAddress()) { @@ -285,6 +313,10 @@ namespace ams::kern { return this->ipc_lock_count; } + constexpr u16 GetIpcDisableMergeCount() const { + return this->ipc_disable_merge_count; + } + constexpr KMemoryPermission GetPermission() const { return this->perm; } @@ -299,25 +331,29 @@ namespace ams::kern { constexpr KMemoryInfo GetMemoryInfo() const { return { - .address = GetInteger(this->GetAddress()), - .size = this->GetSize(), - .state = this->memory_state, - .perm = this->perm, - .attribute = this->attribute, - .original_perm = this->original_perm, - .ipc_lock_count = this->ipc_lock_count, - .device_use_count = this->device_use_count, + .address = GetInteger(this->GetAddress()), + .size = this->GetSize(), + .state = this->memory_state, + .device_disable_merge_left_count = this->device_disable_merge_left_count, + .device_disable_merge_right_count = this->device_disable_merge_right_count, + .ipc_lock_count = this->ipc_lock_count, + .device_use_count = this->device_use_count, + .ipc_disable_merge_count = this->ipc_disable_merge_count, + .perm = this->perm, + .attribute = this->attribute, + .original_perm = this->original_perm, + .disable_merge_attribute = this->disable_merge_attribute, }; } public: constexpr KMemoryBlock() - : address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), perm(), original_perm(), attribute() + : device_disable_merge_left_count(), device_disable_merge_right_count(), address(), num_pages(), memory_state(KMemoryState_None), ipc_lock_count(), device_use_count(), ipc_disable_merge_count(), perm(), original_perm(), attribute(), disable_merge_attribute() { /* ... */ } constexpr KMemoryBlock(KProcessAddress addr, size_t np, KMemoryState ms, KMemoryPermission p, KMemoryAttribute attr) - : address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), perm(p), original_perm(KMemoryPermission_None), attribute(attr) + : device_disable_merge_left_count(), device_disable_merge_right_count(), address(addr), num_pages(np), memory_state(ms), ipc_lock_count(0), device_use_count(0), perm(p), ipc_disable_merge_count(), original_perm(KMemoryPermission_None), attribute(attr), disable_merge_attribute() { /* ... */ } @@ -351,21 +387,29 @@ namespace ams::kern { this->device_use_count == rhs.device_use_count; } + constexpr bool CanMergeWith(const KMemoryBlock &rhs) const { + return this->HasSameProperties(rhs) && + (this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight) == 0 && + (rhs.disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft) == 0; + } + constexpr bool Contains(KProcessAddress addr) const { MESOSPHERE_ASSERT_THIS(); return this->GetAddress() <= addr && addr <= this->GetEndAddress(); } - constexpr void Add(size_t np) { + constexpr void Add(const KMemoryBlock &added_block) { MESOSPHERE_ASSERT_THIS(); - MESOSPHERE_ASSERT(np > 0); - MESOSPHERE_ASSERT(this->GetAddress() + np * PageSize - 1 < this->GetEndAddress() + np * PageSize - 1); + MESOSPHERE_ASSERT(added_block.GetNumPages() > 0); + MESOSPHERE_ASSERT(this->GetAddress() + added_block.GetSize() - 1 < this->GetEndAddress() + added_block.GetSize() - 1); - this->num_pages += np; + this->num_pages += added_block.GetNumPages(); + this->disable_merge_attribute = static_cast(this->disable_merge_attribute | added_block.disable_merge_attribute); + this->device_disable_merge_right_count = added_block.device_disable_merge_right_count; } - constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) { + constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a, bool set_disable_merge_attr, u8 set_mask, u8 clear_mask) { MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT(this->original_perm == KMemoryPermission_None); MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == 0); @@ -373,6 +417,13 @@ namespace ams::kern { this->memory_state = s; this->perm = p; this->attribute = static_cast(a | (this->attribute & (KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared))); + + if (set_disable_merge_attr && set_mask != 0) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute | set_mask); + } + if (clear_mask != 0) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute & ~clear_mask); + } } constexpr void Split(KMemoryBlock *block, KProcessAddress addr) { @@ -381,20 +432,55 @@ namespace ams::kern { MESOSPHERE_ASSERT(this->Contains(addr)); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), PageSize)); - block->address = this->address; - block->num_pages = (addr - this->GetAddress()) / PageSize; - block->memory_state = this->memory_state; - block->ipc_lock_count = this->ipc_lock_count; - block->device_use_count = this->device_use_count; - block->perm = this->perm; - block->original_perm = this->original_perm; - block->attribute = this->attribute; + block->address = this->address; + block->num_pages = (addr - this->GetAddress()) / PageSize; + block->memory_state = this->memory_state; + block->ipc_lock_count = this->ipc_lock_count; + block->device_use_count = this->device_use_count; + block->perm = this->perm; + block->original_perm = this->original_perm; + block->attribute = this->attribute; + block->disable_merge_attribute = static_cast(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllLeft); + block->ipc_disable_merge_count = this->ipc_disable_merge_count; + block->device_disable_merge_left_count = this->device_disable_merge_left_count; + block->device_disable_merge_right_count = 0; this->address = addr; this->num_pages -= block->num_pages; + + this->ipc_disable_merge_count = 0; + this->device_disable_merge_left_count = 0; + this->disable_merge_attribute = static_cast(this->disable_merge_attribute & KMemoryBlockDisableMergeAttribute_AllRight); } - constexpr void ShareToDevice(KMemoryPermission new_perm) { + constexpr void UpdateDeviceDisableMergeStateForShareLeft(KMemoryPermission new_perm, bool left, bool right) { + /* New permission/right aren't used. */ + MESOSPHERE_UNUSED(new_perm, right); + + if (left) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceLeft); + const u16 new_device_disable_merge_left_count = ++this->device_disable_merge_left_count; + MESOSPHERE_ABORT_UNLESS(device_disable_merge_left_count > 0); + } + } + + constexpr void UpdateDeviceDisableMergeStateForShareRight(KMemoryPermission new_perm, bool left, bool right) { + /* New permission/left aren't used. */ + MESOSPHERE_UNUSED(new_perm, left); + + if (right) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_DeviceRight); + const u16 new_device_disable_merge_right_count = ++this->device_disable_merge_right_count; + MESOSPHERE_ABORT_UNLESS(new_device_disable_merge_right_count > 0); + } + } + + constexpr void UpdateDeviceDisableMergeStateForShare(KMemoryPermission new_perm, bool left, bool right) { + this->UpdateDeviceDisableMergeStateForShareLeft(new_perm, left, right); + this->UpdateDeviceDisableMergeStateForShareRight(new_perm, left, right); + } + + constexpr void ShareToDevice(KMemoryPermission new_perm, bool left, bool right) { /* New permission isn't used. */ MESOSPHERE_UNUSED(new_perm); @@ -406,9 +492,47 @@ namespace ams::kern { MESOSPHERE_ABORT_UNLESS(new_count > 0); this->attribute = static_cast(this->attribute | KMemoryAttribute_DeviceShared); + + this->UpdateDeviceDisableMergeStateForShare(new_perm, left, right); } - constexpr void UnshareToDevice(KMemoryPermission new_perm) { + constexpr void UpdateDeviceDisableMergeStateForUnshareLeft(KMemoryPermission new_perm, bool left, bool right) { + /* New permission/right aren't used. */ + MESOSPHERE_UNUSED(new_perm, right); + + if (left) { + if (!this->device_disable_merge_left_count) { + return; + } + --this->device_disable_merge_left_count; + } + + this->device_disable_merge_left_count = std::min(this->device_disable_merge_left_count, this->device_use_count); + + if (this->device_disable_merge_left_count == 0) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceLeft); + } + } + + constexpr void UpdateDeviceDisableMergeStateForUnshareRight(KMemoryPermission new_perm, bool left, bool right) { + /* New permission/left aren't used. */ + MESOSPHERE_UNUSED(new_perm, left); + + if (right) { + const u16 old_device_disable_merge_right_count = this->device_disable_merge_right_count--; + MESOSPHERE_ASSERT(old_device_disable_merge_right_count > 0); + if (old_device_disable_merge_right_count == 1) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_DeviceRight); + } + } + } + + constexpr void UpdateDeviceDisableMergeStateForUnshare(KMemoryPermission new_perm, bool left, bool right) { + this->UpdateDeviceDisableMergeStateForUnshareLeft(new_perm, left, right); + this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right); + } + + constexpr void UnshareToDevice(KMemoryPermission new_perm, bool left, bool right) { /* New permission isn't used. */ MESOSPHERE_UNUSED(new_perm); @@ -422,9 +546,29 @@ namespace ams::kern { if (old_count == 1) { this->attribute = static_cast(this->attribute & ~KMemoryAttribute_DeviceShared); } + + this->UpdateDeviceDisableMergeStateForUnshare(new_perm, left, right); } - constexpr void LockForIpc(KMemoryPermission new_perm) { + constexpr void UnshareToDeviceRight(KMemoryPermission new_perm, bool left, bool right) { + /* New permission isn't used. */ + MESOSPHERE_UNUSED(new_perm); + + /* We must be shared. */ + MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_DeviceShared) == KMemoryAttribute_DeviceShared); + + /* Unhare. */ + const u16 old_count = this->device_use_count--; + MESOSPHERE_ABORT_UNLESS(old_count > 0); + + if (old_count == 1) { + this->attribute = static_cast(this->attribute & ~KMemoryAttribute_DeviceShared); + } + + this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right); + } + + constexpr void LockForIpc(KMemoryPermission new_perm, bool left, bool right) { /* We must either be locked or have a zero lock count. */ MESOSPHERE_ASSERT((this->attribute & KMemoryAttribute_IpcLocked) == KMemoryAttribute_IpcLocked || this->ipc_lock_count == 0); @@ -441,9 +585,16 @@ namespace ams::kern { this->perm = static_cast((new_perm & KMemoryPermission_IpcLockChangeMask) | (this->original_perm & ~KMemoryPermission_IpcLockChangeMask)); } this->attribute = static_cast(this->attribute | KMemoryAttribute_IpcLocked); + + if (left) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute | KMemoryBlockDisableMergeAttribute_IpcLeft); + const u16 new_ipc_disable_merge_count = ++this->ipc_disable_merge_count; + MESOSPHERE_ABORT_UNLESS(new_ipc_disable_merge_count > 0); + } + MESOSPHERE_UNUSED(right); } - constexpr void UnlockForIpc(KMemoryPermission new_perm) { + constexpr void UnlockForIpc(KMemoryPermission new_perm, bool left, bool right) { /* New permission isn't used. */ MESOSPHERE_UNUSED(new_perm); @@ -461,6 +612,19 @@ namespace ams::kern { this->original_perm = KMemoryPermission_None; this->attribute = static_cast(this->attribute & ~KMemoryAttribute_IpcLocked); } + + if (left) { + const u16 old_ipc_disable_merge_count = this->ipc_disable_merge_count--; + MESOSPHERE_ASSERT(old_ipc_disable_merge_count > 0); + if (old_ipc_disable_merge_count == 1) { + this->disable_merge_attribute = static_cast(this->disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute_IpcLeft); + } + } + MESOSPHERE_UNUSED(right); + } + + constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const { + return this->disable_merge_attribute; } }; static_assert(std::is_trivially_destructible::value); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp index 7f71b853b..18f919b9a 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_block_manager.hpp @@ -82,6 +82,8 @@ namespace ams::kern { MemoryBlockTree memory_block_tree; KProcessAddress start_address; KProcessAddress end_address; + private: + void CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages); public: constexpr KMemoryBlockManager() : memory_block_tree(), start_address(), end_address() { /* ... */ } @@ -94,8 +96,8 @@ namespace ams::kern { KProcessAddress FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; - void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr); - void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm), KMemoryPermission perm); + void Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr); + void UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm, bool left, bool right), KMemoryPermission perm); void UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr); diff --git a/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp b/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp index df93139db..ce0bbbed4 100644 --- a/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_block_manager.cpp @@ -147,7 +147,35 @@ namespace ams::kern { return Null; } - void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr) { + void KMemoryBlockManager::CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages) { + /* Find the iterator now that we've updated. */ + iterator it = this->FindIterator(address); + if (address != this->start_address) { + it--; + } + + /* Coalesce blocks that we can. */ + while (true) { + iterator prev = it++; + if (it == this->memory_block_tree.end()) { + break; + } + + if (prev->CanMergeWith(*it)) { + KMemoryBlock *block = std::addressof(*it); + this->memory_block_tree.erase(it); + prev->Add(*block); + allocator->Free(block); + it = prev; + } + + if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) { + break; + } + } + } + + void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); @@ -193,39 +221,14 @@ namespace ams::kern { } /* Update block state. */ - it->Update(state, perm, attr); + it->Update(state, perm, attr, cur_address == address, set_disable_attr, clear_disable_attr); cur_address += cur_info.GetSize(); remaining_pages -= cur_info.GetNumPages(); } it++; } - /* Find the iterator now that we've updated. */ - it = this->FindIterator(address); - if (address != this->start_address) { - it--; - } - - /* Coalesce blocks that we can. */ - while (true) { - iterator prev = it++; - if (it == this->memory_block_tree.end()) { - break; - } - - if (prev->HasSameProperties(*it)) { - KMemoryBlock *block = std::addressof(*it); - const size_t pages = it->GetNumPages(); - this->memory_block_tree.erase(it); - allocator->Free(block); - prev->Add(pages); - it = prev; - } - - if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) { - break; - } - } + this->CoalesceForUpdate(allocator, address, num_pages); } void KMemoryBlockManager::UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr) { @@ -265,7 +268,7 @@ namespace ams::kern { } /* Update block state. */ - it->Update(state, perm, attr); + it->Update(state, perm, attr, false, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); cur_address += cur_info.GetSize(); remaining_pages -= cur_info.GetNumPages(); } else { @@ -281,35 +284,10 @@ namespace ams::kern { it++; } - /* Find the iterator now that we've updated. */ - it = this->FindIterator(address); - if (address != this->start_address) { - it--; - } - - /* Coalesce blocks that we can. */ - while (true) { - iterator prev = it++; - if (it == this->memory_block_tree.end()) { - break; - } - - if (prev->HasSameProperties(*it)) { - KMemoryBlock *block = std::addressof(*it); - const size_t pages = it->GetNumPages(); - this->memory_block_tree.erase(it); - allocator->Free(block); - prev->Add(pages); - it = prev; - } - - if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) { - break; - } - } + this->CoalesceForUpdate(allocator, address, num_pages); } - void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm), KMemoryPermission perm) { + void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, void (KMemoryBlock::*lock_func)(KMemoryPermission new_perm, bool left, bool right), KMemoryPermission perm) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); @@ -317,8 +295,8 @@ namespace ams::kern { KProcessAddress cur_address = address; size_t remaining_pages = num_pages; iterator it = this->FindIterator(address); - iterator prev = it, next = it; - bool check_coalesce_prev = false, check_coalesce_next = false; + + const KProcessAddress end_address = address + (num_pages * PageSize); while (remaining_pages > 0) { const size_t remaining_size = remaining_pages * PageSize; @@ -334,10 +312,6 @@ namespace ams::kern { cur_info = it->GetMemoryInfo(); cur_address = cur_info.GetAddress(); - } else if (cur_address == address && cur_address != this->start_address) { - /* If there's a previous, we should check for coalescing. */ - check_coalesce_prev = true; - prev--; } if (cur_info.GetSize() > remaining_size) { @@ -348,47 +322,16 @@ namespace ams::kern { it = this->memory_block_tree.insert(*new_block); cur_info = it->GetMemoryInfo(); - } else if (cur_info.GetSize() == remaining_size) { - /* Otherwise if we can map precisely, we may need to check for coalescing against next block. */ - next = it; - ++next; - if (next != this->memory_block_tree.end()) { - check_coalesce_next = true; - } } /* Call the locked update function. */ - (std::addressof(*it)->*lock_func)(perm); + (std::addressof(*it)->*lock_func)(perm, cur_info.GetAddress() == address, cur_info.GetEndAddress() == end_address); cur_address += cur_info.GetSize(); remaining_pages -= cur_info.GetNumPages(); it++; } - /* If we should try to coalesce prev, do so. */ - if (check_coalesce_prev) { - it = prev; - it++; - if (prev->HasSameProperties(*it)) { - KMemoryBlock *block = std::addressof(*it); - const size_t pages = it->GetNumPages(); - this->memory_block_tree.erase(it); - allocator->Free(block); - prev->Add(pages); - } - } - - /* If we should try to coalesce next, do so. */ - if (check_coalesce_next) { - it = next; - it--; - if (it->HasSameProperties(*next)) { - KMemoryBlock *block = std::addressof(*next); - const size_t pages = next->GetNumPages(); - this->memory_block_tree.erase(next); - allocator->Free(block); - it->Add(pages); - } - } + this->CoalesceForUpdate(allocator, address, num_pages); } /* Debug. */ @@ -403,8 +346,8 @@ namespace ams::kern { const KMemoryInfo prev_info = prev->GetMemoryInfo(); const KMemoryInfo cur_info = it->GetMemoryInfo(); - /* Sequential blocks with same properties should be coalesced. */ - if (prev->HasSameProperties(*it)) { + /* Sequential blocks which can be merged should be merged. */ + if (prev->CanMergeWith(*it)) { return false; } diff --git a/libraries/libmesosphere/source/kern_k_page_table_base.cpp b/libraries/libmesosphere/source/kern_k_page_table_base.cpp index 0ba9b8442..7087707f3 100644 --- a/libraries/libmesosphere/source/kern_k_page_table_base.cpp +++ b/libraries/libmesosphere/source/kern_k_page_table_base.cpp @@ -544,7 +544,7 @@ namespace ams::kern { } /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, new_attr); + this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, new_attr, KMemoryBlockDisableMergeAttribute_Locked, KMemoryBlockDisableMergeAttribute_None); /* If we have an output group, open. */ if (out_pg) { @@ -598,7 +598,7 @@ namespace ams::kern { } /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, new_attr); + this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, new_attr, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Locked); return ResultSuccess(); } @@ -741,8 +741,8 @@ namespace ams::kern { unprot_guard.Cancel(); /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, new_src_perm, new_src_attr); - this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_Stack, KMemoryPermission_UserReadWrite, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, new_src_perm, new_src_attr, KMemoryBlockDisableMergeAttribute_Locked, KMemoryBlockDisableMergeAttribute_None); + this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_Stack, KMemoryPermission_UserReadWrite, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); } return ResultSuccess(); @@ -804,8 +804,8 @@ namespace ams::kern { remap_guard.Cancel(); /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, KMemoryPermission_UserReadWrite, KMemoryAttribute_None); - this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, KMemoryPermission_UserReadWrite, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Locked); + this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); } return ResultSuccess(); @@ -869,8 +869,8 @@ namespace ams::kern { unprot_guard.Cancel(); /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, new_perm, KMemoryAttribute_Locked); - this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_AliasCode, new_perm, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, new_perm, KMemoryAttribute_Locked, KMemoryBlockDisableMergeAttribute_Locked, KMemoryBlockDisableMergeAttribute_None); + this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_AliasCode, new_perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); } return ResultSuccess(); @@ -965,8 +965,8 @@ namespace ams::kern { remap_guard.Cancel(); /* Apply the memory block updates. */ - this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None); - this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); + this->memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Locked); /* Note that we reprotected pages. */ reprotected_pages = true; @@ -1100,7 +1100,7 @@ namespace ams::kern { MESOSPHERE_ABORT_UNLESS(map_end_address != map_address); /* Determine if we should disable head merge. */ - const bool disable_head_merge = info.GetAddress() >= GetInteger(start_address) /* TODO */; + const bool disable_head_merge = info.GetAddress() >= GetInteger(start_address) && (info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute_Normal) != 0; const KPageProperties map_properties = { info.GetPermission(), false, false, disable_head_merge ? DisableMergeAttribute_DisableHead : DisableMergeAttribute_None }; /* While we have pages to map, map them. */ @@ -1311,7 +1311,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, Null, false, properties, OperationType_ChangePermissions, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, old_state, new_perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, old_state, new_perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); return ResultSuccess(); } @@ -1368,7 +1368,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, Null, false, properties, operation, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, new_state, new_perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, new_state, new_perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); /* Ensure cache coherency, if we're setting pages as executable. */ if (is_x) { @@ -1415,7 +1415,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, Null, false, properties, OperationType_ChangePermissionsAndRefresh, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, old_state, old_perm, new_attr); + this->memory_block_manager.Update(&allocator, addr, num_pages, old_state, old_perm, new_attr, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); return ResultSuccess(); } @@ -1463,7 +1463,7 @@ namespace ams::kern { GetCurrentProcess().ReleaseResource(ams::svc::LimitableResource_PhysicalMemoryMax, num_pages * PageSize); /* Apply the memory block update. */ - this->memory_block_manager.Update(std::addressof(allocator), this->heap_region_start + size, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), this->heap_region_start + size, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, size == 0 ? KMemoryBlockDisableMergeAttribute_Normal : KMemoryBlockDisableMergeAttribute_None); /* Update the current heap end. */ this->current_heap_end = this->heap_region_start + size; @@ -1528,7 +1528,7 @@ namespace ams::kern { memory_reservation.Commit(); /* Apply the memory block update. */ - this->memory_block_manager.Update(std::addressof(allocator), this->current_heap_end, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), this->current_heap_end, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None, this->heap_region_start == this->current_heap_end ? KMemoryBlockDisableMergeAttribute_Normal : KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); /* Update the current heap end. */ this->current_heap_end = this->heap_region_start + size; @@ -1555,14 +1555,18 @@ namespace ams::kern { /* If the address is invalid, create a fake block. */ if (!this->Contains(addr, 1)) { *out_info = { - .address = GetInteger(this->address_space_end), - .size = 0 - GetInteger(this->address_space_end), - .state = static_cast(ams::svc::MemoryState_Inaccessible), - .perm = KMemoryPermission_None, - .attribute = KMemoryAttribute_None, - .original_perm = KMemoryPermission_None, - .ipc_lock_count = 0, - .device_use_count = 0, + .address = GetInteger(this->address_space_end), + .size = 0 - GetInteger(this->address_space_end), + .state = static_cast(ams::svc::MemoryState_Inaccessible), + .device_disable_merge_left_count = 0, + .device_disable_merge_right_count = 0, + .ipc_lock_count = 0, + .device_use_count = 0, + .ipc_disable_merge_count = 0, + .perm = KMemoryPermission_None, + .attribute = KMemoryAttribute_None, + .original_perm = KMemoryPermission_None, + .disable_merge_attribute = KMemoryBlockDisableMergeAttribute_None, }; out_page_info->flags = 0; @@ -1724,7 +1728,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, phys_addr, true, properties, OperationType_Map, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, KMemoryState_Io, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, KMemoryState_Io, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* We successfully mapped the pages. */ return ResultSuccess(); @@ -1791,7 +1795,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), addr, num_pages, phys_addr, true, properties, OperationType_Map, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, KMemoryState_Static, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, KMemoryState_Static, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* We successfully mapped the pages. */ return ResultSuccess(); @@ -1843,7 +1847,7 @@ namespace ams::kern { } /* Update the blocks. */ - this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* We successfully mapped the pages. */ *out_addr = addr; @@ -1873,7 +1877,7 @@ namespace ams::kern { R_TRY(this->AllocateAndMapPagesImpl(updater.GetPageList(), address, num_pages, perm)); /* Update the blocks. */ - this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); return ResultSuccess(); } @@ -1902,7 +1906,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), address, num_pages, Null, false, unmap_properties, OperationType_Unmap, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); return ResultSuccess(); } @@ -1936,7 +1940,7 @@ namespace ams::kern { R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, state, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, state, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* We successfully mapped the pages. */ *out_addr = addr; @@ -1970,7 +1974,7 @@ namespace ams::kern { R_TRY(this->MapPageGroupImpl(updater.GetPageList(), addr, pg, properties, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, addr, num_pages, state, perm, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, addr, num_pages, state, perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* We successfully mapped the pages. */ return ResultSuccess(); @@ -2006,7 +2010,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), address, num_pages, Null, false, properties, OperationType_Unmap, false)); /* Update the blocks. */ - this->memory_block_manager.Update(&allocator, address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(&allocator, address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); return ResultSuccess(); } @@ -3261,7 +3265,7 @@ namespace ams::kern { } /* Update memory blocks to reflect our changes */ - this->memory_block_manager.Update(std::addressof(allocator), dst_addr, aligned_src_size / PageSize, dst_state, test_perm, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), dst_addr, aligned_src_size / PageSize, dst_state, test_perm, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* Set the output address. */ *out_addr = dst_addr + (src_start - aligned_src_start); @@ -3357,7 +3361,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), aligned_start, aligned_num_pages, Null, false, unmap_properties, OperationType_Unmap, false)); /* Update memory blocks. */ - this->memory_block_manager.Update(std::addressof(allocator), aligned_start, aligned_num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), aligned_start, aligned_num_pages, KMemoryState_None, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); /* Release from the resource limit as relevant. */ if (auto *resource_limit = server_process->GetResourceLimit(); resource_limit != nullptr) { @@ -3431,7 +3435,7 @@ namespace ams::kern { size_t cur_size = cur_info.GetSize(); bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission(); bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1; - bool first = false /* TODO */; + bool first = cur_info.GetIpcDisableMergeCount() == 1 && (cur_info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute_Locked) == 0; while ((GetInteger(cur_address) + cur_size - 1) < mapped_last) { /* Check that we have a next block. */ @@ -3490,7 +3494,7 @@ namespace ams::kern { size_t cur_size = cur_info.GetSize(); bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission(); bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1; - bool first = false /* TODO */; + bool first = cur_info.GetIpcDisableMergeCount() == 1 && (cur_info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute_Locked) == 0; while ((cur_address + cur_size - 1) < mapping_last) { /* Check that we have a next block. */ @@ -3532,7 +3536,7 @@ namespace ams::kern { } /* Process the last block. */ - const auto lock_count = cur_info.GetIpcLockCount() /* TODO */; + const auto lock_count = cur_info.GetIpcLockCount() + (next_it != this->memory_block_manager.end() ? (next_it->GetIpcDisableMergeCount() - next_it->GetIpcLockCount()) : 0); if ((first || cur_needs_set_perm || (lock_count == 1)) && !cur_perm_eq) { const DisableMergeAttribute head_body_attr = first ? DisableMergeAttribute_EnableHeadAndBody : DisableMergeAttribute_None; const DisableMergeAttribute tail_attr = lock_count == 1 ? DisableMergeAttribute_EnableTail : DisableMergeAttribute_None; @@ -3582,12 +3586,15 @@ namespace ams::kern { { /* Check if we actually need to fix the protections on the block. */ if (cur_end == src_map_end || info.GetAddress() <= GetInteger(src_map_start) || (info.GetPermission() & KMemoryPermission_IpcLockChangeMask) != prot_perm) { - const bool start_nc = (info.GetAddress() == GetInteger(src_map_start)) ? (/* TODO */ true) : info.GetAddress() <= GetInteger(src_map_start); + const bool start_nc = (info.GetAddress() == GetInteger(src_map_start)) ? ((info.GetDisableMergeAttribute() & (KMemoryBlockDisableMergeAttribute_Locked | KMemoryBlockDisableMergeAttribute_IpcLeft)) == 0) : info.GetAddress() <= GetInteger(src_map_start); const DisableMergeAttribute head_body_attr = start_nc ? DisableMergeAttribute_EnableHeadAndBody : DisableMergeAttribute_None; DisableMergeAttribute tail_attr; if (cur_end == src_map_end && info.GetEndAddress() == src_map_end) { - const auto lock_count = info.GetIpcLockCount() /* TODO */; + auto next_it = it; + ++next_it; + + const auto lock_count = info.GetIpcLockCount() + (next_it != this->memory_block_manager.end() ? (next_it->GetIpcDisableMergeCount() - next_it->GetIpcLockCount()) : 0); tail_attr = lock_count == 0 ? DisableMergeAttribute_EnableTail : DisableMergeAttribute_None; } else { tail_attr = DisableMergeAttribute_None; @@ -4074,7 +4081,7 @@ namespace ams::kern { GetCurrentProcess().ReleaseResource(ams::svc::LimitableResource_PhysicalMemoryMax, mapped_size); /* Update memory blocks. */ - this->memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_None); /* We succeeded. */ remap_guard.Cancel(); @@ -4125,7 +4132,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), address, num_pages, pg, map_properties, OperationType_MapGroup, false)); /* Apply the memory block update. */ - this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState_Normal, KMemoryPermission_UserReadWrite, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_Normal, KMemoryBlockDisableMergeAttribute_None); /* Update our mapped unsafe size. */ this->mapped_unsafe_physical_memory += size; @@ -4160,7 +4167,7 @@ namespace ams::kern { R_TRY(this->Operate(updater.GetPageList(), address, num_pages, Null, false, unmap_properties, OperationType_Unmap, false)); /* Apply the memory block update. */ - this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); + this->memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); /* Release the unsafe memory from the limit. */ Kernel::GetUnsafeMemory().Release(size);