diff options
Diffstat (limited to 'src/core/hle/kernel/k_page_table.cpp')
| -rw-r--r-- | src/core/hle/kernel/k_page_table.cpp | 1350 |
1 files changed, 1048 insertions, 302 deletions
diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 912853e5c..d975de844 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -1,6 +1,5 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later #include "common/alignment.h" #include "common/assert.h" @@ -10,7 +9,7 @@ #include "core/hle/kernel/k_address_space_info.h" #include "core/hle/kernel/k_memory_block.h" #include "core/hle/kernel/k_memory_block_manager.h" -#include "core/hle/kernel/k_page_linked_list.h" +#include "core/hle/kernel/k_page_group.h" #include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_resource_limit.h" @@ -36,29 +35,11 @@ constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceT case FileSys::ProgramAddressSpaceType::Is39Bit: return 39; default: - UNREACHABLE(); + ASSERT(false); return {}; } } -constexpr u64 GetAddressInRange(const KMemoryInfo& info, VAddr addr) { - if (info.GetAddress() < addr) { - return addr; - } - return info.GetAddress(); -} - -constexpr std::size_t GetSizeInRange(const KMemoryInfo& info, VAddr start, VAddr end) { - std::size_t size{info.GetSize()}; - if (info.GetAddress() < start) { - size -= start - info.GetAddress(); - } - if (info.GetEndAddress() > end) { - size -= info.GetEndAddress() - end; - } - return size; -} - } // namespace KPageTable::KPageTable(Core::System& system_) @@ -66,9 +47,9 @@ KPageTable::KPageTable(Core::System& system_) KPageTable::~KPageTable() = default; -ResultCode KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, - bool enable_aslr, VAddr code_addr, - std::size_t code_size, KMemoryManager::Pool pool) { +Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, + VAddr code_addr, std::size_t code_size, + KMemoryManager::Pool pool) { const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { return KAddressSpaceInfo::GetAddressSpaceStart(address_space_width, type); @@ -84,7 +65,6 @@ ResultCode KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_ std::size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; std::size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; - ASSERT(start <= code_addr); ASSERT(code_addr < code_addr + code_size); ASSERT(code_addr + code_size - 1 <= end - 1); @@ -147,7 +127,7 @@ ResultCode KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_ const std::size_t needed_size{ (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; if (alloc_size < needed_size) { - UNREACHABLE(); + ASSERT(false); return ResultOutOfMemory; } @@ -277,8 +257,8 @@ ResultCode KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_ return InitializeMemoryLayout(start, end); } -ResultCode KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, - KMemoryPermission perm) { +Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, + KMemoryPermission perm) { const u64 size{num_pages * PageSize}; // Validate the mapping request. @@ -291,89 +271,367 @@ ResultCode KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemory R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + KPageGroup pg; + R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + &pg, num_pages, + KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, allocation_option))); - KPageLinkedList page_linked_list; - R_TRY(system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool, - allocation_option)); - R_TRY(Operate(addr, num_pages, page_linked_list, OperationType::MapGroup)); + R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); block_manager->Update(addr, num_pages, state, perm); return ResultSuccess; } -ResultCode KPageTable::MapCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { +Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size) { + // Validate the mapping request. + R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), + ResultInvalidMemoryRegion); + + // Lock the table. KScopedLightLock lk(general_lock); - const std::size_t num_pages{size / PageSize}; + // Verify that the source memory is normal heap. + KMemoryState src_state{}; + KMemoryPermission src_perm{}; + std::size_t num_src_allocator_blocks{}; + R_TRY(this->CheckMemoryState(&src_state, &src_perm, nullptr, &num_src_allocator_blocks, + src_address, size, KMemoryState::All, KMemoryState::Normal, + KMemoryPermission::All, KMemoryPermission::UserReadWrite, + KMemoryAttribute::All, KMemoryAttribute::None)); - KMemoryState state{}; - KMemoryPermission perm{}; - CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, nullptr, src_addr, size, - KMemoryState::All, KMemoryState::Normal, KMemoryPermission::All, - KMemoryPermission::UserReadWrite, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); + // Verify that the destination memory is unmapped. + std::size_t num_dst_allocator_blocks{}; + R_TRY(this->CheckMemoryState(&num_dst_allocator_blocks, dst_address, size, KMemoryState::All, + KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); - if (IsRegionMapped(dst_addr, size)) { - return ResultInvalidCurrentMemory; + // Map the code memory. + { + // Determine the number of pages being operated on. + const std::size_t num_pages = size / PageSize; + + // Create page groups for the memory being mapped. + KPageGroup pg; + AddRegionToPages(src_address, num_pages, pg); + + // Reprotect the source as kernel-read/not mapped. + const auto new_perm = static_cast<KMemoryPermission>(KMemoryPermission::KernelRead | + KMemoryPermission::NotMapped); + R_TRY(Operate(src_address, num_pages, new_perm, OperationType::ChangePermissions)); + + // Ensure that we unprotect the source pages on failure. + auto unprot_guard = SCOPE_GUARD({ + ASSERT(this->Operate(src_address, num_pages, src_perm, OperationType::ChangePermissions) + .IsSuccess()); + }); + + // Map the alias pages. + R_TRY(MapPages(dst_address, pg, new_perm)); + + // We successfully mapped the alias pages, so we don't need to unprotect the src pages on + // failure. + unprot_guard.Cancel(); + + // Apply the memory block updates. + block_manager->Update(src_address, num_pages, src_state, new_perm, + KMemoryAttribute::Locked); + block_manager->Update(dst_address, num_pages, KMemoryState::AliasCode, new_perm, + KMemoryAttribute::None); } - KPageLinkedList page_linked_list; - AddRegionToPages(src_addr, num_pages, page_linked_list); + return ResultSuccess; +} + +Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, + ICacheInvalidationStrategy icache_invalidation_strategy) { + // Validate the mapping request. + R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), + ResultInvalidMemoryRegion); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Verify that the source memory is locked normal heap. + std::size_t num_src_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_src_allocator_blocks), src_address, size, + KMemoryState::All, KMemoryState::Normal, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::Locked)); + + // Verify that the destination memory is aliasable code. + std::size_t num_dst_allocator_blocks{}; + R_TRY(this->CheckMemoryStateContiguous( + std::addressof(num_dst_allocator_blocks), dst_address, size, KMemoryState::FlagCanCodeAlias, + KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::All, KMemoryAttribute::None)); + // Determine whether any pages being unmapped are code. + bool any_code_pages = false; { - auto block_guard = detail::ScopeExit( - [&] { Operate(src_addr, num_pages, perm, OperationType::ChangePermissions); }); + KMemoryBlockManager::const_iterator it = block_manager->FindIterator(dst_address); + while (true) { + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + // Check if the memory has code flag. + if ((info.GetState() & KMemoryState::FlagCode) != KMemoryState::None) { + any_code_pages = true; + break; + } - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, - OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::None)); + // Check if we're done. + if (dst_address + size - 1 <= info.GetLastAddress()) { + break; + } - block_guard.Cancel(); + // Advance. + ++it; + } } - block_manager->Update(src_addr, num_pages, state, KMemoryPermission::None, - KMemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, KMemoryState::AliasCode); + // Ensure that we maintain the instruction cache. + bool reprotected_pages = false; + SCOPE_EXIT({ + if (reprotected_pages && any_code_pages) { + if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { + system.InvalidateCpuInstructionCacheRange(dst_address, size); + } else { + system.InvalidateCpuInstructionCaches(); + } + } + }); + + // Unmap. + { + // Determine the number of pages being operated on. + const std::size_t num_pages = size / PageSize; + + // Unmap the aliased copy of the pages. + R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + + // Try to set the permissions for the source pages back to what they should be. + R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, + OperationType::ChangePermissions)); + + // Apply the memory block updates. + block_manager->Update(dst_address, num_pages, KMemoryState::None); + block_manager->Update(src_address, num_pages, KMemoryState::Normal, + KMemoryPermission::UserReadWrite); + + // Note that we reprotected pages. + reprotected_pages = true; + } return ResultSuccess; } -ResultCode KPageTable::UnmapCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { - KScopedLightLock lk(general_lock); +VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, + std::size_t num_pages, std::size_t alignment, std::size_t offset, + std::size_t guard_pages) { + VAddr address = 0; + + if (num_pages <= region_num_pages) { + if (this->IsAslrEnabled()) { + // Try to directly find a free area up to 8 times. + for (std::size_t i = 0; i < 8; i++) { + const std::size_t random_offset = + KSystemControl::GenerateRandomRange( + 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * + alignment; + const VAddr candidate = + Common::AlignDown((region_start + random_offset), alignment) + offset; + + KMemoryInfo info = this->QueryInfoImpl(candidate); + + if (info.state != KMemoryState::Free) { + continue; + } + if (region_start > candidate) { + continue; + } + if (info.GetAddress() + guard_pages * PageSize > candidate) { + continue; + } + + const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; + if (candidate_end > info.GetLastAddress()) { + continue; + } + if (candidate_end > region_start + region_num_pages * PageSize - 1) { + continue; + } + + address = candidate; + break; + } + // Fall back to finding the first free area with a random offset. + if (address == 0) { + // NOTE: Nintendo does not account for guard pages here. + // This may theoretically cause an offset to be chosen that cannot be mapped. We + // will account for guard pages. + const std::size_t offset_pages = KSystemControl::GenerateRandomRange( + 0, region_num_pages - num_pages - guard_pages); + address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, + region_num_pages - offset_pages, num_pages, + alignment, offset, guard_pages); + } + } - if (!size) { - return ResultSuccess; + // Find the first free area. + if (address == 0) { + address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); + } } - const std::size_t num_pages{size / PageSize}; + return address; +} - CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, nullptr, src_addr, size, - KMemoryState::All, KMemoryState::Normal, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); +Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { + ASSERT(this->IsLockedByCurrentThread()); - KMemoryState state{}; - CASCADE_CODE(CheckMemoryState( - &state, nullptr, nullptr, nullptr, dst_addr, PageSize, KMemoryState::FlagCanCodeAlias, - KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - CASCADE_CODE(CheckMemoryState(dst_addr, size, KMemoryState::All, state, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None)); - CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); + const size_t size = num_pages * PageSize; - block_manager->Update(dst_addr, num_pages, KMemoryState::Free); - block_manager->Update(src_addr, num_pages, KMemoryState::Normal, - KMemoryPermission::UserReadWrite); + // We're making a new group, not adding to an existing one. + R_UNLESS(pg.Empty(), ResultInvalidCurrentMemory); + + // Begin traversal. + Common::PageTable::TraversalContext context; + Common::PageTable::TraversalEntry next_entry; + R_UNLESS(page_table_impl.BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); - system.InvalidateCpuInstructionCacheRange(dst_addr, size); + // Prepare tracking variables. + PAddr cur_addr = next_entry.phys_addr; + size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); + size_t tot_size = cur_size; + + // Iterate, adding to group as we go. + const auto& memory_layout = system.Kernel().MemoryLayout(); + while (tot_size < size) { + R_UNLESS(page_table_impl.ContinueTraversal(next_entry, context), + ResultInvalidCurrentMemory); + + if (next_entry.phys_addr != (cur_addr + cur_size)) { + const size_t cur_pages = cur_size / PageSize; + + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); + + cur_addr = next_entry.phys_addr; + cur_size = next_entry.block_size; + } else { + cur_size += next_entry.block_size; + } + + tot_size += next_entry.block_size; + } + + // Ensure we add the right amount for the last block. + if (tot_size > size) { + cur_size -= (tot_size - size); + } + + // Add the last block. + const size_t cur_pages = cur_size / PageSize; + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); return ResultSuccess; } -ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, - KPageTable& src_page_table, VAddr src_addr) { +bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t num_pages) { + ASSERT(this->IsLockedByCurrentThread()); + + const size_t size = num_pages * PageSize; + const auto& pg = pg_ll.Nodes(); + const auto& memory_layout = system.Kernel().MemoryLayout(); + + // Empty groups are necessarily invalid. + if (pg.empty()) { + return false; + } + + // We're going to validate that the group we'd expect is the group we see. + auto cur_it = pg.begin(); + PAddr cur_block_address = cur_it->GetAddress(); + size_t cur_block_pages = cur_it->GetNumPages(); + + auto UpdateCurrentIterator = [&]() { + if (cur_block_pages == 0) { + if ((++cur_it) == pg.end()) { + return false; + } + + cur_block_address = cur_it->GetAddress(); + cur_block_pages = cur_it->GetNumPages(); + } + return true; + }; + + // Begin traversal. + Common::PageTable::TraversalContext context; + Common::PageTable::TraversalEntry next_entry; + if (!page_table_impl.BeginTraversal(next_entry, context, addr)) { + return false; + } + + // Prepare tracking variables. + PAddr cur_addr = next_entry.phys_addr; + size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); + size_t tot_size = cur_size; + + // Iterate, comparing expected to actual. + while (tot_size < size) { + if (!page_table_impl.ContinueTraversal(next_entry, context)) { + return false; + } + + if (next_entry.phys_addr != (cur_addr + cur_size)) { + const size_t cur_pages = cur_size / PageSize; + + if (!IsHeapPhysicalAddress(memory_layout, cur_addr)) { + return false; + } + + if (!UpdateCurrentIterator()) { + return false; + } + + if (cur_block_address != cur_addr || cur_block_pages < cur_pages) { + return false; + } + + cur_block_address += cur_size; + cur_block_pages -= cur_pages; + cur_addr = next_entry.phys_addr; + cur_size = next_entry.block_size; + } else { + cur_size += next_entry.block_size; + } + + tot_size += next_entry.block_size; + } + + // Ensure we compare the right amount for the last block. + if (tot_size > size) { + cur_size -= (tot_size - size); + } + + if (!IsHeapPhysicalAddress(memory_layout, cur_addr)) { + return false; + } + + if (!UpdateCurrentIterator()) { + return false; + } + + return cur_block_address == cur_addr && cur_block_pages == (cur_size / PageSize); +} + +Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, + VAddr src_addr) { KScopedLightLock lk(general_lock); const std::size_t num_pages{size / PageSize}; @@ -397,156 +655,486 @@ ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None); + system.InvalidateCpuInstructionCaches(); + return ResultSuccess; } -ResultCode KPageTable::MapPhysicalMemory(VAddr addr, std::size_t size) { +Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { // Lock the physical memory lock. KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); - // Lock the table. - KScopedLightLock lk(general_lock); - - std::size_t mapped_size{}; - const VAddr end_addr{addr + size}; + // Calculate the last address for convenience. + const VAddr last_address = address + size - 1; - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state != KMemoryState::Free) { - mapped_size += GetSizeInRange(info, addr, end_addr); - } - }); + // Define iteration variables. + VAddr cur_address; + std::size_t mapped_size; - if (mapped_size == size) { - return ResultSuccess; - } + // The entire mapping process can be retried. + while (true) { + // Check if the memory is already mapped. + { + // Lock the table. + KScopedLightLock lk(general_lock); + + // Iterate over the memory. + cur_address = address; + mapped_size = 0; + + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); + + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + if (info.GetState() != KMemoryState::Free) { + mapped_size += (last_address + 1 - cur_address); + } + break; + } + + // Track the memory if it's mapped. + if (info.GetState() != KMemoryState::Free) { + mapped_size += VAddr(info.GetEndAddress()) - cur_address; + } + + // Advance. + cur_address = info.GetEndAddress(); + ++it; + } - const std::size_t remaining_size{size - mapped_size}; - const std::size_t remaining_pages{remaining_size / PageSize}; + // If the size mapped is the size requested, we've nothing to do. + R_SUCCEED_IF(size == mapped_size); + } - // Reserve the memory from the process resource limit. - KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, - remaining_size); - if (!memory_reservation.Succeeded()) { - LOG_ERROR(Kernel, "Could not reserve remaining {:X} bytes", remaining_size); - return ResultLimitReached; + // Allocate and map the memory. + { + // Reserve the memory from the process resource limit. + KScopedResourceReservation memory_reservation( + system.Kernel().CurrentProcess()->GetResourceLimit(), + LimitableResource::PhysicalMemory, size - mapped_size); + R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); + + // Allocate pages for the new memory. + KPageGroup pg; + R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + &pg, (size - mapped_size) / PageSize, + KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); + + // Map the memory. + { + // Lock the table. + KScopedLightLock lk(general_lock); + + size_t num_allocator_blocks = 0; + + // Verify that nobody has mapped memory since we first checked. + { + // Iterate over the memory. + size_t checked_mapped_size = 0; + cur_address = address; + + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); + + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + const bool is_free = info.GetState() == KMemoryState::Free; + if (is_free) { + if (info.GetAddress() < address) { + ++num_allocator_blocks; + } + if (last_address < info.GetLastAddress()) { + ++num_allocator_blocks; + } + } + + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + if (!is_free) { + checked_mapped_size += (last_address + 1 - cur_address); + } + break; + } + + // Track the memory if it's mapped. + if (!is_free) { + checked_mapped_size += VAddr(info.GetEndAddress()) - cur_address; + } + + // Advance. + cur_address = info.GetEndAddress(); + ++it; + } + + // If the size now isn't what it was before, somebody mapped or unmapped + // concurrently. If this happened, retry. + if (mapped_size != checked_mapped_size) { + continue; + } + } + + // Reset the current tracking address, and make sure we clean up on failure. + cur_address = address; + auto unmap_guard = detail::ScopeExit([&] { + if (cur_address > address) { + const VAddr last_unmap_address = cur_address - 1; + + // Iterate, unmapping the pages. + cur_address = address; + + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); + + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + // If the memory state is free, we mapped it and need to unmap it. + if (info.GetState() == KMemoryState::Free) { + // Determine the range to unmap. + const size_t cur_pages = + std::min(VAddr(info.GetEndAddress()) - cur_address, + last_unmap_address + 1 - cur_address) / + PageSize; + + // Unmap. + ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None, + OperationType::Unmap) + .IsSuccess()); + } + + // Check if we're done. + if (last_unmap_address <= info.GetLastAddress()) { + break; + } + + // Advance. + cur_address = info.GetEndAddress(); + ++it; + } + } + }); + + // Iterate over the memory. + auto pg_it = pg.Nodes().begin(); + PAddr pg_phys_addr = pg_it->GetAddress(); + size_t pg_pages = pg_it->GetNumPages(); + + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); + + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + // If it's unmapped, we need to map it. + if (info.GetState() == KMemoryState::Free) { + // Determine the range to map. + size_t map_pages = std::min(VAddr(info.GetEndAddress()) - cur_address, + last_address + 1 - cur_address) / + PageSize; + + // While we have pages to map, map them. + while (map_pages > 0) { + // Check if we're at the end of the physical block. + if (pg_pages == 0) { + // Ensure there are more pages to map. + ASSERT(pg_it != pg.Nodes().end()); + + // Advance our physical block. + ++pg_it; + pg_phys_addr = pg_it->GetAddress(); + pg_pages = pg_it->GetNumPages(); + } + + // Map whatever we can. + const size_t cur_pages = std::min(pg_pages, map_pages); + R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::UserReadWrite, + OperationType::Map, pg_phys_addr)); + + // Advance. + cur_address += cur_pages * PageSize; + map_pages -= cur_pages; + + pg_phys_addr += cur_pages * PageSize; + pg_pages -= cur_pages; + } + } + + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + break; + } + + // Advance. + cur_address = info.GetEndAddress(); + ++it; + } + + // We succeeded, so commit the memory reservation. + memory_reservation.Commit(); + + // Increase our tracked mapped size. + mapped_physical_memory_size += (size - mapped_size); + + // Update the relevant memory blocks. + block_manager->Update(address, size / PageSize, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryState::Normal, KMemoryPermission::UserReadWrite, + KMemoryAttribute::None); + + // Cancel our guard. + unmap_guard.Cancel(); + + return ResultSuccess; + } + } } +} - KPageLinkedList page_linked_list; +Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { + // Lock the physical memory lock. + KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); - CASCADE_CODE(system.Kernel().MemoryManager().Allocate(page_linked_list, remaining_pages, - memory_pool, allocation_option)); + // Lock the table. + KScopedLightLock lk(general_lock); - // We succeeded, so commit the memory reservation. - memory_reservation.Commit(); + // Calculate the last address for convenience. + const VAddr last_address = address + size - 1; - // Map the memory. - auto node{page_linked_list.Nodes().begin()}; - PAddr map_addr{node->GetAddress()}; - std::size_t src_num_pages{node->GetNumPages()}; - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state != KMemoryState::Free) { - return; - } + // Define iteration variables. + VAddr cur_address = 0; + std::size_t mapped_size = 0; + std::size_t num_allocator_blocks = 0; - std::size_t dst_num_pages{GetSizeInRange(info, addr, end_addr) / PageSize}; - VAddr dst_addr{GetAddressInRange(info, addr)}; + // Check if the memory is mapped. + { + // Iterate over the memory. + cur_address = address; + mapped_size = 0; + + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); + + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); + + // Verify the memory's state. + const bool is_normal = info.GetState() == KMemoryState::Normal && + info.GetAttribute() == KMemoryAttribute::None; + const bool is_free = info.GetState() == KMemoryState::Free; + R_UNLESS(is_normal || is_free, ResultInvalidCurrentMemory); + + if (is_normal) { + R_UNLESS(info.GetAttribute() == KMemoryAttribute::None, ResultInvalidCurrentMemory); + + if (info.GetAddress() < address) { + ++num_allocator_blocks; + } + if (last_address < info.GetLastAddress()) { + ++num_allocator_blocks; + } + } - while (dst_num_pages) { - if (!src_num_pages) { - node = std::next(node); - map_addr = node->GetAddress(); - src_num_pages = node->GetNumPages(); + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + if (is_normal) { + mapped_size += (last_address + 1 - cur_address); + } + break; } - const std::size_t num_pages{std::min(src_num_pages, dst_num_pages)}; - Operate(dst_addr, num_pages, KMemoryPermission::UserReadWrite, OperationType::Map, - map_addr); + // Track the memory if it's mapped. + if (is_normal) { + mapped_size += VAddr(info.GetEndAddress()) - cur_address; + } - dst_addr += num_pages * PageSize; - map_addr += num_pages * PageSize; - src_num_pages -= num_pages; - dst_num_pages -= num_pages; + // Advance. + cur_address = info.GetEndAddress(); + ++it; } - }); - mapped_physical_memory_size += remaining_size; - - const std::size_t num_pages{size / PageSize}; - block_manager->Update(addr, num_pages, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryState::Normal, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None); - - return ResultSuccess; -} + // If there's nothing mapped, we've nothing to do. + R_SUCCEED_IF(mapped_size == 0); + } -ResultCode KPageTable::UnmapPhysicalMemory(VAddr addr, std::size_t size) { - // Lock the physical memory lock. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + // Make a page group for the unmap region. + KPageGroup pg; + { + auto& impl = this->PageTableImpl(); + + // Begin traversal. + Common::PageTable::TraversalContext context; + Common::PageTable::TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0}; + bool cur_valid = false; + Common::PageTable::TraversalEntry next_entry; + bool next_valid = false; + size_t tot_size = 0; + + cur_address = address; + next_valid = impl.BeginTraversal(next_entry, context, cur_address); + next_entry.block_size = + (next_entry.block_size - (next_entry.phys_addr & (next_entry.block_size - 1))); + + // Iterate, building the group. + while (true) { + if ((!next_valid && !cur_valid) || + (next_valid && cur_valid && + next_entry.phys_addr == cur_entry.phys_addr + cur_entry.block_size)) { + cur_entry.block_size += next_entry.block_size; + } else { + if (cur_valid) { + // ASSERT(IsHeapPhysicalAddress(cur_entry.phys_addr)); + R_TRY(pg.AddBlock(cur_entry.phys_addr, cur_entry.block_size / PageSize)); + } + + // Update tracking variables. + tot_size += cur_entry.block_size; + cur_entry = next_entry; + cur_valid = next_valid; + } - // Lock the table. - KScopedLightLock lk(general_lock); + if (cur_entry.block_size + tot_size >= size) { + break; + } - const VAddr end_addr{addr + size}; - ResultCode result{ResultSuccess}; - std::size_t mapped_size{}; + next_valid = impl.ContinueTraversal(next_entry, context); + } - // Verify that the region can be unmapped - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state == KMemoryState::Normal) { - if (info.attribute != KMemoryAttribute::None) { - result = ResultInvalidCurrentMemory; - return; + // Add the last block. + if (cur_valid) { + // ASSERT(IsHeapPhysicalAddress(cur_entry.phys_addr)); + R_TRY(pg.AddBlock(cur_entry.phys_addr, (size - tot_size) / PageSize)); + } + } + ASSERT(pg.GetNumPages() == mapped_size / PageSize); + + // Reset the current tracking address, and make sure we clean up on failure. + cur_address = address; + auto remap_guard = detail::ScopeExit([&] { + if (cur_address > address) { + const VAddr last_map_address = cur_address - 1; + cur_address = address; + + // Iterate over the memory we unmapped. + auto it = block_manager->FindIterator(cur_address); + auto pg_it = pg.Nodes().begin(); + PAddr pg_phys_addr = pg_it->GetAddress(); + size_t pg_pages = pg_it->GetNumPages(); + + while (true) { + // Get the memory info for the pages we unmapped, convert to property. + const KMemoryInfo info = it->GetMemoryInfo(); + + // If the memory is normal, we unmapped it and need to re-map it. + if (info.GetState() == KMemoryState::Normal) { + // Determine the range to map. + size_t map_pages = std::min(VAddr(info.GetEndAddress()) - cur_address, + last_map_address + 1 - cur_address) / + PageSize; + + // While we have pages to map, map them. + while (map_pages > 0) { + // Check if we're at the end of the physical block. + if (pg_pages == 0) { + // Ensure there are more pages to map. + ASSERT(pg_it != pg.Nodes().end()); + + // Advance our physical block. + ++pg_it; + pg_phys_addr = pg_it->GetAddress(); + pg_pages = pg_it->GetNumPages(); + } + + // Map whatever we can. + const size_t cur_pages = std::min(pg_pages, map_pages); + ASSERT(this->Operate(cur_address, cur_pages, info.GetPermission(), + OperationType::Map, pg_phys_addr) == ResultSuccess); + + // Advance. + cur_address += cur_pages * PageSize; + map_pages -= cur_pages; + + pg_phys_addr += cur_pages * PageSize; + pg_pages -= cur_pages; + } + } + + // Check if we're done. + if (last_map_address <= info.GetLastAddress()) { + break; + } + + // Advance. + ++it; } - mapped_size += GetSizeInRange(info, addr, end_addr); - } else if (info.state != KMemoryState::Free) { - result = ResultInvalidCurrentMemory; } }); - if (result.IsError()) { - return result; - } - - if (!mapped_size) { - return ResultSuccess; - } + // Iterate over the memory, unmapping as we go. + auto it = block_manager->FindIterator(cur_address); + while (true) { + // Check that the iterator is valid. + ASSERT(it != block_manager->end()); - // Unmap each region within the range - KPageLinkedList page_linked_list; - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state == KMemoryState::Normal) { - const std::size_t block_size{GetSizeInRange(info, addr, end_addr)}; - const std::size_t block_num_pages{block_size / PageSize}; - const VAddr block_addr{GetAddressInRange(info, addr)}; + // Get the memory info. + const KMemoryInfo info = it->GetMemoryInfo(); - AddRegionToPages(block_addr, block_size / PageSize, page_linked_list); + // If the memory state is normal, we need to unmap it. + if (info.GetState() == KMemoryState::Normal) { + // Determine the range to unmap. + const size_t cur_pages = std::min(VAddr(info.GetEndAddress()) - cur_address, + last_address + 1 - cur_address) / + PageSize; - if (result = Operate(block_addr, block_num_pages, KMemoryPermission::None, - OperationType::Unmap); - result.IsError()) { - return; - } + // Unmap. + R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::None, OperationType::Unmap)); } - }); - if (result.IsError()) { - return result; - } - const std::size_t num_pages{size / PageSize}; - system.Kernel().MemoryManager().Free(page_linked_list, num_pages, memory_pool, - allocation_option); + // Check if we're done. + if (last_address <= info.GetLastAddress()) { + break; + } - block_manager->Update(addr, num_pages, KMemoryState::Free); + // Advance. + cur_address = info.GetEndAddress(); + ++it; + } + // Release the memory resource. + mapped_physical_memory_size -= mapped_size; auto process{system.Kernel().CurrentProcess()}; process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); - mapped_physical_memory_size -= mapped_size; + + // Update memory blocks. + block_manager->Update(address, size / PageSize, KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None); + + // TODO(bunnei): This is a workaround until the next set of changes, where we add reference + // counting for mapped pages. Until then, we must manually close the reference to the page + // group. + system.Kernel().MemoryManager().Close(pg); + + // We succeeded. + remap_guard.Cancel(); return ResultSuccess; } -ResultCode KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { +Result KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { KScopedLightLock lk(general_lock); KMemoryState src_state{}; @@ -559,7 +1147,7 @@ ResultCode KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t siz return ResultInvalidCurrentMemory; } - KPageLinkedList page_linked_list; + KPageGroup page_linked_list; const std::size_t num_pages{size / PageSize}; AddRegionToPages(src_addr, num_pages, page_linked_list); @@ -585,7 +1173,7 @@ ResultCode KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t siz return ResultSuccess; } -ResultCode KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { +Result KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { KScopedLightLock lk(general_lock); KMemoryState src_state{}; @@ -600,8 +1188,8 @@ ResultCode KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t s KMemoryPermission::None, KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - KPageLinkedList src_pages; - KPageLinkedList dst_pages; + KPageGroup src_pages; + KPageGroup dst_pages; const std::size_t num_pages{size / PageSize}; AddRegionToPages(src_addr, num_pages, src_pages); @@ -627,8 +1215,8 @@ ResultCode KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t s return ResultSuccess; } -ResultCode KPageTable::MapPages(VAddr addr, const KPageLinkedList& page_linked_list, - KMemoryPermission perm) { +Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, + KMemoryPermission perm) { ASSERT(this->IsLockedByCurrentThread()); VAddr cur_addr{addr}; @@ -651,8 +1239,8 @@ ResultCode KPageTable::MapPages(VAddr addr, const KPageLinkedList& page_linked_l return ResultSuccess; } -ResultCode KPageTable::MapPages(VAddr address, KPageLinkedList& page_linked_list, - KMemoryState state, KMemoryPermission perm) { +Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, + KMemoryPermission perm) { // Check that the map is in range. const std::size_t num_pages{page_linked_list.GetNumPages()}; const std::size_t size{num_pages * PageSize}; @@ -675,15 +1263,54 @@ ResultCode KPageTable::MapPages(VAddr address, KPageLinkedList& page_linked_list return ResultSuccess; } -ResultCode KPageTable::UnmapPages(VAddr addr, const KPageLinkedList& page_linked_list) { +Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, + PAddr phys_addr, bool is_pa_valid, VAddr region_start, + std::size_t region_num_pages, KMemoryState state, + KMemoryPermission perm) { + ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); + + // Ensure this is a valid map request. + R_UNLESS(this->CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Find a random address to map at. + VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, + this->GetNumGuardPages()); + R_UNLESS(addr != 0, ResultOutOfMemory); + ASSERT(Common::IsAligned(addr, alignment)); + ASSERT(this->CanContain(addr, num_pages * PageSize, state)); + ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryAttribute::None) + .IsSuccess()); + + // Perform mapping operation. + if (is_pa_valid) { + R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); + } else { + UNIMPLEMENTED(); + } + + // Update the blocks. + block_manager->Update(addr, num_pages, state, perm); + + // We successfully mapped the pages. + *out_addr = addr; + return ResultSuccess; +} + +Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { ASSERT(this->IsLockedByCurrentThread()); VAddr cur_addr{addr}; for (const auto& node : page_linked_list.Nodes()) { - const std::size_t num_pages{(addr - cur_addr) / PageSize}; - if (const auto result{ - Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap)}; + if (const auto result{Operate(cur_addr, node.GetNumPages(), KMemoryPermission::None, + OperationType::Unmap)}; result.IsError()) { return result; } @@ -694,8 +1321,7 @@ ResultCode KPageTable::UnmapPages(VAddr addr, const KPageLinkedList& page_linked return ResultSuccess; } -ResultCode KPageTable::UnmapPages(VAddr addr, KPageLinkedList& page_linked_list, - KMemoryState state) { +Result KPageTable::UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state) { // Check that the unmap is in range. const std::size_t num_pages{page_linked_list.GetNumPages()}; const std::size_t size{num_pages * PageSize}; @@ -718,8 +1344,57 @@ ResultCode KPageTable::UnmapPages(VAddr addr, KPageLinkedList& page_linked_list, return ResultSuccess; } -ResultCode KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, - Svc::MemoryPermission svc_perm) { +Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state) { + // Check that the unmap is in range. + const std::size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the memory state. + std::size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, state, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::None)); + + // Perform the unmap. + R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + + // Update the blocks. + block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); + + return ResultSuccess; +} + +Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr) { + // Ensure that the page group isn't null. + ASSERT(out != nullptr); + + // Make sure that the region we're mapping is valid for the table. + const size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check if state allows us to create the group. + R_TRY(this->CheckMemoryState(address, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Create a new page group for the region. + R_TRY(this->MakePageGroup(*out, address, num_pages)); + + return ResultSuccess; +} + +Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, + Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. @@ -753,7 +1428,7 @@ ResultCode KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, new_state = KMemoryState::AliasCodeData; break; default: - UNREACHABLE(); + ASSERT(false); } } @@ -791,7 +1466,7 @@ KMemoryInfo KPageTable::QueryInfo(VAddr addr) { return QueryInfoImpl(addr); } -ResultCode KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { +Result KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { KScopedLightLock lk(general_lock); KMemoryState state{}; @@ -809,7 +1484,7 @@ ResultCode KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemo return ResultSuccess; } -ResultCode KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { +Result KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { KScopedLightLock lk(general_lock); KMemoryState state{}; @@ -824,8 +1499,8 @@ ResultCode KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { return ResultSuccess; } -ResultCode KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, - Svc::MemoryPermission svc_perm) { +Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, + Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. @@ -852,7 +1527,7 @@ ResultCode KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, return ResultSuccess; } -ResultCode KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr) { +Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr) { const size_t num_pages = size / PageSize; ASSERT((static_cast<KMemoryAttribute>(mask) | KMemoryAttribute::SetMask) == KMemoryAttribute::SetMask); @@ -887,7 +1562,7 @@ ResultCode KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask return ResultSuccess; } -ResultCode KPageTable::SetMaxHeapSize(std::size_t size) { +Result KPageTable::SetMaxHeapSize(std::size_t size) { // Lock the table. KScopedLightLock lk(general_lock); @@ -899,7 +1574,7 @@ ResultCode KPageTable::SetMaxHeapSize(std::size_t size) { return ResultSuccess; } -ResultCode KPageTable::SetHeapSize(VAddr* out, std::size_t size) { +Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { // Lock the physical memory mutex. KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); @@ -966,9 +1641,16 @@ ResultCode KPageTable::SetHeapSize(VAddr* out, std::size_t size) { R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the heap extension. - KPageLinkedList page_linked_list; - R_TRY(system.Kernel().MemoryManager().Allocate(page_linked_list, allocation_size / PageSize, - memory_pool, allocation_option)); + KPageGroup pg; + R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + &pg, allocation_size / PageSize, + KMemoryManager::EncodeOption(memory_pool, allocation_option))); + + // Clear all the newly allocated pages. + for (const auto& it : pg.Nodes()) { + std::memset(system.DeviceMemory().GetPointer(it.GetAddress()), heap_fill_value, + it.GetSize()); + } // Map the pages. { @@ -987,7 +1669,7 @@ ResultCode KPageTable::SetHeapSize(VAddr* out, std::size_t size) { // Map the pages. const auto num_pages = allocation_size / PageSize; - R_TRY(Operate(current_heap_end, num_pages, page_linked_list, OperationType::MapGroup)); + R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); // Clear all the newly allocated pages. for (std::size_t cur_page = 0; cur_page < num_pages; ++cur_page) { @@ -1034,9 +1716,10 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, if (is_map_only) { R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); } else { - KPageLinkedList page_group; - R_TRY(system.Kernel().MemoryManager().Allocate(page_group, needed_num_pages, memory_pool, - allocation_option)); + KPageGroup page_group; + R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + &page_group, needed_num_pages, + KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); } @@ -1045,11 +1728,11 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, return addr; } -ResultCode KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { +Result KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { KScopedLightLock lk(general_lock); KMemoryPermission perm{}; - if (const ResultCode result{CheckMemoryState( + if (const Result result{CheckMemoryState( nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, @@ -1068,11 +1751,11 @@ ResultCode KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { return ResultSuccess; } -ResultCode KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { +Result KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { KScopedLightLock lk(general_lock); KMemoryPermission perm{}; - if (const ResultCode result{CheckMemoryState( + if (const Result result{CheckMemoryState( nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, @@ -1091,61 +1774,24 @@ ResultCode KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) return ResultSuccess; } -ResultCode KPageTable::LockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, - KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->ShareToDevice(permission); - }, - new_perm); - - return ResultSuccess; +Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size) { + return this->LockMemoryAndOpen( + out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, + KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, + KMemoryAttribute::None, + static_cast<KMemoryPermission>(KMemoryPermission::NotMapped | + KMemoryPermission::KernelReadWrite), + KMemoryAttribute::Locked); } -ResultCode KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::UserReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::All, KMemoryAttribute::Locked)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->UnshareToDevice(permission); - }, - new_perm); - - return ResultSuccess; +Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg) { + return this->UnlockMemory( + addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, + KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); } -ResultCode KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { +Result KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { block_manager = std::make_unique<KMemoryBlockManager>(start, end); return ResultSuccess; @@ -1170,13 +1816,11 @@ bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { } void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages, - KPageLinkedList& page_linked_list) { + KPageGroup& page_linked_list) { VAddr addr{start}; while (addr < start + (num_pages * PageSize)) { const PAddr paddr{GetPhysicalAddr(addr)}; - if (!paddr) { - UNREACHABLE(); - } + ASSERT(paddr != 0); page_linked_list.AddBlock(paddr, 1); addr += PageSize; } @@ -1191,8 +1835,8 @@ VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_page IsKernel() ? 1 : 4); } -ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageLinkedList& page_group, - OperationType operation) { +Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, + OperationType operation) { ASSERT(this->IsLockedByCurrentThread()); ASSERT(Common::IsAligned(addr, PageSize)); @@ -1207,7 +1851,7 @@ ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageLin system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress()); break; default: - UNREACHABLE(); + ASSERT(false); } addr += size; @@ -1216,8 +1860,8 @@ ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageLin return ResultSuccess; } -ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, - OperationType operation, PAddr map_addr) { +Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, + OperationType operation, PAddr map_addr) { ASSERT(this->IsLockedByCurrentThread()); ASSERT(num_pages > 0); @@ -1238,12 +1882,12 @@ ResultCode KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermiss case OperationType::ChangePermissionsAndRefresh: break; default: - UNREACHABLE(); + ASSERT(false); } return ResultSuccess; } -constexpr VAddr KPageTable::GetRegionAddress(KMemoryState state) const { +VAddr KPageTable::GetRegionAddress(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: @@ -1275,11 +1919,10 @@ constexpr VAddr KPageTable::GetRegionAddress(KMemoryState state) const { return code_region_start; default: UNREACHABLE(); - return {}; } } -constexpr std::size_t KPageTable::GetRegionSize(KMemoryState state) const { +std::size_t KPageTable::GetRegionSize(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: @@ -1311,7 +1954,6 @@ constexpr std::size_t KPageTable::GetRegionSize(KMemoryState state) const { return code_region_end - code_region_start; default: UNREACHABLE(); - return {}; } } @@ -1361,10 +2003,10 @@ bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) co } } -ResultCode KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, - KMemoryAttribute attr) const { +Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr) const { // Validate the states match expectation. R_UNLESS((info.state & state_mask) == state, ResultInvalidCurrentMemory); R_UNLESS((info.perm & perm_mask) == perm, ResultInvalidCurrentMemory); @@ -1373,12 +2015,11 @@ ResultCode KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState st return ResultSuccess; } -ResultCode KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, - std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, - KMemoryAttribute attr_mask, - KMemoryAttribute attr) const { +Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, + std::size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. @@ -1416,12 +2057,12 @@ ResultCode KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed return ResultSuccess; } -ResultCode KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, std::size_t* out_blocks_needed, - VAddr addr, std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, - KMemoryAttribute attr, KMemoryAttribute ignore_attr) const { +Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, + KMemoryAttribute* out_attr, std::size_t* out_blocks_needed, + VAddr addr, std::size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, KMemoryAttribute ignore_attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. @@ -1478,4 +2119,109 @@ ResultCode KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermissi return ResultSuccess; } +Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr addr, size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryPermission new_perm, KMemoryAttribute lock_attr) { + // Validate basic preconditions. + ASSERT((lock_attr & attr) == KMemoryAttribute::None); + ASSERT((lock_attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == + KMemoryAttribute::None); + + // Validate the lock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check that the output page group is empty, if it exists. + if (out_pg) { + ASSERT(out_pg->GetNumPages() == 0); + } + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Get the physical address, if we're supposed to. + if (out_paddr != nullptr) { + ASSERT(this->GetPhysicalAddressLocked(out_paddr, addr)); + } + + // Make the page group, if we're supposed to. + if (out_pg != nullptr) { + R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + +Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, KMemoryPermission new_perm, + KMemoryAttribute lock_attr, const KPageGroup* pg) { + // Validate basic preconditions. + ASSERT((attr_mask & lock_attr) == lock_attr); + ASSERT((attr & lock_attr) == lock_attr); + + // Validate the unlock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Check the page group. + if (pg != nullptr) { + R_UNLESS(this->IsValidPageGroup(*pg, addr, num_pages), ResultInvalidMemoryRegion); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + } // namespace Kernel |