diff options
Diffstat (limited to '')
| -rw-r--r-- | src/core/hle/kernel/vm_manager.cpp | 401 |
1 files changed, 340 insertions, 61 deletions
diff --git a/src/core/hle/kernel/vm_manager.cpp b/src/core/hle/kernel/vm_manager.cpp index 6d6980aba..c7af87073 100644 --- a/src/core/hle/kernel/vm_manager.cpp +++ b/src/core/hle/kernel/vm_manager.cpp @@ -5,13 +5,15 @@ #include <algorithm> #include <iterator> #include <utility> +#include "common/alignment.h" #include "common/assert.h" #include "common/logging/log.h" #include "common/memory_hook.h" -#include "core/arm/arm_interface.h" #include "core/core.h" #include "core/file_sys/program_metadata.h" #include "core/hle/kernel/errors.h" +#include "core/hle/kernel/process.h" +#include "core/hle/kernel/resource_limit.h" #include "core/hle/kernel/vm_manager.h" #include "core/memory.h" #include "core/memory_setup.h" @@ -49,10 +51,14 @@ bool VirtualMemoryArea::CanBeMergedWith(const VirtualMemoryArea& next) const { type != next.type) { return false; } - if (type == VMAType::AllocatedMemoryBlock && - (backing_block != next.backing_block || offset + size != next.offset)) { + if ((attribute & MemoryAttribute::DeviceMapped) == MemoryAttribute::DeviceMapped) { + // TODO: Can device mapped memory be merged sanely? + // Not merging it may cause inaccuracies versus hardware when memory layout is queried. return false; } + if (type == VMAType::AllocatedMemoryBlock) { + return true; + } if (type == VMAType::BackingMemory && backing_memory + size != next.backing_memory) { return false; } @@ -68,9 +74,7 @@ VMManager::VMManager(Core::System& system) : system{system} { Reset(FileSys::ProgramAddressSpaceType::Is39Bit); } -VMManager::~VMManager() { - Reset(FileSys::ProgramAddressSpaceType::Is39Bit); -} +VMManager::~VMManager() = default; void VMManager::Reset(FileSys::ProgramAddressSpaceType type) { Clear(); @@ -100,9 +104,9 @@ bool VMManager::IsValidHandle(VMAHandle handle) const { } ResultVal<VMManager::VMAHandle> VMManager::MapMemoryBlock(VAddr target, - std::shared_ptr<std::vector<u8>> block, + std::shared_ptr<PhysicalMemory> block, std::size_t offset, u64 size, - MemoryState state) { + MemoryState state, VMAPermission perm) { ASSERT(block != nullptr); ASSERT(offset + size <= block->size()); @@ -111,17 +115,8 @@ ResultVal<VMManager::VMAHandle> VMManager::MapMemoryBlock(VAddr target, VirtualMemoryArea& final_vma = vma_handle->second; ASSERT(final_vma.size == size); - system.ArmInterface(0).MapBackingMemory(target, size, block->data() + offset, - VMAPermission::ReadWriteExecute); - system.ArmInterface(1).MapBackingMemory(target, size, block->data() + offset, - VMAPermission::ReadWriteExecute); - system.ArmInterface(2).MapBackingMemory(target, size, block->data() + offset, - VMAPermission::ReadWriteExecute); - system.ArmInterface(3).MapBackingMemory(target, size, block->data() + offset, - VMAPermission::ReadWriteExecute); - final_vma.type = VMAType::AllocatedMemoryBlock; - final_vma.permissions = VMAPermission::ReadWrite; + final_vma.permissions = perm; final_vma.state = state; final_vma.backing_block = std::move(block); final_vma.offset = offset; @@ -139,11 +134,6 @@ ResultVal<VMManager::VMAHandle> VMManager::MapBackingMemory(VAddr target, u8* me VirtualMemoryArea& final_vma = vma_handle->second; ASSERT(final_vma.size == size); - system.ArmInterface(0).MapBackingMemory(target, size, memory, VMAPermission::ReadWriteExecute); - system.ArmInterface(1).MapBackingMemory(target, size, memory, VMAPermission::ReadWriteExecute); - system.ArmInterface(2).MapBackingMemory(target, size, memory, VMAPermission::ReadWriteExecute); - system.ArmInterface(3).MapBackingMemory(target, size, memory, VMAPermission::ReadWriteExecute); - final_vma.type = VMAType::BackingMemory; final_vma.permissions = VMAPermission::ReadWrite; final_vma.state = state; @@ -154,22 +144,33 @@ ResultVal<VMManager::VMAHandle> VMManager::MapBackingMemory(VAddr target, u8* me } ResultVal<VAddr> VMManager::FindFreeRegion(u64 size) const { - // Find the first Free VMA. - const VAddr base = GetASLRRegionBaseAddress(); - const VMAHandle vma_handle = std::find_if(vma_map.begin(), vma_map.end(), [&](const auto& vma) { - if (vma.second.type != VMAType::Free) - return false; + return FindFreeRegion(GetASLRRegionBaseAddress(), GetASLRRegionEndAddress(), size); +} - const VAddr vma_end = vma.second.base + vma.second.size; - return vma_end > base && vma_end >= base + size; - }); +ResultVal<VAddr> VMManager::FindFreeRegion(VAddr begin, VAddr end, u64 size) const { + ASSERT(begin < end); + ASSERT(size <= end - begin); - if (vma_handle == vma_map.end()) { + const VMAHandle vma_handle = + std::find_if(vma_map.begin(), vma_map.end(), [begin, end, size](const auto& vma) { + if (vma.second.type != VMAType::Free) { + return false; + } + const VAddr vma_base = vma.second.base; + const VAddr vma_end = vma_base + vma.second.size; + const VAddr assumed_base = (begin < vma_base) ? vma_base : begin; + const VAddr used_range = assumed_base + size; + + return vma_base <= assumed_base && assumed_base < used_range && used_range < end && + used_range <= vma_end; + }); + + if (vma_handle == vma_map.cend()) { // TODO(Subv): Find the correct error code here. return ResultCode(-1); } - const VAddr target = std::max(base, vma_handle->second.base); + const VAddr target = std::max(begin, vma_handle->second.base); return MakeResult<VAddr>(target); } @@ -221,11 +222,6 @@ ResultCode VMManager::UnmapRange(VAddr target, u64 size) { ASSERT(FindVMA(target)->second.size >= size); - system.ArmInterface(0).UnmapMemory(target, size); - system.ArmInterface(1).UnmapMemory(target, size); - system.ArmInterface(2).UnmapMemory(target, size); - system.ArmInterface(3).UnmapMemory(target, size); - return RESULT_SUCCESS; } @@ -265,7 +261,7 @@ ResultVal<VAddr> VMManager::SetHeapSize(u64 size) { if (heap_memory == nullptr) { // Initialize heap - heap_memory = std::make_shared<std::vector<u8>>(size); + heap_memory = std::make_shared<PhysicalMemory>(size); heap_end = heap_region_base + size; } else { UnmapRange(heap_region_base, GetCurrentHeapSize()); @@ -299,6 +295,162 @@ ResultVal<VAddr> VMManager::SetHeapSize(u64 size) { return MakeResult<VAddr>(heap_region_base); } +ResultCode VMManager::MapPhysicalMemory(VAddr target, u64 size) { + // Check how much memory we've already mapped. + const auto mapped_size_result = SizeOfAllocatedVMAsInRange(target, size); + if (mapped_size_result.Failed()) { + return mapped_size_result.Code(); + } + + // If we've already mapped the desired amount, return early. + const std::size_t mapped_size = *mapped_size_result; + if (mapped_size == size) { + return RESULT_SUCCESS; + } + + // Check that we can map the memory we want. + const auto res_limit = system.CurrentProcess()->GetResourceLimit(); + const u64 physmem_remaining = res_limit->GetMaxResourceValue(ResourceType::PhysicalMemory) - + res_limit->GetCurrentResourceValue(ResourceType::PhysicalMemory); + if (physmem_remaining < (size - mapped_size)) { + return ERR_RESOURCE_LIMIT_EXCEEDED; + } + + // Keep track of the memory regions we unmap. + std::vector<std::pair<u64, u64>> mapped_regions; + ResultCode result = RESULT_SUCCESS; + + // Iterate, trying to map memory. + { + const auto end_addr = target + size; + const auto last_addr = end_addr - 1; + VAddr cur_addr = target; + + auto iter = FindVMA(target); + ASSERT(iter != vma_map.end()); + + while (true) { + const auto& vma = iter->second; + const auto vma_start = vma.base; + const auto vma_end = vma_start + vma.size; + const auto vma_last = vma_end - 1; + + // Map the memory block + const auto map_size = std::min(end_addr - cur_addr, vma_end - cur_addr); + if (vma.state == MemoryState::Unmapped) { + const auto map_res = + MapMemoryBlock(cur_addr, std::make_shared<PhysicalMemory>(map_size), 0, + map_size, MemoryState::Heap, VMAPermission::ReadWrite); + result = map_res.Code(); + if (result.IsError()) { + break; + } + + mapped_regions.emplace_back(cur_addr, map_size); + } + + // Break once we hit the end of the range. + if (last_addr <= vma_last) { + break; + } + + // Advance to the next block. + cur_addr = vma_end; + iter = FindVMA(cur_addr); + ASSERT(iter != vma_map.end()); + } + } + + // If we failed, unmap memory. + if (result.IsError()) { + for (const auto [unmap_address, unmap_size] : mapped_regions) { + ASSERT_MSG(UnmapRange(unmap_address, unmap_size).IsSuccess(), + "Failed to unmap memory range."); + } + + return result; + } + + // Update amount of mapped physical memory. + physical_memory_mapped += size - mapped_size; + + return RESULT_SUCCESS; +} + +ResultCode VMManager::UnmapPhysicalMemory(VAddr target, u64 size) { + // Check how much memory is currently mapped. + const auto mapped_size_result = SizeOfUnmappablePhysicalMemoryInRange(target, size); + if (mapped_size_result.Failed()) { + return mapped_size_result.Code(); + } + + // If we've already unmapped all the memory, return early. + const std::size_t mapped_size = *mapped_size_result; + if (mapped_size == 0) { + return RESULT_SUCCESS; + } + + // Keep track of the memory regions we unmap. + std::vector<std::pair<u64, u64>> unmapped_regions; + ResultCode result = RESULT_SUCCESS; + + // Try to unmap regions. + { + const auto end_addr = target + size; + const auto last_addr = end_addr - 1; + VAddr cur_addr = target; + + auto iter = FindVMA(target); + ASSERT(iter != vma_map.end()); + + while (true) { + const auto& vma = iter->second; + const auto vma_start = vma.base; + const auto vma_end = vma_start + vma.size; + const auto vma_last = vma_end - 1; + + // Unmap the memory block + const auto unmap_size = std::min(end_addr - cur_addr, vma_end - cur_addr); + if (vma.state == MemoryState::Heap) { + result = UnmapRange(cur_addr, unmap_size); + if (result.IsError()) { + break; + } + + unmapped_regions.emplace_back(cur_addr, unmap_size); + } + + // Break once we hit the end of the range. + if (last_addr <= vma_last) { + break; + } + + // Advance to the next block. + cur_addr = vma_end; + iter = FindVMA(cur_addr); + ASSERT(iter != vma_map.end()); + } + } + + // If we failed, re-map regions. + // TODO: Preserve memory contents? + if (result.IsError()) { + for (const auto [map_address, map_size] : unmapped_regions) { + const auto remap_res = + MapMemoryBlock(map_address, std::make_shared<PhysicalMemory>(map_size), 0, map_size, + MemoryState::Heap, VMAPermission::None); + ASSERT_MSG(remap_res.Succeeded(), "Failed to remap a memory block."); + } + + return result; + } + + // Update mapped amount + physical_memory_mapped -= mapped_size; + + return RESULT_SUCCESS; +} + ResultCode VMManager::MapCodeMemory(VAddr dst_address, VAddr src_address, u64 size) { constexpr auto ignore_attribute = MemoryAttribute::LockedForIPC | MemoryAttribute::DeviceMapped; const auto src_check_result = CheckRangeState( @@ -438,7 +590,7 @@ ResultCode VMManager::MirrorMemory(VAddr dst_addr, VAddr src_addr, u64 size, Mem ASSERT_MSG(vma_offset + size <= vma->second.size, "Shared memory exceeds bounds of mapped block"); - const std::shared_ptr<std::vector<u8>>& backing_block = vma->second.backing_block; + const std::shared_ptr<PhysicalMemory>& backing_block = vma->second.backing_block; const std::size_t backing_block_offset = vma->second.offset + vma_offset; CASCADE_RESULT(auto new_vma, @@ -446,12 +598,12 @@ ResultCode VMManager::MirrorMemory(VAddr dst_addr, VAddr src_addr, u64 size, Mem // Protect mirror with permissions from old region Reprotect(new_vma, vma->second.permissions); // Remove permissions from old region - Reprotect(vma, VMAPermission::None); + ReprotectRange(src_addr, size, VMAPermission::None); return RESULT_SUCCESS; } -void VMManager::RefreshMemoryBlockMappings(const std::vector<u8>* block) { +void VMManager::RefreshMemoryBlockMappings(const PhysicalMemory* block) { // If this ever proves to have a noticeable performance impact, allow users of the function to // specify a specific range of addresses to limit the scan to. for (const auto& p : vma_map) { @@ -579,14 +731,14 @@ VMManager::VMAIter VMManager::SplitVMA(VMAIter vma_handle, u64 offset_in_vma) { VMManager::VMAIter VMManager::MergeAdjacent(VMAIter iter) { const VMAIter next_vma = std::next(iter); if (next_vma != vma_map.end() && iter->second.CanBeMergedWith(next_vma->second)) { - iter->second.size += next_vma->second.size; + MergeAdjacentVMA(iter->second, next_vma->second); vma_map.erase(next_vma); } if (iter != vma_map.begin()) { VMAIter prev_vma = std::prev(iter); if (prev_vma->second.CanBeMergedWith(iter->second)) { - prev_vma->second.size += iter->second.size; + MergeAdjacentVMA(prev_vma->second, iter->second); vma_map.erase(iter); iter = prev_vma; } @@ -595,6 +747,44 @@ VMManager::VMAIter VMManager::MergeAdjacent(VMAIter iter) { return iter; } +void VMManager::MergeAdjacentVMA(VirtualMemoryArea& left, const VirtualMemoryArea& right) { + ASSERT(left.CanBeMergedWith(right)); + + // Always merge allocated memory blocks, even when they don't share the same backing block. + if (left.type == VMAType::AllocatedMemoryBlock && + (left.backing_block != right.backing_block || left.offset + left.size != right.offset)) { + const auto right_begin = right.backing_block->begin() + right.offset; + const auto right_end = right_begin + right.size; + + // Check if we can save work. + if (left.offset == 0 && left.size == left.backing_block->size()) { + // Fast case: left is an entire backing block. + left.backing_block->insert(left.backing_block->end(), right_begin, right_end); + } else { + // Slow case: make a new memory block for left and right. + const auto left_begin = left.backing_block->begin() + left.offset; + const auto left_end = left_begin + left.size; + const auto left_size = static_cast<std::size_t>(std::distance(left_begin, left_end)); + const auto right_size = static_cast<std::size_t>(std::distance(right_begin, right_end)); + + auto new_memory = std::make_shared<PhysicalMemory>(); + new_memory->reserve(left_size + right_size); + new_memory->insert(new_memory->end(), left_begin, left_end); + new_memory->insert(new_memory->end(), right_begin, right_end); + + left.backing_block = std::move(new_memory); + left.offset = 0; + } + + // Page table update is needed, because backing memory changed. + left.size += right.size; + UpdatePageTableForVMA(left); + } else { + // Just update the size. + left.size += right.size; + } +} + void VMManager::UpdatePageTableForVMA(const VirtualMemoryArea& vma) { switch (vma.type) { case VMAType::Free: @@ -616,9 +806,11 @@ void VMManager::UpdatePageTableForVMA(const VirtualMemoryArea& vma) { void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType type) { u64 map_region_size = 0; u64 heap_region_size = 0; - u64 new_map_region_size = 0; + u64 stack_region_size = 0; u64 tls_io_region_size = 0; + u64 stack_and_tls_io_end = 0; + switch (type) { case FileSys::ProgramAddressSpaceType::Is32Bit: case FileSys::ProgramAddressSpaceType::Is32BitNoMap: @@ -634,6 +826,7 @@ void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType ty map_region_size = 0; heap_region_size = 0x80000000; } + stack_and_tls_io_end = 0x40000000; break; case FileSys::ProgramAddressSpaceType::Is36Bit: address_space_width = 36; @@ -643,6 +836,7 @@ void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType ty aslr_region_end = aslr_region_base + 0xFF8000000; map_region_size = 0x180000000; heap_region_size = 0x180000000; + stack_and_tls_io_end = 0x80000000; break; case FileSys::ProgramAddressSpaceType::Is39Bit: address_space_width = 39; @@ -652,7 +846,7 @@ void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType ty aslr_region_end = aslr_region_base + 0x7FF8000000; map_region_size = 0x1000000000; heap_region_size = 0x180000000; - new_map_region_size = 0x80000000; + stack_region_size = 0x80000000; tls_io_region_size = 0x1000000000; break; default: @@ -660,6 +854,8 @@ void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType ty return; } + const u64 stack_and_tls_io_begin = aslr_region_base; + address_space_base = 0; address_space_end = 1ULL << address_space_width; @@ -670,15 +866,20 @@ void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType ty heap_region_end = heap_region_base + heap_region_size; heap_end = heap_region_base; - new_map_region_base = heap_region_end; - new_map_region_end = new_map_region_base + new_map_region_size; + stack_region_base = heap_region_end; + stack_region_end = stack_region_base + stack_region_size; - tls_io_region_base = new_map_region_end; + tls_io_region_base = stack_region_end; tls_io_region_end = tls_io_region_base + tls_io_region_size; - if (new_map_region_size == 0) { - new_map_region_base = address_space_base; - new_map_region_end = address_space_end; + if (stack_region_size == 0) { + stack_region_base = stack_and_tls_io_begin; + stack_region_end = stack_and_tls_io_end; + } + + if (tls_io_region_size == 0) { + tls_io_region_base = stack_and_tls_io_begin; + tls_io_region_end = stack_and_tls_io_end; } } @@ -758,6 +959,84 @@ VMManager::CheckResults VMManager::CheckRangeState(VAddr address, u64 size, Memo std::make_tuple(initial_state, initial_permissions, initial_attributes & ~ignore_mask)); } +ResultVal<std::size_t> VMManager::SizeOfAllocatedVMAsInRange(VAddr address, + std::size_t size) const { + const VAddr end_addr = address + size; + const VAddr last_addr = end_addr - 1; + std::size_t mapped_size = 0; + + VAddr cur_addr = address; + auto iter = FindVMA(cur_addr); + ASSERT(iter != vma_map.end()); + + while (true) { + const auto& vma = iter->second; + const VAddr vma_start = vma.base; + const VAddr vma_end = vma_start + vma.size; + const VAddr vma_last = vma_end - 1; + + // Add size if relevant. + if (vma.state != MemoryState::Unmapped) { + mapped_size += std::min(end_addr - cur_addr, vma_end - cur_addr); + } + + // Break once we hit the end of the range. + if (last_addr <= vma_last) { + break; + } + + // Advance to the next block. + cur_addr = vma_end; + iter = std::next(iter); + ASSERT(iter != vma_map.end()); + } + + return MakeResult(mapped_size); +} + +ResultVal<std::size_t> VMManager::SizeOfUnmappablePhysicalMemoryInRange(VAddr address, + std::size_t size) const { + const VAddr end_addr = address + size; + const VAddr last_addr = end_addr - 1; + std::size_t mapped_size = 0; + + VAddr cur_addr = address; + auto iter = FindVMA(cur_addr); + ASSERT(iter != vma_map.end()); + + while (true) { + const auto& vma = iter->second; + const auto vma_start = vma.base; + const auto vma_end = vma_start + vma.size; + const auto vma_last = vma_end - 1; + const auto state = vma.state; + const auto attr = vma.attribute; + + // Memory within region must be free or mapped heap. + if (!((state == MemoryState::Heap && attr == MemoryAttribute::None) || + (state == MemoryState::Unmapped))) { + return ERR_INVALID_ADDRESS_STATE; + } + + // Add size if relevant. + if (state != MemoryState::Unmapped) { + mapped_size += std::min(end_addr - cur_addr, vma_end - cur_addr); + } + + // Break once we hit the end of the range. + if (last_addr <= vma_last) { + break; + } + + // Advance to the next block. + cur_addr = vma_end; + iter = std::next(iter); + ASSERT(iter != vma_map.end()); + } + + return MakeResult(mapped_size); +} + u64 VMManager::GetTotalPhysicalMemoryAvailable() const { LOG_WARNING(Kernel, "(STUBBED) called"); return 0xF8000000; @@ -870,21 +1149,21 @@ bool VMManager::IsWithinMapRegion(VAddr address, u64 size) const { return IsInsideAddressRange(address, size, GetMapRegionBaseAddress(), GetMapRegionEndAddress()); } -VAddr VMManager::GetNewMapRegionBaseAddress() const { - return new_map_region_base; +VAddr VMManager::GetStackRegionBaseAddress() const { + return stack_region_base; } -VAddr VMManager::GetNewMapRegionEndAddress() const { - return new_map_region_end; +VAddr VMManager::GetStackRegionEndAddress() const { + return stack_region_end; } -u64 VMManager::GetNewMapRegionSize() const { - return new_map_region_end - new_map_region_base; +u64 VMManager::GetStackRegionSize() const { + return stack_region_end - stack_region_base; } -bool VMManager::IsWithinNewMapRegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetNewMapRegionBaseAddress(), - GetNewMapRegionEndAddress()); +bool VMManager::IsWithinStackRegion(VAddr address, u64 size) const { + return IsInsideAddressRange(address, size, GetStackRegionBaseAddress(), + GetStackRegionEndAddress()); } VAddr VMManager::GetTLSIORegionBaseAddress() const { |