diff options
Diffstat (limited to 'src/core/hle/kernel/k_page_table.cpp')
| -rw-r--r-- | src/core/hle/kernel/k_page_table.cpp | 379 |
1 files changed, 329 insertions, 50 deletions
diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 0602de1f7..47ea3c89c 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -346,7 +346,8 @@ ResultCode KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std:: return ResultSuccess; } -ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, + ICacheInvalidationStrategy icache_invalidation_strategy) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); @@ -396,7 +397,11 @@ ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std bool reprotected_pages = false; SCOPE_EXIT({ if (reprotected_pages && any_code_pages) { - system.InvalidateCpuInstructionCacheRange(dst_address, size); + if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { + system.InvalidateCpuInstructionCacheRange(dst_address, size); + } else { + system.InvalidateCpuInstructionCaches(); + } } }); @@ -424,6 +429,120 @@ ResultCode KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std return ResultSuccess; } +VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, + std::size_t num_pages, std::size_t alignment, std::size_t offset, + std::size_t guard_pages) { + VAddr address = 0; + + if (num_pages <= region_num_pages) { + if (this->IsAslrEnabled()) { + // Try to directly find a free area up to 8 times. + for (std::size_t i = 0; i < 8; i++) { + const std::size_t random_offset = + KSystemControl::GenerateRandomRange( + 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * + alignment; + const VAddr candidate = + Common::AlignDown((region_start + random_offset), alignment) + offset; + + KMemoryInfo info = this->QueryInfoImpl(candidate); + + if (info.state != KMemoryState::Free) { + continue; + } + if (region_start > candidate) { + continue; + } + if (info.GetAddress() + guard_pages * PageSize > candidate) { + continue; + } + + const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; + if (candidate_end > info.GetLastAddress()) { + continue; + } + if (candidate_end > region_start + region_num_pages * PageSize - 1) { + continue; + } + + address = candidate; + break; + } + // Fall back to finding the first free area with a random offset. + if (address == 0) { + // NOTE: Nintendo does not account for guard pages here. + // This may theoretically cause an offset to be chosen that cannot be mapped. We + // will account for guard pages. + const std::size_t offset_pages = KSystemControl::GenerateRandomRange( + 0, region_num_pages - num_pages - guard_pages); + address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, + region_num_pages - offset_pages, num_pages, + alignment, offset, guard_pages); + } + } + + // Find the first free area. + if (address == 0) { + address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); + } + } + + return address; +} + +ResultCode KPageTable::MakePageGroup(KPageLinkedList& pg, VAddr addr, size_t num_pages) { + ASSERT(this->IsLockedByCurrentThread()); + + const size_t size = num_pages * PageSize; + + // We're making a new group, not adding to an existing one. + R_UNLESS(pg.Empty(), ResultInvalidCurrentMemory); + + // Begin traversal. + Common::PageTable::TraversalContext context; + Common::PageTable::TraversalEntry next_entry; + R_UNLESS(page_table_impl.BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); + + // Prepare tracking variables. + PAddr cur_addr = next_entry.phys_addr; + size_t cur_size = next_entry.block_size - (cur_addr & (next_entry.block_size - 1)); + size_t tot_size = cur_size; + + // Iterate, adding to group as we go. + const auto& memory_layout = system.Kernel().MemoryLayout(); + while (tot_size < size) { + R_UNLESS(page_table_impl.ContinueTraversal(next_entry, context), + ResultInvalidCurrentMemory); + + if (next_entry.phys_addr != (cur_addr + cur_size)) { + const size_t cur_pages = cur_size / PageSize; + + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); + + cur_addr = next_entry.phys_addr; + cur_size = next_entry.block_size; + } else { + cur_size += next_entry.block_size; + } + + tot_size += next_entry.block_size; + } + + // Ensure we add the right amount for the last block. + if (tot_size > size) { + cur_size -= (tot_size - size); + } + + // Add the last block. + const size_t cur_pages = cur_size / PageSize; + R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); + R_TRY(pg.AddBlock(cur_addr, cur_pages)); + + return ResultSuccess; +} + ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, VAddr src_addr) { KScopedLightLock lk(general_lock); @@ -449,6 +568,8 @@ ResultCode KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None); + system.InvalidateCpuInstructionCaches(); + return ResultSuccess; } @@ -1055,6 +1176,46 @@ ResultCode KPageTable::MapPages(VAddr address, KPageLinkedList& page_linked_list return ResultSuccess; } +ResultCode KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, + PAddr phys_addr, bool is_pa_valid, VAddr region_start, + std::size_t region_num_pages, KMemoryState state, + KMemoryPermission perm) { + ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); + + // Ensure this is a valid map request. + R_UNLESS(this->CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Find a random address to map at. + VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, + this->GetNumGuardPages()); + R_UNLESS(addr != 0, ResultOutOfMemory); + ASSERT(Common::IsAligned(addr, alignment)); + ASSERT(this->CanContain(addr, num_pages * PageSize, state)); + ASSERT(this->CheckMemoryState(addr, num_pages * PageSize, KMemoryState::All, KMemoryState::Free, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryAttribute::None) + .IsSuccess()); + + // Perform mapping operation. + if (is_pa_valid) { + R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); + } else { + UNIMPLEMENTED(); + } + + // Update the blocks. + block_manager->Update(addr, num_pages, state, perm); + + // We successfully mapped the pages. + *out_addr = addr; + return ResultSuccess; +} + ResultCode KPageTable::UnmapPages(VAddr addr, const KPageLinkedList& page_linked_list) { ASSERT(this->IsLockedByCurrentThread()); @@ -1097,6 +1258,55 @@ ResultCode KPageTable::UnmapPages(VAddr addr, KPageLinkedList& page_linked_list, return ResultSuccess; } +ResultCode KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state) { + // Check that the unmap is in range. + const std::size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the memory state. + std::size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, state, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::None)); + + // Perform the unmap. + R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + + // Update the blocks. + block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); + + return ResultSuccess; +} + +ResultCode KPageTable::MakeAndOpenPageGroup(KPageLinkedList* out, VAddr address, size_t num_pages, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr) { + // Ensure that the page group isn't null. + ASSERT(out != nullptr); + + // Make sure that the region we're mapping is valid for the table. + const size_t size = num_pages * PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check if state allows us to create the group. + R_TRY(this->CheckMemoryState(address, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Create a new page group for the region. + R_TRY(this->MakePageGroup(*out, address, num_pages)); + + return ResultSuccess; +} + ResultCode KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; @@ -1479,57 +1689,21 @@ ResultCode KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) } ResultCode KPageTable::LockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, - KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->ShareToDevice(permission); - }, - new_perm); - - return ResultSuccess; + return this->LockMemoryAndOpen( + nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, + KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, + KMemoryAttribute::All, KMemoryAttribute::None, + static_cast<KMemoryPermission>(KMemoryPermission::NotMapped | + KMemoryPermission::KernelReadWrite), + KMemoryAttribute::Locked); } ResultCode KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission new_perm = KMemoryPermission::UserReadWrite; - - KMemoryPermission old_perm{}; - - if (const ResultCode result{CheckMemoryState( - nullptr, &old_perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, - KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::All, KMemoryAttribute::Locked)}; - result.IsError()) { - return result; - } - - new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->UnshareToDevice(permission); - }, - new_perm); - - return ResultSuccess; + return this->UnlockMemory(addr, size, KMemoryState::FlagCanCodeMemory, + KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, + KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, + KMemoryAttribute::Locked, nullptr); } ResultCode KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { @@ -1865,4 +2039,109 @@ ResultCode KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermissi return ResultSuccess; } +ResultCode KPageTable::LockMemoryAndOpen(KPageLinkedList* out_pg, PAddr* out_paddr, VAddr addr, + size_t size, KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr, + KMemoryPermission new_perm, KMemoryAttribute lock_attr) { + // Validate basic preconditions. + ASSERT((lock_attr & attr) == KMemoryAttribute::None); + ASSERT((lock_attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == + KMemoryAttribute::None); + + // Validate the lock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check that the output page group is empty, if it exists. + if (out_pg) { + ASSERT(out_pg->GetNumPages() == 0); + } + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Get the physical address, if we're supposed to. + if (out_paddr != nullptr) { + ASSERT(this->GetPhysicalAddressLocked(out_paddr, addr)); + } + + // Make the page group, if we're supposed to. + if (out_pg != nullptr) { + R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + +ResultCode KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, + KMemoryState state, KMemoryPermission perm_mask, + KMemoryPermission perm, KMemoryAttribute attr_mask, + KMemoryAttribute attr, KMemoryPermission new_perm, + KMemoryAttribute lock_attr, const KPageLinkedList* pg) { + // Validate basic preconditions. + ASSERT((attr_mask & lock_attr) == lock_attr); + ASSERT((attr & lock_attr) == lock_attr); + + // Validate the unlock request. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(general_lock); + + // Check the state. + KMemoryState old_state{}; + KMemoryPermission old_perm{}; + KMemoryAttribute old_attr{}; + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), + std::addressof(old_attr), std::addressof(num_allocator_blocks), + addr, size, state_mask | KMemoryState::FlagReferenceCounted, + state | KMemoryState::FlagReferenceCounted, perm_mask, perm, + attr_mask, attr)); + + // Check the page group. + if (pg != nullptr) { + UNIMPLEMENTED_MSG("PageGroup support is unimplemented!"); + } + + // Decide on new perm and attr. + new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; + KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); + + // Update permission, if we need to. + if (new_perm != old_perm) { + R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); + } + + // Apply the memory block updates. + block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + + return ResultSuccess; +} + } // namespace Kernel |