diff options
Diffstat (limited to 'src/core/memory.cpp')
| -rw-r--r-- | src/core/memory.cpp | 1008 |
1 files changed, 589 insertions, 419 deletions
diff --git a/src/core/memory.cpp b/src/core/memory.cpp index fa49f3dd0..91bf07a92 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -17,529 +17,699 @@ #include "core/hle/kernel/process.h" #include "core/hle/kernel/vm_manager.h" #include "core/memory.h" -#include "core/memory_setup.h" #include "video_core/gpu.h" namespace Memory { -static Common::PageTable* current_page_table = nullptr; +// Implementation class used to keep the specifics of the memory subsystem hidden +// from outside classes. This also allows modification to the internals of the memory +// subsystem without needing to rebuild all files that make use of the memory interface. +struct Memory::Impl { + explicit Impl(Core::System& system_) : system{system_} {} -void SetCurrentPageTable(Kernel::Process& process) { - current_page_table = &process.VMManager().page_table; + void SetCurrentPageTable(Kernel::Process& process) { + current_page_table = &process.VMManager().page_table; - const std::size_t address_space_width = process.VMManager().GetAddressSpaceWidth(); + const std::size_t address_space_width = process.VMManager().GetAddressSpaceWidth(); - auto& system = Core::System::GetInstance(); - system.ArmInterface(0).PageTableChanged(*current_page_table, address_space_width); - system.ArmInterface(1).PageTableChanged(*current_page_table, address_space_width); - system.ArmInterface(2).PageTableChanged(*current_page_table, address_space_width); - system.ArmInterface(3).PageTableChanged(*current_page_table, address_space_width); -} + system.ArmInterface(0).PageTableChanged(*current_page_table, address_space_width); + system.ArmInterface(1).PageTableChanged(*current_page_table, address_space_width); + system.ArmInterface(2).PageTableChanged(*current_page_table, address_space_width); + system.ArmInterface(3).PageTableChanged(*current_page_table, address_space_width); + } -static void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* memory, - Common::PageType type) { - LOG_DEBUG(HW_Memory, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * PAGE_SIZE, - (base + size) * PAGE_SIZE); - - // During boot, current_page_table might not be set yet, in which case we need not flush - if (Core::System::GetInstance().IsPoweredOn()) { - auto& gpu = Core::System::GetInstance().GPU(); - for (u64 i = 0; i < size; i++) { - const auto page = base + i; - if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) { - gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE); - } - } + void MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) { + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); + MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, Common::PageType::Memory); } - VAddr end = base + size; - ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", - base + page_table.pointers.size()); + void MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer mmio_handler) { + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); + MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, + Common::PageType::Special); + + const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); + const Common::SpecialRegion region{Common::SpecialRegion::Type::IODevice, + std::move(mmio_handler)}; + page_table.special_regions.add( + std::make_pair(interval, std::set<Common::SpecialRegion>{region})); + } - std::fill(page_table.attributes.begin() + base, page_table.attributes.begin() + end, type); + void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) { + ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); + ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); + MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, + Common::PageType::Unmapped); - if (memory == nullptr) { - std::fill(page_table.pointers.begin() + base, page_table.pointers.begin() + end, memory); - } else { - while (base != end) { - page_table.pointers[base] = memory; + const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); + page_table.special_regions.erase(interval); + } - base += 1; - memory += PAGE_SIZE; - } + void AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer hook) { + const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); + const Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)}; + page_table.special_regions.add( + std::make_pair(interval, std::set<Common::SpecialRegion>{region})); } -} -void MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) { - ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); - MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, Common::PageType::Memory); -} + void RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer hook) { + const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); + const Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)}; + page_table.special_regions.subtract( + std::make_pair(interval, std::set<Common::SpecialRegion>{region})); + } -void MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size, - Common::MemoryHookPointer mmio_handler) { - ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); - MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, Common::PageType::Special); + bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const { + const auto& page_table = process.VMManager().page_table; - auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); - Common::SpecialRegion region{Common::SpecialRegion::Type::IODevice, std::move(mmio_handler)}; - page_table.special_regions.add( - std::make_pair(interval, std::set<Common::SpecialRegion>{region})); -} + const u8* const page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; + if (page_pointer != nullptr) { + return true; + } -void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) { - ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size); - ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base); - MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, Common::PageType::Unmapped); + if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) { + return true; + } - auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); - page_table.special_regions.erase(interval); -} + if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) { + return false; + } -void AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size, - Common::MemoryHookPointer hook) { - auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); - Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)}; - page_table.special_regions.add( - std::make_pair(interval, std::set<Common::SpecialRegion>{region})); -} + return false; + } -void RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size, - Common::MemoryHookPointer hook) { - auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1); - Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)}; - page_table.special_regions.subtract( - std::make_pair(interval, std::set<Common::SpecialRegion>{region})); -} + bool IsValidVirtualAddress(VAddr vaddr) const { + return IsValidVirtualAddress(*system.CurrentProcess(), vaddr); + } -/** - * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) - * using a VMA from the current process - */ -static u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) { - const auto& vm_manager = process.VMManager(); - - const auto it = vm_manager.FindVMA(vaddr); - DEBUG_ASSERT(vm_manager.IsValidHandle(it)); - - u8* direct_pointer = nullptr; - const auto& vma = it->second; - switch (vma.type) { - case Kernel::VMAType::AllocatedMemoryBlock: - direct_pointer = vma.backing_block->data() + vma.offset; - break; - case Kernel::VMAType::BackingMemory: - direct_pointer = vma.backing_memory; - break; - case Kernel::VMAType::Free: - return nullptr; - default: - UNREACHABLE(); + /** + * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) + * using a VMA from the current process + */ + u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) { + const auto& vm_manager = process.VMManager(); + + const auto it = vm_manager.FindVMA(vaddr); + DEBUG_ASSERT(vm_manager.IsValidHandle(it)); + + u8* direct_pointer = nullptr; + const auto& vma = it->second; + switch (vma.type) { + case Kernel::VMAType::AllocatedMemoryBlock: + direct_pointer = vma.backing_block->data() + vma.offset; + break; + case Kernel::VMAType::BackingMemory: + direct_pointer = vma.backing_memory; + break; + case Kernel::VMAType::Free: + return nullptr; + default: + UNREACHABLE(); + } + + return direct_pointer + (vaddr - vma.base); } - return direct_pointer + (vaddr - vma.base); -} + /** + * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) + * using a VMA from the current process. + */ + u8* GetPointerFromVMA(VAddr vaddr) { + return GetPointerFromVMA(*system.CurrentProcess(), vaddr); + } -/** - * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) - * using a VMA from the current process. - */ -static u8* GetPointerFromVMA(VAddr vaddr) { - return GetPointerFromVMA(*Core::System::GetInstance().CurrentProcess(), vaddr); -} + u8* GetPointer(const VAddr vaddr) { + u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; + if (page_pointer != nullptr) { + return page_pointer + (vaddr & PAGE_MASK); + } -template <typename T> -T Read(const VAddr vaddr) { - const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; - if (page_pointer) { - // NOTE: Avoid adding any extra logic to this fast-path block - T value; - std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T)); - return value; - } - - Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; - switch (type) { - case Common::PageType::Unmapped: - LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); - return 0; - case Common::PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); - break; - case Common::PageType::RasterizerCachedMemory: { - auto host_ptr{GetPointerFromVMA(vaddr)}; - Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), sizeof(T)); - T value; - std::memcpy(&value, host_ptr, sizeof(T)); - return value; - } - default: - UNREACHABLE(); - } - return {}; -} + if (current_page_table->attributes[vaddr >> PAGE_BITS] == + Common::PageType::RasterizerCachedMemory) { + return GetPointerFromVMA(vaddr); + } -template <typename T> -void Write(const VAddr vaddr, const T data) { - u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; - if (page_pointer) { - // NOTE: Avoid adding any extra logic to this fast-path block - std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T)); - return; - } - - Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; - switch (type) { - case Common::PageType::Unmapped: - LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, - static_cast<u32>(data), vaddr); - return; - case Common::PageType::Memory: - ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); - break; - case Common::PageType::RasterizerCachedMemory: { - auto host_ptr{GetPointerFromVMA(vaddr)}; - Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), sizeof(T)); - std::memcpy(host_ptr, &data, sizeof(T)); - break; - } - default: - UNREACHABLE(); + LOG_ERROR(HW_Memory, "Unknown GetPointer @ 0x{:016X}", vaddr); + return nullptr; } -} -bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) { - const auto& page_table = process.VMManager().page_table; + u8 Read8(const VAddr addr) { + return Read<u8>(addr); + } - const u8* page_pointer = page_table.pointers[vaddr >> PAGE_BITS]; - if (page_pointer) - return true; + u16 Read16(const VAddr addr) { + return Read<u16_le>(addr); + } - if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) - return true; + u32 Read32(const VAddr addr) { + return Read<u32_le>(addr); + } - if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) - return false; + u64 Read64(const VAddr addr) { + return Read<u64_le>(addr); + } - return false; -} + void Write8(const VAddr addr, const u8 data) { + Write<u8>(addr, data); + } -bool IsValidVirtualAddress(const VAddr vaddr) { - return IsValidVirtualAddress(*Core::System::GetInstance().CurrentProcess(), vaddr); -} + void Write16(const VAddr addr, const u16 data) { + Write<u16_le>(addr, data); + } -bool IsKernelVirtualAddress(const VAddr vaddr) { - return KERNEL_REGION_VADDR <= vaddr && vaddr < KERNEL_REGION_END; -} + void Write32(const VAddr addr, const u32 data) { + Write<u32_le>(addr, data); + } -u8* GetPointer(const VAddr vaddr) { - u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; - if (page_pointer) { - return page_pointer + (vaddr & PAGE_MASK); + void Write64(const VAddr addr, const u64 data) { + Write<u64_le>(addr, data); } - if (current_page_table->attributes[vaddr >> PAGE_BITS] == - Common::PageType::RasterizerCachedMemory) { - return GetPointerFromVMA(vaddr); + std::string ReadCString(VAddr vaddr, std::size_t max_length) { + std::string string; + string.reserve(max_length); + for (std::size_t i = 0; i < max_length; ++i) { + const char c = Read8(vaddr); + if (c == '\0') { + break; + } + string.push_back(c); + ++vaddr; + } + string.shrink_to_fit(); + return string; } - LOG_ERROR(HW_Memory, "Unknown GetPointer @ 0x{:016X}", vaddr); - return nullptr; -} + void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer, + const std::size_t size) { + const auto& page_table = process.VMManager().page_table; + + std::size_t remaining_size = size; + std::size_t page_index = src_addr >> PAGE_BITS; + std::size_t page_offset = src_addr & PAGE_MASK; + + while (remaining_size > 0) { + const std::size_t copy_amount = + std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); + const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); + + switch (page_table.attributes[page_index]) { + case Common::PageType::Unmapped: { + LOG_ERROR(HW_Memory, + "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, src_addr, size); + std::memset(dest_buffer, 0, copy_amount); + break; + } + case Common::PageType::Memory: { + DEBUG_ASSERT(page_table.pointers[page_index]); -std::string ReadCString(VAddr vaddr, std::size_t max_length) { - std::string string; - string.reserve(max_length); - for (std::size_t i = 0; i < max_length; ++i) { - char c = Read8(vaddr); - if (c == '\0') - break; - string.push_back(c); - ++vaddr; + const u8* const src_ptr = page_table.pointers[page_index] + page_offset; + std::memcpy(dest_buffer, src_ptr, copy_amount); + break; + } + case Common::PageType::RasterizerCachedMemory: { + const u8* const host_ptr = GetPointerFromVMA(process, current_vaddr); + system.GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount); + std::memcpy(dest_buffer, host_ptr, copy_amount); + break; + } + default: + UNREACHABLE(); + } + + page_index++; + page_offset = 0; + dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; + remaining_size -= copy_amount; + } } - string.shrink_to_fit(); - return string; -} -void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { - if (vaddr == 0) { - return; + void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) { + ReadBlock(*system.CurrentProcess(), src_addr, dest_buffer, size); } - // Iterate over a contiguous CPU address space, which corresponds to the specified GPU address - // space, marking the region as un/cached. The region is marked un/cached at a granularity of - // CPU pages, hence why we iterate on a CPU page basis (note: GPU page size is different). This - // assumes the specified GPU address region is contiguous as well. + void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer, + const std::size_t size) { + const auto& page_table = process.VMManager().page_table; + std::size_t remaining_size = size; + std::size_t page_index = dest_addr >> PAGE_BITS; + std::size_t page_offset = dest_addr & PAGE_MASK; + + while (remaining_size > 0) { + const std::size_t copy_amount = + std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); + const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); + + switch (page_table.attributes[page_index]) { + case Common::PageType::Unmapped: { + LOG_ERROR(HW_Memory, + "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, dest_addr, size); + break; + } + case Common::PageType::Memory: { + DEBUG_ASSERT(page_table.pointers[page_index]); + + u8* const dest_ptr = page_table.pointers[page_index] + page_offset; + std::memcpy(dest_ptr, src_buffer, copy_amount); + break; + } + case Common::PageType::RasterizerCachedMemory: { + u8* const host_ptr = GetPointerFromVMA(process, current_vaddr); + system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount); + std::memcpy(host_ptr, src_buffer, copy_amount); + break; + } + default: + UNREACHABLE(); + } + + page_index++; + page_offset = 0; + src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; + remaining_size -= copy_amount; + } + } - u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1; - for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) { - Common::PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; + void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { + WriteBlock(*system.CurrentProcess(), dest_addr, src_buffer, size); + } - if (cached) { - // Switch page type to cached if now cached - switch (page_type) { - case Common::PageType::Unmapped: - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. + void ZeroBlock(const Kernel::Process& process, const VAddr dest_addr, const std::size_t size) { + const auto& page_table = process.VMManager().page_table; + std::size_t remaining_size = size; + std::size_t page_index = dest_addr >> PAGE_BITS; + std::size_t page_offset = dest_addr & PAGE_MASK; + + while (remaining_size > 0) { + const std::size_t copy_amount = + std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); + const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); + + switch (page_table.attributes[page_index]) { + case Common::PageType::Unmapped: { + LOG_ERROR(HW_Memory, + "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, dest_addr, size); break; - case Common::PageType::Memory: - page_type = Common::PageType::RasterizerCachedMemory; - current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; + } + case Common::PageType::Memory: { + DEBUG_ASSERT(page_table.pointers[page_index]); + + u8* dest_ptr = page_table.pointers[page_index] + page_offset; + std::memset(dest_ptr, 0, copy_amount); break; - case Common::PageType::RasterizerCachedMemory: - // There can be more than one GPU region mapped per CPU region, so it's common that - // this area is already marked as cached. + } + case Common::PageType::RasterizerCachedMemory: { + u8* const host_ptr = GetPointerFromVMA(process, current_vaddr); + system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount); + std::memset(host_ptr, 0, copy_amount); break; + } default: UNREACHABLE(); } - } else { - // Switch page type to uncached if now uncached - switch (page_type) { - case Common::PageType::Unmapped: - // It is not necessary for a process to have this region mapped into its address - // space, for example, a system module need not have a VRAM mapping. + + page_index++; + page_offset = 0; + remaining_size -= copy_amount; + } + } + + void ZeroBlock(const VAddr dest_addr, const std::size_t size) { + ZeroBlock(*system.CurrentProcess(), dest_addr, size); + } + + void CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr, + const std::size_t size) { + const auto& page_table = process.VMManager().page_table; + std::size_t remaining_size = size; + std::size_t page_index = src_addr >> PAGE_BITS; + std::size_t page_offset = src_addr & PAGE_MASK; + + while (remaining_size > 0) { + const std::size_t copy_amount = + std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); + const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); + + switch (page_table.attributes[page_index]) { + case Common::PageType::Unmapped: { + LOG_ERROR(HW_Memory, + "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", + current_vaddr, src_addr, size); + ZeroBlock(process, dest_addr, copy_amount); break; - case Common::PageType::Memory: - // There can be more than one GPU region mapped per CPU region, so it's common that - // this area is already unmarked as cached. + } + case Common::PageType::Memory: { + DEBUG_ASSERT(page_table.pointers[page_index]); + const u8* src_ptr = page_table.pointers[page_index] + page_offset; + WriteBlock(process, dest_addr, src_ptr, copy_amount); break; + } case Common::PageType::RasterizerCachedMemory: { - u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK); - if (pointer == nullptr) { - // It's possible that this function has been called while updating the pagetable - // after unmapping a VMA. In that case the underlying VMA will no longer exist, - // and we should just leave the pagetable entry blank. - page_type = Common::PageType::Unmapped; - } else { - page_type = Common::PageType::Memory; - current_page_table->pointers[vaddr >> PAGE_BITS] = pointer; - } + const u8* const host_ptr = GetPointerFromVMA(process, current_vaddr); + system.GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount); + WriteBlock(process, dest_addr, host_ptr, copy_amount); break; } default: UNREACHABLE(); } + + page_index++; + page_offset = 0; + dest_addr += static_cast<VAddr>(copy_amount); + src_addr += static_cast<VAddr>(copy_amount); + remaining_size -= copy_amount; } } -} -u8 Read8(const VAddr addr) { - return Read<u8>(addr); -} + void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) { + return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size); + } -u16 Read16(const VAddr addr) { - return Read<u16_le>(addr); -} + void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { + if (vaddr == 0) { + return; + } -u32 Read32(const VAddr addr) { - return Read<u32_le>(addr); -} + // Iterate over a contiguous CPU address space, which corresponds to the specified GPU + // address space, marking the region as un/cached. The region is marked un/cached at a + // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size + // is different). This assumes the specified GPU address region is contiguous as well. + + u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1; + for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) { + Common::PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; + + if (cached) { + // Switch page type to cached if now cached + switch (page_type) { + case Common::PageType::Unmapped: + // It is not necessary for a process to have this region mapped into its address + // space, for example, a system module need not have a VRAM mapping. + break; + case Common::PageType::Memory: + page_type = Common::PageType::RasterizerCachedMemory; + current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; + break; + case Common::PageType::RasterizerCachedMemory: + // There can be more than one GPU region mapped per CPU region, so it's common + // that this area is already marked as cached. + break; + default: + UNREACHABLE(); + } + } else { + // Switch page type to uncached if now uncached + switch (page_type) { + case Common::PageType::Unmapped: + // It is not necessary for a process to have this region mapped into its address + // space, for example, a system module need not have a VRAM mapping. + break; + case Common::PageType::Memory: + // There can be more than one GPU region mapped per CPU region, so it's common + // that this area is already unmarked as cached. + break; + case Common::PageType::RasterizerCachedMemory: { + u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK); + if (pointer == nullptr) { + // It's possible that this function has been called while updating the + // pagetable after unmapping a VMA. In that case the underlying VMA will no + // longer exist, and we should just leave the pagetable entry blank. + page_type = Common::PageType::Unmapped; + } else { + page_type = Common::PageType::Memory; + current_page_table->pointers[vaddr >> PAGE_BITS] = pointer; + } + break; + } + default: + UNREACHABLE(); + } + } + } + } -u64 Read64(const VAddr addr) { - return Read<u64_le>(addr); -} + /** + * Maps a region of pages as a specific type. + * + * @param page_table The page table to use to perform the mapping. + * @param base The base address to begin mapping at. + * @param size The total size of the range in bytes. + * @param memory The memory to map. + * @param type The page type to map the memory as. + */ + void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* memory, + Common::PageType type) { + LOG_DEBUG(HW_Memory, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * PAGE_SIZE, + (base + size) * PAGE_SIZE); + + // During boot, current_page_table might not be set yet, in which case we need not flush + if (system.IsPoweredOn()) { + auto& gpu = system.GPU(); + for (u64 i = 0; i < size; i++) { + const auto page = base + i; + if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) { + gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE); + } + } + } -void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer, - const std::size_t size) { - const auto& page_table = process.VMManager().page_table; - - std::size_t remaining_size = size; - std::size_t page_index = src_addr >> PAGE_BITS; - std::size_t page_offset = src_addr & PAGE_MASK; - - while (remaining_size > 0) { - const std::size_t copy_amount = - std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); - const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); - - switch (page_table.attributes[page_index]) { - case Common::PageType::Unmapped: { - LOG_ERROR(HW_Memory, - "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - current_vaddr, src_addr, size); - std::memset(dest_buffer, 0, copy_amount); - break; + const VAddr end = base + size; + ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}", + base + page_table.pointers.size()); + + std::fill(page_table.attributes.begin() + base, page_table.attributes.begin() + end, type); + + if (memory == nullptr) { + std::fill(page_table.pointers.begin() + base, page_table.pointers.begin() + end, + memory); + } else { + while (base != end) { + page_table.pointers[base] = memory; + + base += 1; + memory += PAGE_SIZE; + } + } + } + + /** + * Reads a particular data type out of memory at the given virtual address. + * + * @param vaddr The virtual address to read the data type from. + * + * @tparam T The data type to read out of memory. This type *must* be + * trivially copyable, otherwise the behavior of this function + * is undefined. + * + * @returns The instance of T read from the specified virtual address. + */ + template <typename T> + T Read(const VAddr vaddr) { + const u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; + if (page_pointer != nullptr) { + // NOTE: Avoid adding any extra logic to this fast-path block + T value; + std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T)); + return value; } - case Common::PageType::Memory: { - DEBUG_ASSERT(page_table.pointers[page_index]); - const u8* src_ptr = page_table.pointers[page_index] + page_offset; - std::memcpy(dest_buffer, src_ptr, copy_amount); + const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; + switch (type) { + case Common::PageType::Unmapped: + LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr); + return 0; + case Common::PageType::Memory: + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); break; + case Common::PageType::RasterizerCachedMemory: { + const u8* const host_ptr = GetPointerFromVMA(vaddr); + system.GPU().FlushRegion(ToCacheAddr(host_ptr), sizeof(T)); + T value; + std::memcpy(&value, host_ptr, sizeof(T)); + return value; + } + default: + UNREACHABLE(); } + return {}; + } + + /** + * Writes a particular data type to memory at the given virtual address. + * + * @param vaddr The virtual address to write the data type to. + * + * @tparam T The data type to write to memory. This type *must* be + * trivially copyable, otherwise the behavior of this function + * is undefined. + * + * @returns The instance of T write to the specified virtual address. + */ + template <typename T> + void Write(const VAddr vaddr, const T data) { + u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; + if (page_pointer != nullptr) { + // NOTE: Avoid adding any extra logic to this fast-path block + std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T)); + return; + } + + const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS]; + switch (type) { + case Common::PageType::Unmapped: + LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8, + static_cast<u32>(data), vaddr); + return; + case Common::PageType::Memory: + ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr); + break; case Common::PageType::RasterizerCachedMemory: { - const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)}; - Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount); - std::memcpy(dest_buffer, host_ptr, copy_amount); + u8* const host_ptr{GetPointerFromVMA(vaddr)}; + system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), sizeof(T)); + std::memcpy(host_ptr, &data, sizeof(T)); break; } default: UNREACHABLE(); } - - page_index++; - page_offset = 0; - dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; - remaining_size -= copy_amount; } + + Common::PageTable* current_page_table = nullptr; + Core::System& system; +}; + +Memory::Memory(Core::System& system) : impl{std::make_unique<Impl>(system)} {} +Memory::~Memory() = default; + +void Memory::SetCurrentPageTable(Kernel::Process& process) { + impl->SetCurrentPageTable(process); } -void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) { - ReadBlock(*Core::System::GetInstance().CurrentProcess(), src_addr, dest_buffer, size); +void Memory::MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) { + impl->MapMemoryRegion(page_table, base, size, target); } -void Write8(const VAddr addr, const u8 data) { - Write<u8>(addr, data); +void Memory::MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer mmio_handler) { + impl->MapIoRegion(page_table, base, size, std::move(mmio_handler)); } -void Write16(const VAddr addr, const u16 data) { - Write<u16_le>(addr, data); +void Memory::UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) { + impl->UnmapRegion(page_table, base, size); } -void Write32(const VAddr addr, const u32 data) { - Write<u32_le>(addr, data); +void Memory::AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer hook) { + impl->AddDebugHook(page_table, base, size, std::move(hook)); } -void Write64(const VAddr addr, const u64 data) { - Write<u64_le>(addr, data); +void Memory::RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size, + Common::MemoryHookPointer hook) { + impl->RemoveDebugHook(page_table, base, size, std::move(hook)); } -void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer, - const std::size_t size) { - const auto& page_table = process.VMManager().page_table; - std::size_t remaining_size = size; - std::size_t page_index = dest_addr >> PAGE_BITS; - std::size_t page_offset = dest_addr & PAGE_MASK; - - while (remaining_size > 0) { - const std::size_t copy_amount = - std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); - const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); - - switch (page_table.attributes[page_index]) { - case Common::PageType::Unmapped: { - LOG_ERROR(HW_Memory, - "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - current_vaddr, dest_addr, size); - break; - } - case Common::PageType::Memory: { - DEBUG_ASSERT(page_table.pointers[page_index]); +bool Memory::IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const { + return impl->IsValidVirtualAddress(process, vaddr); +} - u8* dest_ptr = page_table.pointers[page_index] + page_offset; - std::memcpy(dest_ptr, src_buffer, copy_amount); - break; - } - case Common::PageType::RasterizerCachedMemory: { - const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)}; - Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount); - std::memcpy(host_ptr, src_buffer, copy_amount); - break; - } - default: - UNREACHABLE(); - } +bool Memory::IsValidVirtualAddress(const VAddr vaddr) const { + return impl->IsValidVirtualAddress(vaddr); +} - page_index++; - page_offset = 0; - src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; - remaining_size -= copy_amount; - } +u8* Memory::GetPointer(VAddr vaddr) { + return impl->GetPointer(vaddr); } -void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { - WriteBlock(*Core::System::GetInstance().CurrentProcess(), dest_addr, src_buffer, size); +const u8* Memory::GetPointer(VAddr vaddr) const { + return impl->GetPointer(vaddr); } -void ZeroBlock(const Kernel::Process& process, const VAddr dest_addr, const std::size_t size) { - const auto& page_table = process.VMManager().page_table; - std::size_t remaining_size = size; - std::size_t page_index = dest_addr >> PAGE_BITS; - std::size_t page_offset = dest_addr & PAGE_MASK; - - while (remaining_size > 0) { - const std::size_t copy_amount = - std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); - const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); - - switch (page_table.attributes[page_index]) { - case Common::PageType::Unmapped: { - LOG_ERROR(HW_Memory, - "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - current_vaddr, dest_addr, size); - break; - } - case Common::PageType::Memory: { - DEBUG_ASSERT(page_table.pointers[page_index]); +u8 Memory::Read8(const VAddr addr) { + return impl->Read8(addr); +} - u8* dest_ptr = page_table.pointers[page_index] + page_offset; - std::memset(dest_ptr, 0, copy_amount); - break; - } - case Common::PageType::RasterizerCachedMemory: { - const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)}; - Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount); - std::memset(host_ptr, 0, copy_amount); - break; - } - default: - UNREACHABLE(); - } +u16 Memory::Read16(const VAddr addr) { + return impl->Read16(addr); +} - page_index++; - page_offset = 0; - remaining_size -= copy_amount; - } +u32 Memory::Read32(const VAddr addr) { + return impl->Read32(addr); } -void CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr, - const std::size_t size) { - const auto& page_table = process.VMManager().page_table; - std::size_t remaining_size = size; - std::size_t page_index = src_addr >> PAGE_BITS; - std::size_t page_offset = src_addr & PAGE_MASK; - - while (remaining_size > 0) { - const std::size_t copy_amount = - std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size); - const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset); - - switch (page_table.attributes[page_index]) { - case Common::PageType::Unmapped: { - LOG_ERROR(HW_Memory, - "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", - current_vaddr, src_addr, size); - ZeroBlock(process, dest_addr, copy_amount); - break; - } - case Common::PageType::Memory: { - DEBUG_ASSERT(page_table.pointers[page_index]); - const u8* src_ptr = page_table.pointers[page_index] + page_offset; - WriteBlock(process, dest_addr, src_ptr, copy_amount); - break; - } - case Common::PageType::RasterizerCachedMemory: { - const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)}; - Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount); - WriteBlock(process, dest_addr, host_ptr, copy_amount); - break; - } - default: - UNREACHABLE(); - } +u64 Memory::Read64(const VAddr addr) { + return impl->Read64(addr); +} - page_index++; - page_offset = 0; - dest_addr += static_cast<VAddr>(copy_amount); - src_addr += static_cast<VAddr>(copy_amount); - remaining_size -= copy_amount; - } +void Memory::Write8(VAddr addr, u8 data) { + impl->Write8(addr, data); +} + +void Memory::Write16(VAddr addr, u16 data) { + impl->Write16(addr, data); +} + +void Memory::Write32(VAddr addr, u32 data) { + impl->Write32(addr, data); +} + +void Memory::Write64(VAddr addr, u64 data) { + impl->Write64(addr, data); +} + +std::string Memory::ReadCString(VAddr vaddr, std::size_t max_length) { + return impl->ReadCString(vaddr, max_length); +} + +void Memory::ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer, + const std::size_t size) { + impl->ReadBlock(process, src_addr, dest_buffer, size); +} + +void Memory::ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) { + impl->ReadBlock(src_addr, dest_buffer, size); +} + +void Memory::WriteBlock(const Kernel::Process& process, VAddr dest_addr, const void* src_buffer, + std::size_t size) { + impl->WriteBlock(process, dest_addr, src_buffer, size); +} + +void Memory::WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) { + impl->WriteBlock(dest_addr, src_buffer, size); } -void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) { - CopyBlock(*Core::System::GetInstance().CurrentProcess(), dest_addr, src_addr, size); +void Memory::ZeroBlock(const Kernel::Process& process, VAddr dest_addr, std::size_t size) { + impl->ZeroBlock(process, dest_addr, size); +} + +void Memory::ZeroBlock(VAddr dest_addr, std::size_t size) { + impl->ZeroBlock(dest_addr, size); +} + +void Memory::CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr, + const std::size_t size) { + impl->CopyBlock(process, dest_addr, src_addr, size); +} + +void Memory::CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) { + impl->CopyBlock(dest_addr, src_addr, size); +} + +void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) { + impl->RasterizerMarkRegionCached(vaddr, size, cached); +} + +bool IsKernelVirtualAddress(const VAddr vaddr) { + return KERNEL_REGION_VADDR <= vaddr && vaddr < KERNEL_REGION_END; } } // namespace Memory |