summaryrefslogtreecommitdiffstats
path: root/src/core/memory.cpp
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/core/memory.cpp1008
1 files changed, 589 insertions, 419 deletions
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index fa49f3dd0..91bf07a92 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -17,529 +17,699 @@
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/vm_manager.h"
#include "core/memory.h"
-#include "core/memory_setup.h"
#include "video_core/gpu.h"
namespace Memory {
-static Common::PageTable* current_page_table = nullptr;
+// Implementation class used to keep the specifics of the memory subsystem hidden
+// from outside classes. This also allows modification to the internals of the memory
+// subsystem without needing to rebuild all files that make use of the memory interface.
+struct Memory::Impl {
+ explicit Impl(Core::System& system_) : system{system_} {}
-void SetCurrentPageTable(Kernel::Process& process) {
- current_page_table = &process.VMManager().page_table;
+ void SetCurrentPageTable(Kernel::Process& process) {
+ current_page_table = &process.VMManager().page_table;
- const std::size_t address_space_width = process.VMManager().GetAddressSpaceWidth();
+ const std::size_t address_space_width = process.VMManager().GetAddressSpaceWidth();
- auto& system = Core::System::GetInstance();
- system.ArmInterface(0).PageTableChanged(*current_page_table, address_space_width);
- system.ArmInterface(1).PageTableChanged(*current_page_table, address_space_width);
- system.ArmInterface(2).PageTableChanged(*current_page_table, address_space_width);
- system.ArmInterface(3).PageTableChanged(*current_page_table, address_space_width);
-}
+ system.ArmInterface(0).PageTableChanged(*current_page_table, address_space_width);
+ system.ArmInterface(1).PageTableChanged(*current_page_table, address_space_width);
+ system.ArmInterface(2).PageTableChanged(*current_page_table, address_space_width);
+ system.ArmInterface(3).PageTableChanged(*current_page_table, address_space_width);
+ }
-static void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* memory,
- Common::PageType type) {
- LOG_DEBUG(HW_Memory, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * PAGE_SIZE,
- (base + size) * PAGE_SIZE);
-
- // During boot, current_page_table might not be set yet, in which case we need not flush
- if (Core::System::GetInstance().IsPoweredOn()) {
- auto& gpu = Core::System::GetInstance().GPU();
- for (u64 i = 0; i < size; i++) {
- const auto page = base + i;
- if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) {
- gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE);
- }
- }
+ void MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) {
+ ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
+ ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
+ MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, Common::PageType::Memory);
}
- VAddr end = base + size;
- ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}",
- base + page_table.pointers.size());
+ void MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer mmio_handler) {
+ ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
+ ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
+ MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr,
+ Common::PageType::Special);
+
+ const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
+ const Common::SpecialRegion region{Common::SpecialRegion::Type::IODevice,
+ std::move(mmio_handler)};
+ page_table.special_regions.add(
+ std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
+ }
- std::fill(page_table.attributes.begin() + base, page_table.attributes.begin() + end, type);
+ void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) {
+ ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
+ ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
+ MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr,
+ Common::PageType::Unmapped);
- if (memory == nullptr) {
- std::fill(page_table.pointers.begin() + base, page_table.pointers.begin() + end, memory);
- } else {
- while (base != end) {
- page_table.pointers[base] = memory;
+ const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
+ page_table.special_regions.erase(interval);
+ }
- base += 1;
- memory += PAGE_SIZE;
- }
+ void AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer hook) {
+ const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
+ const Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)};
+ page_table.special_regions.add(
+ std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
}
-}
-void MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) {
- ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
- ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
- MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, target, Common::PageType::Memory);
-}
+ void RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer hook) {
+ const auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
+ const Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)};
+ page_table.special_regions.subtract(
+ std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
+ }
-void MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size,
- Common::MemoryHookPointer mmio_handler) {
- ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
- ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
- MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, Common::PageType::Special);
+ bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const {
+ const auto& page_table = process.VMManager().page_table;
- auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
- Common::SpecialRegion region{Common::SpecialRegion::Type::IODevice, std::move(mmio_handler)};
- page_table.special_regions.add(
- std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
-}
+ const u8* const page_pointer = page_table.pointers[vaddr >> PAGE_BITS];
+ if (page_pointer != nullptr) {
+ return true;
+ }
-void UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) {
- ASSERT_MSG((size & PAGE_MASK) == 0, "non-page aligned size: {:016X}", size);
- ASSERT_MSG((base & PAGE_MASK) == 0, "non-page aligned base: {:016X}", base);
- MapPages(page_table, base / PAGE_SIZE, size / PAGE_SIZE, nullptr, Common::PageType::Unmapped);
+ if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory) {
+ return true;
+ }
- auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
- page_table.special_regions.erase(interval);
-}
+ if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special) {
+ return false;
+ }
-void AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
- Common::MemoryHookPointer hook) {
- auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
- Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)};
- page_table.special_regions.add(
- std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
-}
+ return false;
+ }
-void RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
- Common::MemoryHookPointer hook) {
- auto interval = boost::icl::discrete_interval<VAddr>::closed(base, base + size - 1);
- Common::SpecialRegion region{Common::SpecialRegion::Type::DebugHook, std::move(hook)};
- page_table.special_regions.subtract(
- std::make_pair(interval, std::set<Common::SpecialRegion>{region}));
-}
+ bool IsValidVirtualAddress(VAddr vaddr) const {
+ return IsValidVirtualAddress(*system.CurrentProcess(), vaddr);
+ }
-/**
- * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
- * using a VMA from the current process
- */
-static u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) {
- const auto& vm_manager = process.VMManager();
-
- const auto it = vm_manager.FindVMA(vaddr);
- DEBUG_ASSERT(vm_manager.IsValidHandle(it));
-
- u8* direct_pointer = nullptr;
- const auto& vma = it->second;
- switch (vma.type) {
- case Kernel::VMAType::AllocatedMemoryBlock:
- direct_pointer = vma.backing_block->data() + vma.offset;
- break;
- case Kernel::VMAType::BackingMemory:
- direct_pointer = vma.backing_memory;
- break;
- case Kernel::VMAType::Free:
- return nullptr;
- default:
- UNREACHABLE();
+ /**
+ * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
+ * using a VMA from the current process
+ */
+ u8* GetPointerFromVMA(const Kernel::Process& process, VAddr vaddr) {
+ const auto& vm_manager = process.VMManager();
+
+ const auto it = vm_manager.FindVMA(vaddr);
+ DEBUG_ASSERT(vm_manager.IsValidHandle(it));
+
+ u8* direct_pointer = nullptr;
+ const auto& vma = it->second;
+ switch (vma.type) {
+ case Kernel::VMAType::AllocatedMemoryBlock:
+ direct_pointer = vma.backing_block->data() + vma.offset;
+ break;
+ case Kernel::VMAType::BackingMemory:
+ direct_pointer = vma.backing_memory;
+ break;
+ case Kernel::VMAType::Free:
+ return nullptr;
+ default:
+ UNREACHABLE();
+ }
+
+ return direct_pointer + (vaddr - vma.base);
}
- return direct_pointer + (vaddr - vma.base);
-}
+ /**
+ * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
+ * using a VMA from the current process.
+ */
+ u8* GetPointerFromVMA(VAddr vaddr) {
+ return GetPointerFromVMA(*system.CurrentProcess(), vaddr);
+ }
-/**
- * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned)
- * using a VMA from the current process.
- */
-static u8* GetPointerFromVMA(VAddr vaddr) {
- return GetPointerFromVMA(*Core::System::GetInstance().CurrentProcess(), vaddr);
-}
+ u8* GetPointer(const VAddr vaddr) {
+ u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
+ if (page_pointer != nullptr) {
+ return page_pointer + (vaddr & PAGE_MASK);
+ }
-template <typename T>
-T Read(const VAddr vaddr) {
- const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
- if (page_pointer) {
- // NOTE: Avoid adding any extra logic to this fast-path block
- T value;
- std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T));
- return value;
- }
-
- Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
- switch (type) {
- case Common::PageType::Unmapped:
- LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
- return 0;
- case Common::PageType::Memory:
- ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
- break;
- case Common::PageType::RasterizerCachedMemory: {
- auto host_ptr{GetPointerFromVMA(vaddr)};
- Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), sizeof(T));
- T value;
- std::memcpy(&value, host_ptr, sizeof(T));
- return value;
- }
- default:
- UNREACHABLE();
- }
- return {};
-}
+ if (current_page_table->attributes[vaddr >> PAGE_BITS] ==
+ Common::PageType::RasterizerCachedMemory) {
+ return GetPointerFromVMA(vaddr);
+ }
-template <typename T>
-void Write(const VAddr vaddr, const T data) {
- u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
- if (page_pointer) {
- // NOTE: Avoid adding any extra logic to this fast-path block
- std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T));
- return;
- }
-
- Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
- switch (type) {
- case Common::PageType::Unmapped:
- LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
- static_cast<u32>(data), vaddr);
- return;
- case Common::PageType::Memory:
- ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
- break;
- case Common::PageType::RasterizerCachedMemory: {
- auto host_ptr{GetPointerFromVMA(vaddr)};
- Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), sizeof(T));
- std::memcpy(host_ptr, &data, sizeof(T));
- break;
- }
- default:
- UNREACHABLE();
+ LOG_ERROR(HW_Memory, "Unknown GetPointer @ 0x{:016X}", vaddr);
+ return nullptr;
}
-}
-bool IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) {
- const auto& page_table = process.VMManager().page_table;
+ u8 Read8(const VAddr addr) {
+ return Read<u8>(addr);
+ }
- const u8* page_pointer = page_table.pointers[vaddr >> PAGE_BITS];
- if (page_pointer)
- return true;
+ u16 Read16(const VAddr addr) {
+ return Read<u16_le>(addr);
+ }
- if (page_table.attributes[vaddr >> PAGE_BITS] == Common::PageType::RasterizerCachedMemory)
- return true;
+ u32 Read32(const VAddr addr) {
+ return Read<u32_le>(addr);
+ }
- if (page_table.attributes[vaddr >> PAGE_BITS] != Common::PageType::Special)
- return false;
+ u64 Read64(const VAddr addr) {
+ return Read<u64_le>(addr);
+ }
- return false;
-}
+ void Write8(const VAddr addr, const u8 data) {
+ Write<u8>(addr, data);
+ }
-bool IsValidVirtualAddress(const VAddr vaddr) {
- return IsValidVirtualAddress(*Core::System::GetInstance().CurrentProcess(), vaddr);
-}
+ void Write16(const VAddr addr, const u16 data) {
+ Write<u16_le>(addr, data);
+ }
-bool IsKernelVirtualAddress(const VAddr vaddr) {
- return KERNEL_REGION_VADDR <= vaddr && vaddr < KERNEL_REGION_END;
-}
+ void Write32(const VAddr addr, const u32 data) {
+ Write<u32_le>(addr, data);
+ }
-u8* GetPointer(const VAddr vaddr) {
- u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
- if (page_pointer) {
- return page_pointer + (vaddr & PAGE_MASK);
+ void Write64(const VAddr addr, const u64 data) {
+ Write<u64_le>(addr, data);
}
- if (current_page_table->attributes[vaddr >> PAGE_BITS] ==
- Common::PageType::RasterizerCachedMemory) {
- return GetPointerFromVMA(vaddr);
+ std::string ReadCString(VAddr vaddr, std::size_t max_length) {
+ std::string string;
+ string.reserve(max_length);
+ for (std::size_t i = 0; i < max_length; ++i) {
+ const char c = Read8(vaddr);
+ if (c == '\0') {
+ break;
+ }
+ string.push_back(c);
+ ++vaddr;
+ }
+ string.shrink_to_fit();
+ return string;
}
- LOG_ERROR(HW_Memory, "Unknown GetPointer @ 0x{:016X}", vaddr);
- return nullptr;
-}
+ void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer,
+ const std::size_t size) {
+ const auto& page_table = process.VMManager().page_table;
+
+ std::size_t remaining_size = size;
+ std::size_t page_index = src_addr >> PAGE_BITS;
+ std::size_t page_offset = src_addr & PAGE_MASK;
+
+ while (remaining_size > 0) {
+ const std::size_t copy_amount =
+ std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
+ const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
+
+ switch (page_table.attributes[page_index]) {
+ case Common::PageType::Unmapped: {
+ LOG_ERROR(HW_Memory,
+ "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
+ current_vaddr, src_addr, size);
+ std::memset(dest_buffer, 0, copy_amount);
+ break;
+ }
+ case Common::PageType::Memory: {
+ DEBUG_ASSERT(page_table.pointers[page_index]);
-std::string ReadCString(VAddr vaddr, std::size_t max_length) {
- std::string string;
- string.reserve(max_length);
- for (std::size_t i = 0; i < max_length; ++i) {
- char c = Read8(vaddr);
- if (c == '\0')
- break;
- string.push_back(c);
- ++vaddr;
+ const u8* const src_ptr = page_table.pointers[page_index] + page_offset;
+ std::memcpy(dest_buffer, src_ptr, copy_amount);
+ break;
+ }
+ case Common::PageType::RasterizerCachedMemory: {
+ const u8* const host_ptr = GetPointerFromVMA(process, current_vaddr);
+ system.GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount);
+ std::memcpy(dest_buffer, host_ptr, copy_amount);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+
+ page_index++;
+ page_offset = 0;
+ dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
+ remaining_size -= copy_amount;
+ }
}
- string.shrink_to_fit();
- return string;
-}
-void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
- if (vaddr == 0) {
- return;
+ void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
+ ReadBlock(*system.CurrentProcess(), src_addr, dest_buffer, size);
}
- // Iterate over a contiguous CPU address space, which corresponds to the specified GPU address
- // space, marking the region as un/cached. The region is marked un/cached at a granularity of
- // CPU pages, hence why we iterate on a CPU page basis (note: GPU page size is different). This
- // assumes the specified GPU address region is contiguous as well.
+ void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer,
+ const std::size_t size) {
+ const auto& page_table = process.VMManager().page_table;
+ std::size_t remaining_size = size;
+ std::size_t page_index = dest_addr >> PAGE_BITS;
+ std::size_t page_offset = dest_addr & PAGE_MASK;
+
+ while (remaining_size > 0) {
+ const std::size_t copy_amount =
+ std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
+ const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
+
+ switch (page_table.attributes[page_index]) {
+ case Common::PageType::Unmapped: {
+ LOG_ERROR(HW_Memory,
+ "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
+ current_vaddr, dest_addr, size);
+ break;
+ }
+ case Common::PageType::Memory: {
+ DEBUG_ASSERT(page_table.pointers[page_index]);
+
+ u8* const dest_ptr = page_table.pointers[page_index] + page_offset;
+ std::memcpy(dest_ptr, src_buffer, copy_amount);
+ break;
+ }
+ case Common::PageType::RasterizerCachedMemory: {
+ u8* const host_ptr = GetPointerFromVMA(process, current_vaddr);
+ system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount);
+ std::memcpy(host_ptr, src_buffer, copy_amount);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+
+ page_index++;
+ page_offset = 0;
+ src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
+ remaining_size -= copy_amount;
+ }
+ }
- u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1;
- for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) {
- Common::PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS];
+ void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
+ WriteBlock(*system.CurrentProcess(), dest_addr, src_buffer, size);
+ }
- if (cached) {
- // Switch page type to cached if now cached
- switch (page_type) {
- case Common::PageType::Unmapped:
- // It is not necessary for a process to have this region mapped into its address
- // space, for example, a system module need not have a VRAM mapping.
+ void ZeroBlock(const Kernel::Process& process, const VAddr dest_addr, const std::size_t size) {
+ const auto& page_table = process.VMManager().page_table;
+ std::size_t remaining_size = size;
+ std::size_t page_index = dest_addr >> PAGE_BITS;
+ std::size_t page_offset = dest_addr & PAGE_MASK;
+
+ while (remaining_size > 0) {
+ const std::size_t copy_amount =
+ std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
+ const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
+
+ switch (page_table.attributes[page_index]) {
+ case Common::PageType::Unmapped: {
+ LOG_ERROR(HW_Memory,
+ "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
+ current_vaddr, dest_addr, size);
break;
- case Common::PageType::Memory:
- page_type = Common::PageType::RasterizerCachedMemory;
- current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr;
+ }
+ case Common::PageType::Memory: {
+ DEBUG_ASSERT(page_table.pointers[page_index]);
+
+ u8* dest_ptr = page_table.pointers[page_index] + page_offset;
+ std::memset(dest_ptr, 0, copy_amount);
break;
- case Common::PageType::RasterizerCachedMemory:
- // There can be more than one GPU region mapped per CPU region, so it's common that
- // this area is already marked as cached.
+ }
+ case Common::PageType::RasterizerCachedMemory: {
+ u8* const host_ptr = GetPointerFromVMA(process, current_vaddr);
+ system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount);
+ std::memset(host_ptr, 0, copy_amount);
break;
+ }
default:
UNREACHABLE();
}
- } else {
- // Switch page type to uncached if now uncached
- switch (page_type) {
- case Common::PageType::Unmapped:
- // It is not necessary for a process to have this region mapped into its address
- // space, for example, a system module need not have a VRAM mapping.
+
+ page_index++;
+ page_offset = 0;
+ remaining_size -= copy_amount;
+ }
+ }
+
+ void ZeroBlock(const VAddr dest_addr, const std::size_t size) {
+ ZeroBlock(*system.CurrentProcess(), dest_addr, size);
+ }
+
+ void CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr,
+ const std::size_t size) {
+ const auto& page_table = process.VMManager().page_table;
+ std::size_t remaining_size = size;
+ std::size_t page_index = src_addr >> PAGE_BITS;
+ std::size_t page_offset = src_addr & PAGE_MASK;
+
+ while (remaining_size > 0) {
+ const std::size_t copy_amount =
+ std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
+ const auto current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
+
+ switch (page_table.attributes[page_index]) {
+ case Common::PageType::Unmapped: {
+ LOG_ERROR(HW_Memory,
+ "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
+ current_vaddr, src_addr, size);
+ ZeroBlock(process, dest_addr, copy_amount);
break;
- case Common::PageType::Memory:
- // There can be more than one GPU region mapped per CPU region, so it's common that
- // this area is already unmarked as cached.
+ }
+ case Common::PageType::Memory: {
+ DEBUG_ASSERT(page_table.pointers[page_index]);
+ const u8* src_ptr = page_table.pointers[page_index] + page_offset;
+ WriteBlock(process, dest_addr, src_ptr, copy_amount);
break;
+ }
case Common::PageType::RasterizerCachedMemory: {
- u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK);
- if (pointer == nullptr) {
- // It's possible that this function has been called while updating the pagetable
- // after unmapping a VMA. In that case the underlying VMA will no longer exist,
- // and we should just leave the pagetable entry blank.
- page_type = Common::PageType::Unmapped;
- } else {
- page_type = Common::PageType::Memory;
- current_page_table->pointers[vaddr >> PAGE_BITS] = pointer;
- }
+ const u8* const host_ptr = GetPointerFromVMA(process, current_vaddr);
+ system.GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount);
+ WriteBlock(process, dest_addr, host_ptr, copy_amount);
break;
}
default:
UNREACHABLE();
}
+
+ page_index++;
+ page_offset = 0;
+ dest_addr += static_cast<VAddr>(copy_amount);
+ src_addr += static_cast<VAddr>(copy_amount);
+ remaining_size -= copy_amount;
}
}
-}
-u8 Read8(const VAddr addr) {
- return Read<u8>(addr);
-}
+ void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) {
+ return CopyBlock(*system.CurrentProcess(), dest_addr, src_addr, size);
+ }
-u16 Read16(const VAddr addr) {
- return Read<u16_le>(addr);
-}
+ void RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
+ if (vaddr == 0) {
+ return;
+ }
-u32 Read32(const VAddr addr) {
- return Read<u32_le>(addr);
-}
+ // Iterate over a contiguous CPU address space, which corresponds to the specified GPU
+ // address space, marking the region as un/cached. The region is marked un/cached at a
+ // granularity of CPU pages, hence why we iterate on a CPU page basis (note: GPU page size
+ // is different). This assumes the specified GPU address region is contiguous as well.
+
+ u64 num_pages = ((vaddr + size - 1) >> PAGE_BITS) - (vaddr >> PAGE_BITS) + 1;
+ for (unsigned i = 0; i < num_pages; ++i, vaddr += PAGE_SIZE) {
+ Common::PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS];
+
+ if (cached) {
+ // Switch page type to cached if now cached
+ switch (page_type) {
+ case Common::PageType::Unmapped:
+ // It is not necessary for a process to have this region mapped into its address
+ // space, for example, a system module need not have a VRAM mapping.
+ break;
+ case Common::PageType::Memory:
+ page_type = Common::PageType::RasterizerCachedMemory;
+ current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr;
+ break;
+ case Common::PageType::RasterizerCachedMemory:
+ // There can be more than one GPU region mapped per CPU region, so it's common
+ // that this area is already marked as cached.
+ break;
+ default:
+ UNREACHABLE();
+ }
+ } else {
+ // Switch page type to uncached if now uncached
+ switch (page_type) {
+ case Common::PageType::Unmapped:
+ // It is not necessary for a process to have this region mapped into its address
+ // space, for example, a system module need not have a VRAM mapping.
+ break;
+ case Common::PageType::Memory:
+ // There can be more than one GPU region mapped per CPU region, so it's common
+ // that this area is already unmarked as cached.
+ break;
+ case Common::PageType::RasterizerCachedMemory: {
+ u8* pointer = GetPointerFromVMA(vaddr & ~PAGE_MASK);
+ if (pointer == nullptr) {
+ // It's possible that this function has been called while updating the
+ // pagetable after unmapping a VMA. In that case the underlying VMA will no
+ // longer exist, and we should just leave the pagetable entry blank.
+ page_type = Common::PageType::Unmapped;
+ } else {
+ page_type = Common::PageType::Memory;
+ current_page_table->pointers[vaddr >> PAGE_BITS] = pointer;
+ }
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+ }
+ }
+ }
-u64 Read64(const VAddr addr) {
- return Read<u64_le>(addr);
-}
+ /**
+ * Maps a region of pages as a specific type.
+ *
+ * @param page_table The page table to use to perform the mapping.
+ * @param base The base address to begin mapping at.
+ * @param size The total size of the range in bytes.
+ * @param memory The memory to map.
+ * @param type The page type to map the memory as.
+ */
+ void MapPages(Common::PageTable& page_table, VAddr base, u64 size, u8* memory,
+ Common::PageType type) {
+ LOG_DEBUG(HW_Memory, "Mapping {} onto {:016X}-{:016X}", fmt::ptr(memory), base * PAGE_SIZE,
+ (base + size) * PAGE_SIZE);
+
+ // During boot, current_page_table might not be set yet, in which case we need not flush
+ if (system.IsPoweredOn()) {
+ auto& gpu = system.GPU();
+ for (u64 i = 0; i < size; i++) {
+ const auto page = base + i;
+ if (page_table.attributes[page] == Common::PageType::RasterizerCachedMemory) {
+ gpu.FlushAndInvalidateRegion(page << PAGE_BITS, PAGE_SIZE);
+ }
+ }
+ }
-void ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer,
- const std::size_t size) {
- const auto& page_table = process.VMManager().page_table;
-
- std::size_t remaining_size = size;
- std::size_t page_index = src_addr >> PAGE_BITS;
- std::size_t page_offset = src_addr & PAGE_MASK;
-
- while (remaining_size > 0) {
- const std::size_t copy_amount =
- std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
- const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
-
- switch (page_table.attributes[page_index]) {
- case Common::PageType::Unmapped: {
- LOG_ERROR(HW_Memory,
- "Unmapped ReadBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
- current_vaddr, src_addr, size);
- std::memset(dest_buffer, 0, copy_amount);
- break;
+ const VAddr end = base + size;
+ ASSERT_MSG(end <= page_table.pointers.size(), "out of range mapping at {:016X}",
+ base + page_table.pointers.size());
+
+ std::fill(page_table.attributes.begin() + base, page_table.attributes.begin() + end, type);
+
+ if (memory == nullptr) {
+ std::fill(page_table.pointers.begin() + base, page_table.pointers.begin() + end,
+ memory);
+ } else {
+ while (base != end) {
+ page_table.pointers[base] = memory;
+
+ base += 1;
+ memory += PAGE_SIZE;
+ }
+ }
+ }
+
+ /**
+ * Reads a particular data type out of memory at the given virtual address.
+ *
+ * @param vaddr The virtual address to read the data type from.
+ *
+ * @tparam T The data type to read out of memory. This type *must* be
+ * trivially copyable, otherwise the behavior of this function
+ * is undefined.
+ *
+ * @returns The instance of T read from the specified virtual address.
+ */
+ template <typename T>
+ T Read(const VAddr vaddr) {
+ const u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
+ if (page_pointer != nullptr) {
+ // NOTE: Avoid adding any extra logic to this fast-path block
+ T value;
+ std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T));
+ return value;
}
- case Common::PageType::Memory: {
- DEBUG_ASSERT(page_table.pointers[page_index]);
- const u8* src_ptr = page_table.pointers[page_index] + page_offset;
- std::memcpy(dest_buffer, src_ptr, copy_amount);
+ const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
+ switch (type) {
+ case Common::PageType::Unmapped:
+ LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:08X}", sizeof(T) * 8, vaddr);
+ return 0;
+ case Common::PageType::Memory:
+ ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
break;
+ case Common::PageType::RasterizerCachedMemory: {
+ const u8* const host_ptr = GetPointerFromVMA(vaddr);
+ system.GPU().FlushRegion(ToCacheAddr(host_ptr), sizeof(T));
+ T value;
+ std::memcpy(&value, host_ptr, sizeof(T));
+ return value;
+ }
+ default:
+ UNREACHABLE();
}
+ return {};
+ }
+
+ /**
+ * Writes a particular data type to memory at the given virtual address.
+ *
+ * @param vaddr The virtual address to write the data type to.
+ *
+ * @tparam T The data type to write to memory. This type *must* be
+ * trivially copyable, otherwise the behavior of this function
+ * is undefined.
+ *
+ * @returns The instance of T write to the specified virtual address.
+ */
+ template <typename T>
+ void Write(const VAddr vaddr, const T data) {
+ u8* const page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS];
+ if (page_pointer != nullptr) {
+ // NOTE: Avoid adding any extra logic to this fast-path block
+ std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T));
+ return;
+ }
+
+ const Common::PageType type = current_page_table->attributes[vaddr >> PAGE_BITS];
+ switch (type) {
+ case Common::PageType::Unmapped:
+ LOG_ERROR(HW_Memory, "Unmapped Write{} 0x{:08X} @ 0x{:016X}", sizeof(data) * 8,
+ static_cast<u32>(data), vaddr);
+ return;
+ case Common::PageType::Memory:
+ ASSERT_MSG(false, "Mapped memory page without a pointer @ {:016X}", vaddr);
+ break;
case Common::PageType::RasterizerCachedMemory: {
- const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)};
- Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount);
- std::memcpy(dest_buffer, host_ptr, copy_amount);
+ u8* const host_ptr{GetPointerFromVMA(vaddr)};
+ system.GPU().InvalidateRegion(ToCacheAddr(host_ptr), sizeof(T));
+ std::memcpy(host_ptr, &data, sizeof(T));
break;
}
default:
UNREACHABLE();
}
-
- page_index++;
- page_offset = 0;
- dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount;
- remaining_size -= copy_amount;
}
+
+ Common::PageTable* current_page_table = nullptr;
+ Core::System& system;
+};
+
+Memory::Memory(Core::System& system) : impl{std::make_unique<Impl>(system)} {}
+Memory::~Memory() = default;
+
+void Memory::SetCurrentPageTable(Kernel::Process& process) {
+ impl->SetCurrentPageTable(process);
}
-void ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
- ReadBlock(*Core::System::GetInstance().CurrentProcess(), src_addr, dest_buffer, size);
+void Memory::MapMemoryRegion(Common::PageTable& page_table, VAddr base, u64 size, u8* target) {
+ impl->MapMemoryRegion(page_table, base, size, target);
}
-void Write8(const VAddr addr, const u8 data) {
- Write<u8>(addr, data);
+void Memory::MapIoRegion(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer mmio_handler) {
+ impl->MapIoRegion(page_table, base, size, std::move(mmio_handler));
}
-void Write16(const VAddr addr, const u16 data) {
- Write<u16_le>(addr, data);
+void Memory::UnmapRegion(Common::PageTable& page_table, VAddr base, u64 size) {
+ impl->UnmapRegion(page_table, base, size);
}
-void Write32(const VAddr addr, const u32 data) {
- Write<u32_le>(addr, data);
+void Memory::AddDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer hook) {
+ impl->AddDebugHook(page_table, base, size, std::move(hook));
}
-void Write64(const VAddr addr, const u64 data) {
- Write<u64_le>(addr, data);
+void Memory::RemoveDebugHook(Common::PageTable& page_table, VAddr base, u64 size,
+ Common::MemoryHookPointer hook) {
+ impl->RemoveDebugHook(page_table, base, size, std::move(hook));
}
-void WriteBlock(const Kernel::Process& process, const VAddr dest_addr, const void* src_buffer,
- const std::size_t size) {
- const auto& page_table = process.VMManager().page_table;
- std::size_t remaining_size = size;
- std::size_t page_index = dest_addr >> PAGE_BITS;
- std::size_t page_offset = dest_addr & PAGE_MASK;
-
- while (remaining_size > 0) {
- const std::size_t copy_amount =
- std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
- const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
-
- switch (page_table.attributes[page_index]) {
- case Common::PageType::Unmapped: {
- LOG_ERROR(HW_Memory,
- "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
- current_vaddr, dest_addr, size);
- break;
- }
- case Common::PageType::Memory: {
- DEBUG_ASSERT(page_table.pointers[page_index]);
+bool Memory::IsValidVirtualAddress(const Kernel::Process& process, const VAddr vaddr) const {
+ return impl->IsValidVirtualAddress(process, vaddr);
+}
- u8* dest_ptr = page_table.pointers[page_index] + page_offset;
- std::memcpy(dest_ptr, src_buffer, copy_amount);
- break;
- }
- case Common::PageType::RasterizerCachedMemory: {
- const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)};
- Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount);
- std::memcpy(host_ptr, src_buffer, copy_amount);
- break;
- }
- default:
- UNREACHABLE();
- }
+bool Memory::IsValidVirtualAddress(const VAddr vaddr) const {
+ return impl->IsValidVirtualAddress(vaddr);
+}
- page_index++;
- page_offset = 0;
- src_buffer = static_cast<const u8*>(src_buffer) + copy_amount;
- remaining_size -= copy_amount;
- }
+u8* Memory::GetPointer(VAddr vaddr) {
+ return impl->GetPointer(vaddr);
}
-void WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
- WriteBlock(*Core::System::GetInstance().CurrentProcess(), dest_addr, src_buffer, size);
+const u8* Memory::GetPointer(VAddr vaddr) const {
+ return impl->GetPointer(vaddr);
}
-void ZeroBlock(const Kernel::Process& process, const VAddr dest_addr, const std::size_t size) {
- const auto& page_table = process.VMManager().page_table;
- std::size_t remaining_size = size;
- std::size_t page_index = dest_addr >> PAGE_BITS;
- std::size_t page_offset = dest_addr & PAGE_MASK;
-
- while (remaining_size > 0) {
- const std::size_t copy_amount =
- std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
- const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
-
- switch (page_table.attributes[page_index]) {
- case Common::PageType::Unmapped: {
- LOG_ERROR(HW_Memory,
- "Unmapped ZeroBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
- current_vaddr, dest_addr, size);
- break;
- }
- case Common::PageType::Memory: {
- DEBUG_ASSERT(page_table.pointers[page_index]);
+u8 Memory::Read8(const VAddr addr) {
+ return impl->Read8(addr);
+}
- u8* dest_ptr = page_table.pointers[page_index] + page_offset;
- std::memset(dest_ptr, 0, copy_amount);
- break;
- }
- case Common::PageType::RasterizerCachedMemory: {
- const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)};
- Core::System::GetInstance().GPU().InvalidateRegion(ToCacheAddr(host_ptr), copy_amount);
- std::memset(host_ptr, 0, copy_amount);
- break;
- }
- default:
- UNREACHABLE();
- }
+u16 Memory::Read16(const VAddr addr) {
+ return impl->Read16(addr);
+}
- page_index++;
- page_offset = 0;
- remaining_size -= copy_amount;
- }
+u32 Memory::Read32(const VAddr addr) {
+ return impl->Read32(addr);
}
-void CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr,
- const std::size_t size) {
- const auto& page_table = process.VMManager().page_table;
- std::size_t remaining_size = size;
- std::size_t page_index = src_addr >> PAGE_BITS;
- std::size_t page_offset = src_addr & PAGE_MASK;
-
- while (remaining_size > 0) {
- const std::size_t copy_amount =
- std::min(static_cast<std::size_t>(PAGE_SIZE) - page_offset, remaining_size);
- const VAddr current_vaddr = static_cast<VAddr>((page_index << PAGE_BITS) + page_offset);
-
- switch (page_table.attributes[page_index]) {
- case Common::PageType::Unmapped: {
- LOG_ERROR(HW_Memory,
- "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})",
- current_vaddr, src_addr, size);
- ZeroBlock(process, dest_addr, copy_amount);
- break;
- }
- case Common::PageType::Memory: {
- DEBUG_ASSERT(page_table.pointers[page_index]);
- const u8* src_ptr = page_table.pointers[page_index] + page_offset;
- WriteBlock(process, dest_addr, src_ptr, copy_amount);
- break;
- }
- case Common::PageType::RasterizerCachedMemory: {
- const auto& host_ptr{GetPointerFromVMA(process, current_vaddr)};
- Core::System::GetInstance().GPU().FlushRegion(ToCacheAddr(host_ptr), copy_amount);
- WriteBlock(process, dest_addr, host_ptr, copy_amount);
- break;
- }
- default:
- UNREACHABLE();
- }
+u64 Memory::Read64(const VAddr addr) {
+ return impl->Read64(addr);
+}
- page_index++;
- page_offset = 0;
- dest_addr += static_cast<VAddr>(copy_amount);
- src_addr += static_cast<VAddr>(copy_amount);
- remaining_size -= copy_amount;
- }
+void Memory::Write8(VAddr addr, u8 data) {
+ impl->Write8(addr, data);
+}
+
+void Memory::Write16(VAddr addr, u16 data) {
+ impl->Write16(addr, data);
+}
+
+void Memory::Write32(VAddr addr, u32 data) {
+ impl->Write32(addr, data);
+}
+
+void Memory::Write64(VAddr addr, u64 data) {
+ impl->Write64(addr, data);
+}
+
+std::string Memory::ReadCString(VAddr vaddr, std::size_t max_length) {
+ return impl->ReadCString(vaddr, max_length);
+}
+
+void Memory::ReadBlock(const Kernel::Process& process, const VAddr src_addr, void* dest_buffer,
+ const std::size_t size) {
+ impl->ReadBlock(process, src_addr, dest_buffer, size);
+}
+
+void Memory::ReadBlock(const VAddr src_addr, void* dest_buffer, const std::size_t size) {
+ impl->ReadBlock(src_addr, dest_buffer, size);
+}
+
+void Memory::WriteBlock(const Kernel::Process& process, VAddr dest_addr, const void* src_buffer,
+ std::size_t size) {
+ impl->WriteBlock(process, dest_addr, src_buffer, size);
+}
+
+void Memory::WriteBlock(const VAddr dest_addr, const void* src_buffer, const std::size_t size) {
+ impl->WriteBlock(dest_addr, src_buffer, size);
}
-void CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) {
- CopyBlock(*Core::System::GetInstance().CurrentProcess(), dest_addr, src_addr, size);
+void Memory::ZeroBlock(const Kernel::Process& process, VAddr dest_addr, std::size_t size) {
+ impl->ZeroBlock(process, dest_addr, size);
+}
+
+void Memory::ZeroBlock(VAddr dest_addr, std::size_t size) {
+ impl->ZeroBlock(dest_addr, size);
+}
+
+void Memory::CopyBlock(const Kernel::Process& process, VAddr dest_addr, VAddr src_addr,
+ const std::size_t size) {
+ impl->CopyBlock(process, dest_addr, src_addr, size);
+}
+
+void Memory::CopyBlock(VAddr dest_addr, VAddr src_addr, std::size_t size) {
+ impl->CopyBlock(dest_addr, src_addr, size);
+}
+
+void Memory::RasterizerMarkRegionCached(VAddr vaddr, u64 size, bool cached) {
+ impl->RasterizerMarkRegionCached(vaddr, size, cached);
+}
+
+bool IsKernelVirtualAddress(const VAddr vaddr) {
+ return KERNEL_REGION_VADDR <= vaddr && vaddr < KERNEL_REGION_END;
}
} // namespace Memory