From a7837a3791562899bf5e0e98aef851a2f4aaf376 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sat, 5 Jun 2021 06:23:25 -0300 Subject: common/host_memory: Add interface and Windows implementation --- src/common/CMakeLists.txt | 2 + src/common/host_memory.cpp | 320 +++++++++++++++++++++++++++++++++++++++++++++ src/common/host_memory.h | 62 +++++++++ 3 files changed, 384 insertions(+) create mode 100644 src/common/host_memory.cpp create mode 100644 src/common/host_memory.h (limited to 'src/common') diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 2d403d471..97fbdcbf9 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -131,6 +131,8 @@ add_library(common STATIC hash.h hex_util.cpp hex_util.h + host_memory.cpp + host_memory.h intrusive_red_black_tree.h logging/backend.cpp logging/backend.h diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp new file mode 100644 index 000000000..4f5086e90 --- /dev/null +++ b/src/common/host_memory.cpp @@ -0,0 +1,320 @@ +#ifdef __linux__ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#elif defined(_WIN32) // ^^^ Linux ^^^ vvv Windows vvv +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0A00 // Windows 10 + +#include + +#include + +#include +#include + +#pragma comment(lib, "mincore.lib") + +#endif // ^^^ Windows ^^^ + +#include + +#include "common/assert.h" +#include "common/host_memory.h" +#include "common/logging/log.h" + +namespace Common { + +constexpr size_t PageAlignment = 0x1000; + +#ifdef _WIN32 + +class HostMemory::Impl { +public: + explicit Impl(size_t backing_size_, size_t virtual_size_) + : backing_size{backing_size_}, virtual_size{virtual_size_}, process{GetCurrentProcess()} { + // Allocate backing file map + backing_handle = + CreateFileMapping2(INVALID_HANDLE_VALUE, nullptr, FILE_MAP_WRITE | FILE_MAP_READ, + PAGE_READWRITE, SEC_COMMIT, backing_size, nullptr, nullptr, 0); + if (!backing_handle) { + throw std::bad_alloc{}; + } + // Allocate a virtual memory for the backing file map as placeholder + backing_base = static_cast(VirtualAlloc2(process, nullptr, backing_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, + PAGE_NOACCESS, nullptr, 0)); + if (!backing_base) { + Release(); + throw std::bad_alloc{}; + } + // Map backing placeholder + void* const ret = MapViewOfFile3(backing_handle, process, backing_base, 0, backing_size, + MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0); + if (ret != backing_base) { + Release(); + throw std::bad_alloc{}; + } + // Allocate virtual address placeholder + virtual_base = static_cast(VirtualAlloc2(process, nullptr, virtual_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, + PAGE_NOACCESS, nullptr, 0)); + if (!virtual_base) { + Release(); + throw std::bad_alloc{}; + } + } + + ~Impl() { + Release(); + } + + void Map(size_t virtual_offset, size_t host_offset, size_t length) { + std::unique_lock lock{placeholder_mutex}; + if (!IsNiechePlaceholder(virtual_offset, length)) { + Split(virtual_offset, length); + } + ASSERT(placeholders.find({virtual_offset, virtual_offset + length}) == placeholders.end()); + TrackPlaceholder(virtual_offset, host_offset, length); + + MapView(virtual_offset, host_offset, length); + } + + void Unmap(size_t virtual_offset, size_t length) { + std::lock_guard lock{placeholder_mutex}; + + // Unmap until there are no more placeholders + while (UnmapOnePlaceholder(virtual_offset, length)) { + } + } + + void Protect(size_t virtual_offset, size_t length, bool read, bool write) { + DWORD new_flags{}; + if (read && write) { + new_flags = PAGE_READWRITE; + } else if (read && !write) { + new_flags = PAGE_READONLY; + } else if (!read && !write) { + new_flags = PAGE_NOACCESS; + } else { + UNIMPLEMENTED_MSG("Protection flag combination read={} write={}", read, write); + } + DWORD old_flags{}; + if (!VirtualProtect(virtual_base + virtual_offset, length, new_flags, &old_flags)) { + LOG_CRITICAL(HW_Memory, "Failed to change virtual memory protect rules"); + } + } + + const size_t backing_size; ///< Size of the backing memory in bytes + const size_t virtual_size; ///< Size of the virtual address placeholder in bytes + + u8* backing_base{}; + u8* virtual_base{}; + +private: + /// Release all resources in the object + void Release() { + if (!placeholders.empty()) { + for (const auto& placeholder : placeholders) { + if (!UnmapViewOfFile2(process, virtual_base + placeholder.lower(), + MEM_PRESERVE_PLACEHOLDER)) { + LOG_CRITICAL(HW_Memory, "Failed to unmap virtual memory placeholder"); + } + } + Coalesce(0, virtual_size); + } + if (virtual_base) { + if (!VirtualFree(virtual_base, 0, MEM_RELEASE)) { + LOG_CRITICAL(HW_Memory, "Failed to free virtual memory"); + } + } + if (backing_base) { + if (!UnmapViewOfFile2(process, backing_base, MEM_PRESERVE_PLACEHOLDER)) { + LOG_CRITICAL(HW_Memory, "Failed to unmap backing memory placeholder"); + } + if (!VirtualFreeEx(process, backing_base, 0, MEM_RELEASE)) { + LOG_CRITICAL(HW_Memory, "Failed to free backing memory"); + } + } + if (!CloseHandle(backing_handle)) { + LOG_CRITICAL(HW_Memory, "Failed to free backing memory file handle"); + } + } + + /// Unmap one placeholder in the given range (partial unmaps are supported) + /// Return true when there are no more placeholders to unmap + bool UnmapOnePlaceholder(size_t virtual_offset, size_t length) { + const auto it = placeholders.find({virtual_offset, virtual_offset + length}); + const auto begin = placeholders.begin(); + const auto end = placeholders.end(); + if (it == end) { + return false; + } + const size_t placeholder_begin = it->lower(); + const size_t placeholder_end = it->upper(); + const size_t unmap_begin = std::max(virtual_offset, placeholder_begin); + const size_t unmap_end = std::min(virtual_offset + length, placeholder_end); + ASSERT(unmap_begin >= placeholder_begin && unmap_begin < placeholder_end); + ASSERT(unmap_end <= placeholder_end && unmap_end > placeholder_begin); + + const auto host_pointer_it = placeholder_host_pointers.find(placeholder_begin); + ASSERT(host_pointer_it != placeholder_host_pointers.end()); + const size_t host_offset = host_pointer_it->second; + + const bool split_left = unmap_begin > placeholder_begin; + const bool split_right = unmap_end < placeholder_end; + + if (!UnmapViewOfFile2(process, virtual_base + placeholder_begin, + MEM_PRESERVE_PLACEHOLDER)) { + LOG_CRITICAL(HW_Memory, "Failed to unmap placeholder"); + } + // If we have to remap memory regions due to partial unmaps, we are in a data race as + // Windows doesn't support remapping memory without unmapping first. Avoid adding any extra + // logic within the panic region described below. + + // Panic region, we are in a data race right now + if (split_left || split_right) { + Split(unmap_begin, unmap_end - unmap_begin); + } + if (split_left) { + MapView(placeholder_begin, host_offset, unmap_begin - placeholder_begin); + } + if (split_right) { + MapView(unmap_end, host_offset + unmap_end - placeholder_begin, + placeholder_end - unmap_end); + } + // End panic region + + size_t coalesce_begin = unmap_begin; + if (!split_left) { + // Try to coalesce pages to the left + coalesce_begin = it == begin ? 0 : std::prev(it)->upper(); + if (coalesce_begin != placeholder_begin) { + Coalesce(coalesce_begin, unmap_end - coalesce_begin); + } + } + if (!split_right) { + // Try to coalesce pages to the right + const auto next = std::next(it); + const size_t next_begin = next == end ? virtual_size : next->lower(); + if (placeholder_end != next_begin) { + // We can coalesce to the right + Coalesce(coalesce_begin, next_begin - coalesce_begin); + } + } + // Remove and reinsert placeholder trackers + UntrackPlaceholder(it); + if (split_left) { + TrackPlaceholder(placeholder_begin, host_offset, unmap_begin - placeholder_begin); + } + if (split_right) { + TrackPlaceholder(unmap_end, host_offset + unmap_end - placeholder_begin, + placeholder_end - unmap_end); + } + return true; + } + + void MapView(size_t virtual_offset, size_t host_offset, size_t length) { + if (!MapViewOfFile3(backing_handle, process, virtual_base + virtual_offset, host_offset, + length, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0)) { + LOG_CRITICAL(HW_Memory, "Failed to map placeholder"); + } + } + + void Split(size_t virtual_offset, size_t length) { + if (!VirtualFreeEx(process, reinterpret_cast(virtual_base + virtual_offset), length, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { + LOG_CRITICAL(HW_Memory, "Failed to split placeholder"); + } + } + + void Coalesce(size_t virtual_offset, size_t length) { + if (!VirtualFreeEx(process, reinterpret_cast(virtual_base + virtual_offset), length, + MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS)) { + LOG_CRITICAL(HW_Memory, "Failed to coalesce placeholders"); + } + } + + void TrackPlaceholder(size_t virtual_offset, size_t host_offset, size_t length) { + placeholders.insert({virtual_offset, virtual_offset + length}); + placeholder_host_pointers.emplace(virtual_offset, host_offset); + } + + void UntrackPlaceholder(boost::icl::separate_interval_set::iterator it) { + placeholders.erase(it); + placeholder_host_pointers.erase(it->lower()); + } + + /// Return true when a given memory region is a "nieche" and the placeholders don't have to be + /// splitted. + bool IsNiechePlaceholder(size_t virtual_offset, size_t length) const { + const auto it = placeholders.upper_bound({virtual_offset, virtual_offset + length}); + if (it != placeholders.end() && it->lower() == virtual_offset + length) { + const bool is_root = it == placeholders.begin() && virtual_offset == 0; + return is_root || std::prev(it)->upper() == virtual_offset; + } + return false; + } + + HANDLE process{}; ///< Current process handle + HANDLE backing_handle{}; ///< File based backing memory + + std::mutex placeholder_mutex; ///< Mutex for placeholders + boost::icl::separate_interval_set placeholders; ///< Mapped placeholders + std::unordered_map placeholder_host_pointers; ///< Placeholder backing offset +}; + +#else + +#error Please implement the host memory for your platform + +#endif + +HostMemory::HostMemory(size_t backing_size, size_t virtual_size) + : impl{std::make_unique(backing_size, virtual_size)}, + backing_base{impl->backing_base}, virtual_base{impl->virtual_base} {} + +HostMemory::~HostMemory() = default; + +HostMemory::HostMemory(HostMemory&&) noexcept = default; + +HostMemory& HostMemory::operator=(HostMemory&&) noexcept = default; + +void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length) { + ASSERT(virtual_offset % PageAlignment == 0); + ASSERT(host_offset % PageAlignment == 0); + ASSERT(length % PageAlignment == 0); + ASSERT(virtual_offset + length <= impl->virtual_size); + ASSERT(host_offset + length <= impl->backing_size); + if (length == 0) { + return; + } + impl->Map(virtual_offset, host_offset, length); +} + +void HostMemory::Unmap(size_t virtual_offset, size_t length) { + ASSERT(virtual_offset % PageAlignment == 0); + ASSERT(length % PageAlignment == 0); + ASSERT(virtual_offset + length <= impl->virtual_size); + if (length == 0) { + return; + } + impl->Unmap(virtual_offset, length); +} + +void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool write) { + ASSERT(virtual_offset % PageAlignment == 0); + ASSERT(length % PageAlignment == 0); + ASSERT(virtual_offset + length <= impl->virtual_size); + if (length == 0) { + return; + } + impl->Protect(virtual_offset, length, read, write); +} + +} // namespace Common diff --git a/src/common/host_memory.h b/src/common/host_memory.h new file mode 100644 index 000000000..98005df7a --- /dev/null +++ b/src/common/host_memory.h @@ -0,0 +1,62 @@ +// Copyright 2019 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include "common/common_types.h" + +namespace Common { + +/** + * A low level linear memory buffer, which supports multiple mappings + * Its purpose is to rebuild a given sparse memory layout, including mirrors. + */ +class HostMemory { +public: + explicit HostMemory(size_t backing_size, size_t virtual_size); + ~HostMemory(); + + /** + * Copy constructors. They shall return a copy of the buffer without the mappings. + * TODO: Implement them with COW if needed. + */ + HostMemory(const HostMemory& other) = delete; + HostMemory& operator=(const HostMemory& other) = delete; + + /** + * Move constructors. They will move the buffer and the mappings to the new object. + */ + HostMemory(HostMemory&& other) noexcept; + HostMemory& operator=(HostMemory&& other) noexcept; + + void Map(size_t virtual_offset, size_t host_offset, size_t length); + + void Unmap(size_t virtual_offset, size_t length); + + void Protect(size_t virtual_offset, size_t length, bool read, bool write); + + [[nodiscard]] u8* BackingBasePointer() noexcept { + return backing_base; + } + [[nodiscard]] const u8* BackingBasePointer() const noexcept { + return backing_base; + } + + [[nodiscard]] u8* VirtualBasePointer() noexcept { + return virtual_base; + } + [[nodiscard]] const u8* VirtualBasePointer() const noexcept { + return virtual_base; + } + +private: + // Low level handler for the platform dependent memory routines + class Impl; + std::unique_ptr impl; + u8* backing_base{}; + u8* virtual_base{}; +}; + +} // namespace Common -- cgit v1.2.3 From 5105318bbc6843de14f3f949515007d9bf76aa7b Mon Sep 17 00:00:00 2001 From: Markus Wick Date: Sat, 5 Jun 2021 10:07:26 +0200 Subject: common/host_memory: Add Linux implementation --- src/common/host_memory.cpp | 130 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 10 deletions(-) (limited to 'src/common') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 4f5086e90..eb50fbd9f 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -1,11 +1,5 @@ -#ifdef __linux__ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include -#include -#elif defined(_WIN32) // ^^^ Linux ^^^ vvv Windows vvv +#ifdef _WIN32 + #ifdef _WIN32_WINNT #undef _WIN32_WINNT #endif @@ -20,13 +14,23 @@ #pragma comment(lib, "mincore.lib") -#endif // ^^^ Windows ^^^ +#elif defined(__linux__) // ^^^ Windows ^^^ vvv Linux vvv + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include + +#endif // ^^^ Linux ^^^ #include #include "common/assert.h" #include "common/host_memory.h" #include "common/logging/log.h" +#include "common/scope_exit.h" namespace Common { @@ -269,7 +273,113 @@ private: std::unordered_map placeholder_host_pointers; ///< Placeholder backing offset }; -#else +#elif defined(__linux__) // ^^^ Windows ^^^ vvv Linux vvv + +class HostMemory::Impl { +public: + explicit Impl(size_t backing_size_, size_t virtual_size_) + : backing_size{backing_size_}, virtual_size{virtual_size_} { + bool good = false; + SCOPE_EXIT({ + if (!good) { + Release(); + } + }); + + // Backing memory initialization + fd = memfd_create("HostMemory", 0); + if (fd == -1) { + LOG_CRITICAL(HW_Memory, "memfd_create failed: {}", strerror(errno)); + throw std::bad_alloc{}; + } + + // Defined to extend the file with zeros + int ret = ftruncate(fd, backing_size); + if (ret != 0) { + LOG_CRITICAL(HW_Memory, "ftruncate failed with {}, are you out-of-memory?", + strerror(errno)); + throw std::bad_alloc{}; + } + + backing_base = static_cast( + mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); + if (backing_base == MAP_FAILED) { + LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); + throw std::bad_alloc{}; + } + + // Virtual memory initialization + virtual_base = static_cast( + mmap(nullptr, virtual_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (virtual_base == MAP_FAILED) { + LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); + throw std::bad_alloc{}; + } + + good = true; + } + + ~Impl() { + Release(); + } + + void Map(size_t virtual_offset, size_t host_offset, size_t length) { + + void* ret = mmap(virtual_base + virtual_offset, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, host_offset); + ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno)); + } + + void Unmap(size_t virtual_offset, size_t length) { + // The method name is wrong. We're still talking about the virtual range. + // We don't want to unmap, we want to reserve this memory. + + void* ret = mmap(virtual_base + virtual_offset, length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno)); + } + + void Protect(size_t virtual_offset, size_t length, bool read, bool write) { + int flags = 0; + if (read) { + flags |= PROT_READ; + } + if (write) { + flags |= PROT_WRITE; + } + int ret = mprotect(virtual_base + virtual_offset, length, flags); + ASSERT_MSG(ret == 0, "mprotect failed: {}", strerror(errno)); + } + + const size_t backing_size; ///< Size of the backing memory in bytes + const size_t virtual_size; ///< Size of the virtual address placeholder in bytes + + u8* backing_base{reinterpret_cast(MAP_FAILED)}; + u8* virtual_base{reinterpret_cast(MAP_FAILED)}; + +private: + /// Release all resources in the object + void Release() { + if (virtual_base != MAP_FAILED) { + int ret = munmap(virtual_base, virtual_size); + ASSERT_MSG(ret == 0, "munmap failed: {}", strerror(errno)); + } + + if (backing_base != MAP_FAILED) { + int ret = munmap(backing_base, backing_size); + ASSERT_MSG(ret == 0, "munmap failed: {}", strerror(errno)); + } + + if (fd != -1) { + int ret = close(fd); + ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno)); + } + } + + int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create +}; + +#else // ^^^ Linux ^^^ #error Please implement the host memory for your platform -- cgit v1.2.3 From 621f3f5f47bf9619148cc0ab7ed315e05abf79d7 Mon Sep 17 00:00:00 2001 From: Markus Wick Date: Sun, 19 Jan 2020 01:49:30 +0100 Subject: core: Make use of fastmem --- src/common/page_table.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/common') diff --git a/src/common/page_table.h b/src/common/page_table.h index e92b66b2b..8267e8b4d 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -111,6 +111,8 @@ struct PageTable { VirtualBuffer backing_addr; size_t current_address_space_width_in_bits; + + u8* fastmem_arena; }; } // namespace Common -- cgit v1.2.3 From c4609c92eea30558473f02082733c7e59c2d2013 Mon Sep 17 00:00:00 2001 From: Markus Wick Date: Sat, 5 Jun 2021 11:47:08 +0200 Subject: common/host_memory: Optimize for huge tables. In theory, if we have 2 MB continously mapped, this should save one layer of TLB. Let's make it at least more likely by aligning the memory. --- src/common/host_memory.cpp | 29 +++++++++++++++++++---------- src/common/host_memory.h | 6 +++++- 2 files changed, 24 insertions(+), 11 deletions(-) (limited to 'src/common') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index eb50fbd9f..8a328f916 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -27,6 +27,7 @@ #include +#include "common/alignment.h" #include "common/assert.h" #include "common/host_memory.h" #include "common/logging/log.h" @@ -35,6 +36,7 @@ namespace Common { constexpr size_t PageAlignment = 0x1000; +constexpr size_t HugePageSize = 0x200000; #ifdef _WIN32 @@ -385,9 +387,16 @@ private: #endif -HostMemory::HostMemory(size_t backing_size, size_t virtual_size) - : impl{std::make_unique(backing_size, virtual_size)}, - backing_base{impl->backing_base}, virtual_base{impl->virtual_base} {} +HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) + : backing_size(backing_size_), + virtual_size(virtual_size_), impl{std::make_unique( + AlignUp(backing_size, PageAlignment), + AlignUp(virtual_size, PageAlignment) + 3 * HugePageSize)}, + backing_base{impl->backing_base}, virtual_base{impl->virtual_base} { + virtual_base += 2 * HugePageSize - 1; + virtual_base -= reinterpret_cast(virtual_base) & (HugePageSize - 1); + virtual_base_offset = virtual_base - impl->virtual_base; +} HostMemory::~HostMemory() = default; @@ -399,32 +408,32 @@ void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(host_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); - ASSERT(virtual_offset + length <= impl->virtual_size); - ASSERT(host_offset + length <= impl->backing_size); + ASSERT(virtual_offset + length <= virtual_size); + ASSERT(host_offset + length <= backing_size); if (length == 0) { return; } - impl->Map(virtual_offset, host_offset, length); + impl->Map(virtual_offset + virtual_base_offset, host_offset, length); } void HostMemory::Unmap(size_t virtual_offset, size_t length) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); - ASSERT(virtual_offset + length <= impl->virtual_size); + ASSERT(virtual_offset + length <= virtual_size); if (length == 0) { return; } - impl->Unmap(virtual_offset, length); + impl->Unmap(virtual_offset + virtual_base_offset, length); } void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool write) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); - ASSERT(virtual_offset + length <= impl->virtual_size); + ASSERT(virtual_offset + length <= virtual_size); if (length == 0) { return; } - impl->Protect(virtual_offset, length, read, write); + impl->Protect(virtual_offset + virtual_base_offset, length, read, write); } } // namespace Common diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 98005df7a..eaa7d18ab 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -15,7 +15,7 @@ namespace Common { */ class HostMemory { public: - explicit HostMemory(size_t backing_size, size_t virtual_size); + explicit HostMemory(size_t backing_size_, size_t virtual_size_); ~HostMemory(); /** @@ -52,11 +52,15 @@ public: } private: + size_t backing_size{}; + size_t virtual_size{}; + // Low level handler for the platform dependent memory routines class Impl; std::unique_ptr impl; u8* backing_base{}; u8* virtual_base{}; + size_t virtual_base_offset{}; }; } // namespace Common -- cgit v1.2.3 From 5ba28325b262d44fcd7721aa00074955bd794015 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Sun, 6 Jun 2021 09:57:24 +0200 Subject: General: Add settings for fastmem and disabling adress space check. --- src/common/settings.cpp | 8 ++++++++ src/common/settings.h | 4 ++++ 2 files changed, 12 insertions(+) (limited to 'src/common') diff --git a/src/common/settings.cpp b/src/common/settings.cpp index bcb4e4be1..360e878d6 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -90,6 +90,13 @@ bool IsGPULevelHigh() { values.gpu_accuracy.GetValue() == GPUAccuracy::High; } +bool IsFastmemEnabled() { + if (values.cpu_accuracy.GetValue() == CPUAccuracy::DebugMode) { + return values.cpuopt_fastmem; + } + return true; +} + float Volume() { if (values.audio_muted) { return 0.0f; @@ -115,6 +122,7 @@ void RestoreGlobalState(bool is_powered_on) { values.cpuopt_unsafe_unfuse_fma.SetGlobal(true); values.cpuopt_unsafe_reduce_fp_error.SetGlobal(true); values.cpuopt_unsafe_inaccurate_nan.SetGlobal(true); + values.cpuopt_unsafe_fastmem_check.SetGlobal(true); // Renderer values.renderer_backend.SetGlobal(true); diff --git a/src/common/settings.h b/src/common/settings.h index 48085b9a9..1af8c5ac2 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -125,10 +125,12 @@ struct Values { bool cpuopt_const_prop; bool cpuopt_misc_ir; bool cpuopt_reduce_misalign_checks; + bool cpuopt_fastmem; Setting cpuopt_unsafe_unfuse_fma; Setting cpuopt_unsafe_reduce_fp_error; Setting cpuopt_unsafe_inaccurate_nan; + Setting cpuopt_unsafe_fastmem_check; // Renderer Setting renderer_backend; @@ -249,6 +251,8 @@ void SetConfiguringGlobal(bool is_global); bool IsGPULevelExtreme(); bool IsGPULevelHigh(); +bool IsFastmemEnabled(); + float Volume(); std::string GetTimeZoneString(); -- cgit v1.2.3 From ee67460ff0e12a1603431d86fe3919a24b3858fb Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Sun, 6 Jun 2021 20:53:26 -0300 Subject: host_memory: Support staged VirtualProtect calls --- src/common/host_memory.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'src/common') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 8a328f916..c6d65aab9 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -110,9 +110,18 @@ public: } else { UNIMPLEMENTED_MSG("Protection flag combination read={} write={}", read, write); } - DWORD old_flags{}; - if (!VirtualProtect(virtual_base + virtual_offset, length, new_flags, &old_flags)) { - LOG_CRITICAL(HW_Memory, "Failed to change virtual memory protect rules"); + const size_t virtual_end = virtual_offset + length; + + std::lock_guard lock{placeholder_mutex}; + auto [it, end] = placeholders.equal_range({virtual_offset, virtual_end}); + while (it != end) { + const size_t offset = std::max(it->lower(), virtual_offset); + const size_t protect_length = std::min(it->upper(), virtual_end) - offset; + DWORD old_flags{}; + if (!VirtualProtect(virtual_base + offset, protect_length, new_flags, &old_flags)) { + LOG_CRITICAL(HW_Memory, "Failed to change virtual memory protect rules"); + } + ++it; } } -- cgit v1.2.3 From f332d4a9b548e9c7e18c245fd3b90ffc5a94b943 Mon Sep 17 00:00:00 2001 From: ReinUsesLisp Date: Tue, 8 Jun 2021 02:14:12 -0300 Subject: common/host_shader: Load Windows 10 functions dynamically Workaround old headers and libraries shipped on MinGW. --- src/common/host_memory.cpp | 117 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 29 deletions(-) (limited to 'src/common') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index c6d65aab9..9ae384f01 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -1,18 +1,10 @@ #ifdef _WIN32 -#ifdef _WIN32_WINNT -#undef _WIN32_WINNT -#endif -#define _WIN32_WINNT 0x0A00 // Windows 10 - -#include - -#include - #include #include - -#pragma comment(lib, "mincore.lib") +#include +#include +#include "common/dynamic_library.h" #elif defined(__linux__) // ^^^ Windows ^^^ vvv Linux vvv @@ -40,38 +32,99 @@ constexpr size_t HugePageSize = 0x200000; #ifdef _WIN32 +// Manually imported for MinGW compatibility +#ifndef MEM_RESERVE_PLACEHOLDER +#define MEM_RESERVE_PLACEHOLDER 0x0004000 +#endif +#ifndef MEM_REPLACE_PLACEHOLDER +#define MEM_REPLACE_PLACEHOLDER 0x00004000 +#endif +#ifndef MEM_COALESCE_PLACEHOLDERS +#define MEM_COALESCE_PLACEHOLDERS 0x00000001 +#endif +#ifndef MEM_PRESERVE_PLACEHOLDER +#define MEM_PRESERVE_PLACEHOLDER 0x00000002 +#endif + +using PFN_CreateFileMapping2 = _Ret_maybenull_ HANDLE(WINAPI*)( + _In_ HANDLE File, _In_opt_ SECURITY_ATTRIBUTES* SecurityAttributes, _In_ ULONG DesiredAccess, + _In_ ULONG PageProtection, _In_ ULONG AllocationAttributes, _In_ ULONG64 MaximumSize, + _In_opt_ PCWSTR Name, + _Inout_updates_opt_(ParameterCount) MEM_EXTENDED_PARAMETER* ExtendedParameters, + _In_ ULONG ParameterCount); + +using PFN_VirtualAlloc2 = _Ret_maybenull_ PVOID(WINAPI*)( + _In_opt_ HANDLE Process, _In_opt_ PVOID BaseAddress, _In_ SIZE_T Size, + _In_ ULONG AllocationType, _In_ ULONG PageProtection, + _Inout_updates_opt_(ParameterCount) MEM_EXTENDED_PARAMETER* ExtendedParameters, + _In_ ULONG ParameterCount); + +using PFN_MapViewOfFile3 = _Ret_maybenull_ PVOID(WINAPI*)( + _In_ HANDLE FileMapping, _In_opt_ HANDLE Process, _In_opt_ PVOID BaseAddress, + _In_ ULONG64 Offset, _In_ SIZE_T ViewSize, _In_ ULONG AllocationType, _In_ ULONG PageProtection, + _Inout_updates_opt_(ParameterCount) MEM_EXTENDED_PARAMETER* ExtendedParameters, + _In_ ULONG ParameterCount); + +using PFN_UnmapViewOfFile2 = BOOL(WINAPI*)(_In_ HANDLE Process, _In_ PVOID BaseAddress, + _In_ ULONG UnmapFlags); + +template +static void GetFuncAddress(Common::DynamicLibrary& dll, const char* name, T& pfn) { + if (!dll.GetSymbol(name, &pfn)) { + LOG_CRITICAL(HW_Memory, "Failed to load {}", name); + throw std::bad_alloc{}; + } +} + class HostMemory::Impl { public: explicit Impl(size_t backing_size_, size_t virtual_size_) - : backing_size{backing_size_}, virtual_size{virtual_size_}, process{GetCurrentProcess()} { + : backing_size{backing_size_}, virtual_size{virtual_size_}, process{GetCurrentProcess()}, + kernelbase_dll("Kernelbase") { + if (!kernelbase_dll.IsOpen()) { + LOG_CRITICAL(HW_Memory, "Failed to load Kernelbase.dll"); + throw std::bad_alloc{}; + } + GetFuncAddress(kernelbase_dll, "CreateFileMapping2", pfn_CreateFileMapping2); + GetFuncAddress(kernelbase_dll, "VirtualAlloc2", pfn_VirtualAlloc2); + GetFuncAddress(kernelbase_dll, "MapViewOfFile3", pfn_MapViewOfFile3); + GetFuncAddress(kernelbase_dll, "UnmapViewOfFile2", pfn_UnmapViewOfFile2); + // Allocate backing file map backing_handle = - CreateFileMapping2(INVALID_HANDLE_VALUE, nullptr, FILE_MAP_WRITE | FILE_MAP_READ, - PAGE_READWRITE, SEC_COMMIT, backing_size, nullptr, nullptr, 0); + pfn_CreateFileMapping2(INVALID_HANDLE_VALUE, nullptr, FILE_MAP_WRITE | FILE_MAP_READ, + PAGE_READWRITE, SEC_COMMIT, backing_size, nullptr, nullptr, 0); if (!backing_handle) { + LOG_CRITICAL(HW_Memory, "Failed to allocate {} MiB of backing memory", + backing_size >> 20); throw std::bad_alloc{}; } // Allocate a virtual memory for the backing file map as placeholder - backing_base = static_cast(VirtualAlloc2(process, nullptr, backing_size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, - PAGE_NOACCESS, nullptr, 0)); + backing_base = static_cast(pfn_VirtualAlloc2(process, nullptr, backing_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, + PAGE_NOACCESS, nullptr, 0)); if (!backing_base) { Release(); + LOG_CRITICAL(HW_Memory, "Failed to reserve {} MiB of virtual memory", + backing_size >> 20); throw std::bad_alloc{}; } // Map backing placeholder - void* const ret = MapViewOfFile3(backing_handle, process, backing_base, 0, backing_size, - MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0); + void* const ret = pfn_MapViewOfFile3(backing_handle, process, backing_base, 0, backing_size, + MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0); if (ret != backing_base) { Release(); + LOG_CRITICAL(HW_Memory, "Failed to map {} MiB of virtual memory", backing_size >> 20); throw std::bad_alloc{}; } // Allocate virtual address placeholder - virtual_base = static_cast(VirtualAlloc2(process, nullptr, virtual_size, - MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, - PAGE_NOACCESS, nullptr, 0)); + virtual_base = static_cast(pfn_VirtualAlloc2(process, nullptr, virtual_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, + PAGE_NOACCESS, nullptr, 0)); if (!virtual_base) { Release(); + LOG_CRITICAL(HW_Memory, "Failed to reserve {} GiB of virtual memory", + virtual_size >> 30); throw std::bad_alloc{}; } } @@ -136,8 +189,8 @@ private: void Release() { if (!placeholders.empty()) { for (const auto& placeholder : placeholders) { - if (!UnmapViewOfFile2(process, virtual_base + placeholder.lower(), - MEM_PRESERVE_PLACEHOLDER)) { + if (!pfn_UnmapViewOfFile2(process, virtual_base + placeholder.lower(), + MEM_PRESERVE_PLACEHOLDER)) { LOG_CRITICAL(HW_Memory, "Failed to unmap virtual memory placeholder"); } } @@ -149,7 +202,7 @@ private: } } if (backing_base) { - if (!UnmapViewOfFile2(process, backing_base, MEM_PRESERVE_PLACEHOLDER)) { + if (!pfn_UnmapViewOfFile2(process, backing_base, MEM_PRESERVE_PLACEHOLDER)) { LOG_CRITICAL(HW_Memory, "Failed to unmap backing memory placeholder"); } if (!VirtualFreeEx(process, backing_base, 0, MEM_RELEASE)) { @@ -184,8 +237,8 @@ private: const bool split_left = unmap_begin > placeholder_begin; const bool split_right = unmap_end < placeholder_end; - if (!UnmapViewOfFile2(process, virtual_base + placeholder_begin, - MEM_PRESERVE_PLACEHOLDER)) { + if (!pfn_UnmapViewOfFile2(process, virtual_base + placeholder_begin, + MEM_PRESERVE_PLACEHOLDER)) { LOG_CRITICAL(HW_Memory, "Failed to unmap placeholder"); } // If we have to remap memory regions due to partial unmaps, we are in a data race as @@ -235,8 +288,8 @@ private: } void MapView(size_t virtual_offset, size_t host_offset, size_t length) { - if (!MapViewOfFile3(backing_handle, process, virtual_base + virtual_offset, host_offset, - length, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0)) { + if (!pfn_MapViewOfFile3(backing_handle, process, virtual_base + virtual_offset, host_offset, + length, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0)) { LOG_CRITICAL(HW_Memory, "Failed to map placeholder"); } } @@ -279,6 +332,12 @@ private: HANDLE process{}; ///< Current process handle HANDLE backing_handle{}; ///< File based backing memory + DynamicLibrary kernelbase_dll; + PFN_CreateFileMapping2 pfn_CreateFileMapping2{}; + PFN_VirtualAlloc2 pfn_VirtualAlloc2{}; + PFN_MapViewOfFile3 pfn_MapViewOfFile3{}; + PFN_UnmapViewOfFile2 pfn_UnmapViewOfFile2{}; + std::mutex placeholder_mutex; ///< Mutex for placeholders boost::icl::separate_interval_set placeholders; ///< Mapped placeholders std::unordered_map placeholder_host_pointers; ///< Placeholder backing offset -- cgit v1.2.3 From 7f85abb28120fbb57bb813b828ee42f2a2031990 Mon Sep 17 00:00:00 2001 From: Markus Wick Date: Fri, 11 Jun 2021 11:47:23 +0200 Subject: common/host_memory: Implement a fallback if fastmem fails. This falls back to the old approach of using a virtual buffer. Windows is untested, but this build should fix support for Windows < 10 v1803. However without fastmem support at all. --- src/common/host_memory.cpp | 59 +++++++++++++++++++++++++++++++++++----------- src/common/host_memory.h | 4 ++++ 2 files changed, 49 insertions(+), 14 deletions(-) (limited to 'src/common') diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 9ae384f01..8bd70abc7 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -449,21 +449,52 @@ private: int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create }; -#else // ^^^ Linux ^^^ +#else // ^^^ Linux ^^^ vvv Generic vvv -#error Please implement the host memory for your platform +class HostMemory::Impl { +public: + explicit Impl(size_t /*backing_size */, size_t /* virtual_size */) { + // This is just a place holder. + // Please implement fastmem in a propper way on your platform. + throw std::bad_alloc{}; + } -#endif + void Map(size_t virtual_offset, size_t host_offset, size_t length) {} + + void Unmap(size_t virtual_offset, size_t length) {} + + void Protect(size_t virtual_offset, size_t length, bool read, bool write) {} + + u8* backing_base{nullptr}; + u8* virtual_base{nullptr}; +}; + +#endif // ^^^ Generic ^^^ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) - : backing_size(backing_size_), - virtual_size(virtual_size_), impl{std::make_unique( - AlignUp(backing_size, PageAlignment), - AlignUp(virtual_size, PageAlignment) + 3 * HugePageSize)}, - backing_base{impl->backing_base}, virtual_base{impl->virtual_base} { - virtual_base += 2 * HugePageSize - 1; - virtual_base -= reinterpret_cast(virtual_base) & (HugePageSize - 1); - virtual_base_offset = virtual_base - impl->virtual_base; + : backing_size(backing_size_), virtual_size(virtual_size_) { + try { + // Try to allocate a fastmem arena. + // The implementation will fail with std::bad_alloc on errors. + impl = std::make_unique(AlignUp(backing_size, PageAlignment), + AlignUp(virtual_size, PageAlignment) + + 3 * HugePageSize); + backing_base = impl->backing_base; + virtual_base = impl->virtual_base; + + if (virtual_base) { + virtual_base += 2 * HugePageSize - 1; + virtual_base -= reinterpret_cast(virtual_base) & (HugePageSize - 1); + virtual_base_offset = virtual_base - impl->virtual_base; + } + + } catch (const std::bad_alloc&) { + LOG_CRITICAL(HW_Memory, + "Fastmem unavailable, falling back to VirtualBuffer for memory allocation"); + fallback_buffer = std::make_unique>(backing_size); + backing_base = fallback_buffer->data(); + virtual_base = nullptr; + } } HostMemory::~HostMemory() = default; @@ -478,7 +509,7 @@ void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length) { ASSERT(length % PageAlignment == 0); ASSERT(virtual_offset + length <= virtual_size); ASSERT(host_offset + length <= backing_size); - if (length == 0) { + if (length == 0 || !virtual_base || !impl) { return; } impl->Map(virtual_offset + virtual_base_offset, host_offset, length); @@ -488,7 +519,7 @@ void HostMemory::Unmap(size_t virtual_offset, size_t length) { ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); ASSERT(virtual_offset + length <= virtual_size); - if (length == 0) { + if (length == 0 || !virtual_base || !impl) { return; } impl->Unmap(virtual_offset + virtual_base_offset, length); @@ -498,7 +529,7 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool w ASSERT(virtual_offset % PageAlignment == 0); ASSERT(length % PageAlignment == 0); ASSERT(virtual_offset + length <= virtual_size); - if (length == 0) { + if (length == 0 || !virtual_base || !impl) { return; } impl->Protect(virtual_offset + virtual_base_offset, length, read, write); diff --git a/src/common/host_memory.h b/src/common/host_memory.h index eaa7d18ab..9b8326d0f 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -6,6 +6,7 @@ #include #include "common/common_types.h" +#include "common/virtual_buffer.h" namespace Common { @@ -61,6 +62,9 @@ private: u8* backing_base{}; u8* virtual_base{}; size_t virtual_base_offset{}; + + // Fallback if fastmem is not supported on this platform + std::unique_ptr> fallback_buffer; }; } // namespace Common -- cgit v1.2.3