From feb49c822d9cabc5bc7be9eab1f2bf4ba460176a Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 14 Nov 2021 20:55:52 +0100 Subject: NVDRV: Remake ASGPU --- .../hle/service/nvdrv/devices/nvhost_as_gpu.cpp | 460 ++++++++++++++------- src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h | 163 ++++---- 2 files changed, 388 insertions(+), 235 deletions(-) (limited to 'src/core/hle/service') diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 5c70c9a57..344ddfc90 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -6,6 +6,7 @@ #include #include +#include "common/alignment.h" #include "common/assert.h" #include "common/logging/log.h" #include "core/core.h" @@ -21,8 +22,8 @@ namespace Service::Nvidia::Devices { nvhost_as_gpu::nvhost_as_gpu(Core::System& system_, Module& module_, NvCore::Container& core) - : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, - gmmu{std::make_shared(system)} {} + : nvdevice{system_}, module{module_}, container{core}, nvmap{core.GetNvMapFile()}, vm{}, + gmmu{} {} nvhost_as_gpu::~nvhost_as_gpu() = default; NvResult nvhost_as_gpu::Ioctl1(DeviceFD fd, Ioctl command, const std::vector& input, @@ -89,12 +90,49 @@ NvResult nvhost_as_gpu::AllocAsEx(const std::vector& input, std::vector& IoctlAllocAsEx params{}; std::memcpy(¶ms, input.data(), input.size()); - LOG_WARNING(Service_NVDRV, "(STUBBED) called, big_page_size=0x{:X}", params.big_page_size); - if (params.big_page_size == 0) { - params.big_page_size = DEFAULT_BIG_PAGE_SIZE; + LOG_DEBUG(Service_NVDRV, "called, big_page_size=0x{:X}", params.big_page_size); + + std::scoped_lock lock(mutex); + + if (vm.initialised) { + UNREACHABLE_MSG("Cannot initialise an address space twice!"); + return NvResult::InvalidState; } - big_page_size = params.big_page_size; + if (params.big_page_size) { + if (!std::has_single_bit(params.big_page_size)) { + LOG_ERROR(Service_NVDRV, "Non power-of-2 big page size: 0x{:X}!", params.big_page_size); + return NvResult::BadValue; + } + + if (!(params.big_page_size & VM::SUPPORTED_BIG_PAGE_SIZES)) { + LOG_ERROR(Service_NVDRV, "Unsupported big page size: 0x{:X}!", params.big_page_size); + return NvResult::BadValue; + } + + vm.big_page_size = params.big_page_size; + vm.big_page_size_bits = static_cast(std::countr_zero(params.big_page_size)); + + vm.va_range_start = params.big_page_size << VM::VA_START_SHIFT; + } + + // If this is unspecified then default values should be used + if (params.va_range_start) { + vm.va_range_start = params.va_range_start; + vm.va_range_split = params.va_range_split; + vm.va_range_end = params.va_range_end; + } + + const u64 start_pages{vm.va_range_start >> VM::PAGE_SIZE_BITS}; + const u64 end_pages{vm.va_range_split >> VM::PAGE_SIZE_BITS}; + vm.small_page_allocator = std::make_shared(start_pages, end_pages); + + const u64 start_big_pages{vm.va_range_split >> vm.big_page_size_bits}; + const u64 end_big_pages{(vm.va_range_end - vm.va_range_split) >> vm.big_page_size_bits}; + vm.big_page_allocator = std::make_unique(start_big_pages, end_big_pages); + + gmmu = std::make_shared(system, 40, VM::PAGE_SIZE_BITS); + vm.initialised = true; return NvResult::Success; } @@ -106,21 +144,73 @@ NvResult nvhost_as_gpu::AllocateSpace(const std::vector& input, std::vector< LOG_DEBUG(Service_NVDRV, "called, pages={:X}, page_size={:X}, flags={:X}", params.pages, params.page_size, params.flags); - const auto size{static_cast(params.pages) * static_cast(params.page_size)}; - if ((params.flags & AddressSpaceFlags::FixedOffset) != AddressSpaceFlags::None) { - params.offset = *(gmmu->AllocateFixed(params.offset, size)); + std::scoped_lock lock(mutex); + + if (!vm.initialised) { + return NvResult::BadValue; + } + + if (params.page_size != VM::YUZU_PAGESIZE && params.page_size != vm.big_page_size) { + return NvResult::BadValue; + } + + if (params.page_size != vm.big_page_size && + ((params.flags & MappingFlags::Sparse) != MappingFlags::None)) { + UNIMPLEMENTED_MSG("Sparse small pages are not implemented!"); + return NvResult::NotImplemented; + } + + const u32 page_size_bits{params.page_size == VM::YUZU_PAGESIZE ? VM::PAGE_SIZE_BITS + : vm.big_page_size_bits}; + + auto& allocator{params.page_size == VM::YUZU_PAGESIZE ? *vm.small_page_allocator + : *vm.big_page_allocator}; + + if ((params.flags & MappingFlags::Fixed) != MappingFlags::None) { + allocator.AllocateFixed(static_cast(params.offset >> page_size_bits), params.pages); } else { - params.offset = gmmu->Allocate(size, params.align); + params.offset = static_cast(allocator.Allocate(params.pages)) << page_size_bits; + if (!params.offset) { + UNREACHABLE_MSG("Failed to allocate free space in the GPU AS!"); + return NvResult::InsufficientMemory; + } } - auto result = NvResult::Success; - if (!params.offset) { - LOG_CRITICAL(Service_NVDRV, "allocation failed for size {}", size); - result = NvResult::InsufficientMemory; + u64 size{static_cast(params.pages) * params.page_size}; + + if ((params.flags & MappingFlags::Sparse) != MappingFlags::None) { + gmmu->MapSparse(params.offset, size); } + allocation_map[params.offset] = { + .size = size, + .page_size = params.page_size, + .sparse = (params.flags & MappingFlags::Sparse) != MappingFlags::None, + }; + std::memcpy(output.data(), ¶ms, output.size()); - return result; + return NvResult::Success; +} + +void nvhost_as_gpu::FreeMappingLocked(u64 offset) { + auto mapping{mapping_map.at(offset)}; + + if (!mapping->fixed) { + auto& allocator{mapping->big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; + u32 page_size_bits{mapping->big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; + + allocator.Free(static_cast(mapping->offset >> page_size_bits), + static_cast(mapping->size >> page_size_bits)); + } + + // Sparse mappings shouldn't be fully unmapped, just returned to their sparse state + // Only FreeSpace can unmap them fully + if (mapping->sparse_alloc) + gmmu->MapSparse(offset, mapping->size); + else + gmmu->Unmap(offset, mapping->size); + + mapping_map.erase(offset); } NvResult nvhost_as_gpu::FreeSpace(const std::vector& input, std::vector& output) { @@ -130,7 +220,40 @@ NvResult nvhost_as_gpu::FreeSpace(const std::vector& input, std::vector& LOG_DEBUG(Service_NVDRV, "called, offset={:X}, pages={:X}, page_size={:X}", params.offset, params.pages, params.page_size); - gmmu->Unmap(params.offset, static_cast(params.pages) * params.page_size); + std::scoped_lock lock(mutex); + + if (!vm.initialised) { + return NvResult::BadValue; + } + + try { + auto allocation{allocation_map[params.offset]}; + + if (allocation.page_size != params.page_size || + allocation.size != (static_cast(params.pages) * params.page_size)) { + return NvResult::BadValue; + } + + for (const auto& mapping : allocation.mappings) { + FreeMappingLocked(mapping->offset); + } + + // Unset sparse flag if required + if (allocation.sparse) { + gmmu->Unmap(params.offset, allocation.size); + } + + auto& allocator{params.page_size == VM::YUZU_PAGESIZE ? *vm.small_page_allocator + : *vm.big_page_allocator}; + u32 page_size_bits{params.page_size == VM::YUZU_PAGESIZE ? VM::PAGE_SIZE_BITS + : vm.big_page_size_bits}; + + allocator.Free(static_cast(params.offset >> page_size_bits), + static_cast(allocation.size >> page_size_bits)); + allocation_map.erase(params.offset); + } catch ([[maybe_unused]] const std::out_of_range& e) { + return NvResult::BadValue; + } std::memcpy(output.data(), ¶ms, output.size()); return NvResult::Success; @@ -141,43 +264,51 @@ NvResult nvhost_as_gpu::Remap(const std::vector& input, std::vector& out LOG_DEBUG(Service_NVDRV, "called, num_entries=0x{:X}", num_entries); - auto result = NvResult::Success; std::vector entries(num_entries); std::memcpy(entries.data(), input.data(), input.size()); + std::scoped_lock lock(mutex); + + if (!vm.initialised) { + return NvResult::BadValue; + } + for (const auto& entry : entries) { - LOG_DEBUG(Service_NVDRV, "remap entry, offset=0x{:X} handle=0x{:X} pages=0x{:X}", - entry.offset, entry.nvmap_handle, entry.pages); - - if (entry.nvmap_handle == 0) { - // If nvmap handle is null, we should unmap instead. - const auto offset{static_cast(entry.offset) << 0x10}; - const auto size{static_cast(entry.pages) << 0x10}; - gmmu->Unmap(offset, size); - continue; + GPUVAddr virtual_address{static_cast(entry.as_offset_big_pages) + << vm.big_page_size_bits}; + u64 size{static_cast(entry.big_pages) << vm.big_page_size_bits}; + + auto alloc{allocation_map.upper_bound(virtual_address)}; + + if (alloc-- == allocation_map.begin() || + (virtual_address - alloc->first) + size > alloc->second.size) { + LOG_WARNING(Service_NVDRV, "Cannot remap into an unallocated region!"); + return NvResult::BadValue; } - const auto object{nvmap.GetHandle(entry.nvmap_handle)}; - if (!object) { - LOG_CRITICAL(Service_NVDRV, "invalid nvmap_handle={:X}", entry.nvmap_handle); - result = NvResult::InvalidState; - break; + if (!alloc->second.sparse) { + LOG_WARNING(Service_NVDRV, "Cannot remap a non-sparse mapping!"); + return NvResult::BadValue; } - const auto offset{static_cast(entry.offset) << 0x10}; - const auto size{static_cast(entry.pages) << 0x10}; - const auto map_offset{static_cast(entry.map_offset) << 0x10}; - const auto addr{gmmu->Map(object->address + map_offset, offset, size)}; + if (!entry.handle) { + gmmu->MapSparse(virtual_address, size); + } else { + auto handle{nvmap.GetHandle(entry.handle)}; + if (!handle) { + return NvResult::BadValue; + } - if (!addr) { - LOG_CRITICAL(Service_NVDRV, "map returned an invalid address!"); - result = NvResult::InvalidState; - break; + VAddr cpu_address{static_cast( + handle->address + + (static_cast(entry.handle_offset_big_pages) << vm.big_page_size_bits))}; + + gmmu->Map(virtual_address, cpu_address, size); } } std::memcpy(output.data(), entries.data(), output.size()); - return result; + return NvResult::Success; } NvResult nvhost_as_gpu::MapBufferEx(const std::vector& input, std::vector& output) { @@ -187,75 +318,96 @@ NvResult nvhost_as_gpu::MapBufferEx(const std::vector& input, std::vector(buffer_map->CpuAddr() + params.buffer_offset)}; - const auto gpu_addr{static_cast(params.offset + params.buffer_offset)}; + std::scoped_lock lock(mutex); + + if (!vm.initialised) { + return NvResult::BadValue; + } - if (!gmmu->Map(cpu_addr, gpu_addr, params.mapping_size)) { - LOG_CRITICAL(Service_NVDRV, - "remap failed, flags={:X}, nvmap_handle={:X}, buffer_offset={}, " - "mapping_size = {}, offset={}", - params.flags, params.nvmap_handle, params.buffer_offset, - params.mapping_size, params.offset); + // Remaps a subregion of an existing mapping to a different PA + if ((params.flags & MappingFlags::Remap) != MappingFlags::None) { + try { + auto mapping{mapping_map.at(params.offset)}; - std::memcpy(output.data(), ¶ms, output.size()); - return NvResult::InvalidState; + if (mapping->size < params.mapping_size) { + LOG_WARNING(Service_NVDRV, + "Cannot remap a partially mapped GPU address space region: 0x{:X}", + params.offset); + return NvResult::BadValue; } - std::memcpy(output.data(), ¶ms, output.size()); - return NvResult::Success; - } else { - LOG_CRITICAL(Service_NVDRV, "address not mapped offset={}", params.offset); + u64 gpu_address{static_cast(params.offset + params.buffer_offset)}; + VAddr cpu_address{mapping->ptr + params.buffer_offset}; + + gmmu->Map(gpu_address, cpu_address, params.mapping_size); - std::memcpy(output.data(), ¶ms, output.size()); - return NvResult::InvalidState; + return NvResult::Success; + } catch ([[maybe_unused]] const std::out_of_range& e) { + LOG_WARNING(Service_NVDRV, "Cannot remap an unmapped GPU address space region: 0x{:X}", + params.offset); + return NvResult::BadValue; } } - const auto object{nvmap.GetHandle(params.nvmap_handle)}; - if (!object) { - LOG_CRITICAL(Service_NVDRV, "invalid nvmap_handle={:X}", params.nvmap_handle); - std::memcpy(output.data(), ¶ms, output.size()); - return NvResult::InvalidState; + auto handle{nvmap.GetHandle(params.handle)}; + if (!handle) { + return NvResult::BadValue; } - // The real nvservices doesn't make a distinction between handles and ids, and - // object can only have one handle and it will be the same as its id. Assert that this is the - // case to prevent unexpected behavior. - ASSERT(object->id == params.nvmap_handle); + VAddr cpu_address{static_cast(handle->address + params.buffer_offset)}; + u64 size{params.mapping_size ? params.mapping_size : handle->orig_size}; - u64 page_size{params.page_size}; - if (!page_size) { - page_size = object->align; - } + if ((params.flags & MappingFlags::Fixed) != MappingFlags::None) { + auto alloc{allocation_map.upper_bound(params.offset)}; - const auto physical_address{object->address + params.buffer_offset}; - u64 size{params.mapping_size}; - if (!size) { - size = object->size; - } + if (alloc-- == allocation_map.begin() || + (params.offset - alloc->first) + size > alloc->second.size) { + UNREACHABLE_MSG("Cannot perform a fixed mapping into an unallocated region!"); + return NvResult::BadValue; + } - const bool is_alloc{(params.flags & AddressSpaceFlags::FixedOffset) == AddressSpaceFlags::None}; - if (is_alloc) { - params.offset = gmmu->MapAllocate(physical_address, size, page_size); - } else { - params.offset = gmmu->Map(physical_address, params.offset, size); - } + gmmu->Map(params.offset, cpu_address, size); - auto result = NvResult::Success; - if (!params.offset) { - LOG_CRITICAL(Service_NVDRV, "failed to map size={}", size); - result = NvResult::InvalidState; + auto mapping{std::make_shared(cpu_address, params.offset, size, true, false, + alloc->second.sparse)}; + alloc->second.mappings.push_back(mapping); + mapping_map[params.offset] = mapping; } else { - AddBufferMap(params.offset, size, physical_address, is_alloc); + bool big_page{[&]() { + if (Common::IsAligned(handle->align, vm.big_page_size)) + return true; + else if (Common::IsAligned(handle->align, VM::YUZU_PAGESIZE)) + return false; + else { + UNREACHABLE(); + return false; + } + }()}; + + auto& allocator{big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; + u32 page_size{big_page ? vm.big_page_size : VM::YUZU_PAGESIZE}; + u32 page_size_bits{big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; + + params.offset = static_cast(allocator.Allocate( + static_cast(Common::AlignUp(size, page_size) >> page_size_bits))) + << page_size_bits; + if (!params.offset) { + UNREACHABLE_MSG("Failed to allocate free space in the GPU AS!"); + return NvResult::InsufficientMemory; + } + + gmmu->Map(params.offset, cpu_address, size); + + auto mapping{ + std::make_shared(cpu_address, params.offset, size, false, big_page, false)}; + mapping_map[params.offset] = mapping; } std::memcpy(output.data(), ¶ms, output.size()); - return result; + return NvResult::Success; } NvResult nvhost_as_gpu::UnmapBuffer(const std::vector& input, std::vector& output) { @@ -264,13 +416,36 @@ NvResult nvhost_as_gpu::UnmapBuffer(const std::vector& input, std::vectorUnmap(params.offset, *size); - } else { - LOG_ERROR(Service_NVDRV, "invalid offset=0x{:X}", params.offset); + std::scoped_lock lock(mutex); + + if (!vm.initialised) { + return NvResult::BadValue; + } + + try { + auto mapping{mapping_map.at(params.offset)}; + + if (!mapping->fixed) { + auto& allocator{mapping->big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; + u32 page_size_bits{mapping->big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; + + allocator.Free(static_cast(mapping->offset >> page_size_bits), + static_cast(mapping->size >> page_size_bits)); + } + + // Sparse mappings shouldn't be fully unmapped, just returned to their sparse state + // Only FreeSpace can unmap them fully + if (mapping->sparse_alloc) { + gmmu->MapSparse(params.offset, mapping->size); + } else { + gmmu->Unmap(params.offset, mapping->size); + } + + mapping_map.erase(params.offset); + } catch ([[maybe_unused]] const std::out_of_range& e) { + LOG_WARNING(Service_NVDRV, "Couldn't find region to unmap at 0x{:X}", params.offset); } - std::memcpy(output.data(), ¶ms, output.size()); return NvResult::Success; } @@ -284,28 +459,37 @@ NvResult nvhost_as_gpu::BindChannel(const std::vector& input, std::vector{ + VaRegion{ + .offset = vm.small_page_allocator->vaStart << VM::PAGE_SIZE_BITS, + .page_size = VM::YUZU_PAGESIZE, + .pages = vm.small_page_allocator->vaLimit - vm.small_page_allocator->vaStart, + }, + VaRegion{ + .offset = vm.big_page_allocator->vaStart << vm.big_page_size_bits, + .page_size = vm.big_page_size, + .pages = vm.big_page_allocator->vaLimit - vm.big_page_allocator->vaStart, + }, + }; +} + NvResult nvhost_as_gpu::GetVARegions(const std::vector& input, std::vector& output) { IoctlGetVaRegions params{}; std::memcpy(¶ms, input.data(), input.size()); - LOG_WARNING(Service_NVDRV, "(STUBBED) called, buf_addr={:X}, buf_size={:X}", params.buf_addr, - params.buf_size); - - params.buf_size = 0x30; + LOG_DEBUG(Service_NVDRV, "called, buf_addr={:X}, buf_size={:X}", params.buf_addr, + params.buf_size); - params.small = IoctlVaRegion{ - .offset = 0x04000000, - .page_size = DEFAULT_SMALL_PAGE_SIZE, - .pages = 0x3fbfff, - }; + std::scoped_lock lock(mutex); - params.big = IoctlVaRegion{ - .offset = 0x04000000, - .page_size = big_page_size, - .pages = 0x1bffff, - }; + if (!vm.initialised) { + return NvResult::BadValue; + } - // TODO(ogniK): This probably can stay stubbed but should add support way way later + GetVARegionsImpl(params); std::memcpy(output.data(), ¶ms, output.size()); return NvResult::Success; @@ -316,64 +500,24 @@ NvResult nvhost_as_gpu::GetVARegions(const std::vector& input, std::vector nvhost_as_gpu::FindBufferMap(GPUVAddr gpu_addr) const { - const auto end{buffer_mappings.upper_bound(gpu_addr)}; - for (auto iter{buffer_mappings.begin()}; iter != end; ++iter) { - if (gpu_addr >= iter->second.StartAddr() && gpu_addr < iter->second.EndAddr()) { - return iter->second; - } - } - - return std::nullopt; -} - -void nvhost_as_gpu::AddBufferMap(GPUVAddr gpu_addr, std::size_t size, VAddr cpu_addr, - bool is_allocated) { - buffer_mappings[gpu_addr] = {gpu_addr, size, cpu_addr, is_allocated}; -} - -std::optional nvhost_as_gpu::RemoveBufferMap(GPUVAddr gpu_addr) { - if (const auto iter{buffer_mappings.find(gpu_addr)}; iter != buffer_mappings.end()) { - std::size_t size{}; - - if (iter->second.IsAllocated()) { - size = iter->second.Size(); - } - - buffer_mappings.erase(iter); - - return size; - } - - return std::nullopt; -} - Kernel::KEvent* nvhost_as_gpu::QueryEvent(u32 event_id) { LOG_CRITICAL(Service_NVDRV, "Unknown AS GPU Event {}", event_id); return nullptr; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index f5fb33ba7..1d27739e2 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -5,14 +5,19 @@ #pragma once +#include +#include #include #include +#include #include #include +#include "common/address_space.h" #include "common/common_funcs.h" #include "common/common_types.h" #include "common/swap.h" +#include "core/hle/service/nvdrv/core/nvmap.h" #include "core/hle/service/nvdrv/devices/nvdevice.h" namespace Tegra { @@ -30,17 +35,13 @@ class NvMap; namespace Service::Nvidia::Devices { -constexpr u32 DEFAULT_BIG_PAGE_SIZE = 1 << 16; -constexpr u32 DEFAULT_SMALL_PAGE_SIZE = 1 << 12; - -class nvmap; - -enum class AddressSpaceFlags : u32 { - None = 0x0, - FixedOffset = 0x1, - Remap = 0x100, +enum class MappingFlags : u32 { + None = 0, + Fixed = 1 << 0, + Sparse = 1 << 1, + Remap = 1 << 8, }; -DECLARE_ENUM_FLAG_OPERATORS(AddressSpaceFlags); +DECLARE_ENUM_FLAG_OPERATORS(MappingFlags); class nvhost_as_gpu final : public nvdevice { public: @@ -59,46 +60,15 @@ public: Kernel::KEvent* QueryEvent(u32 event_id) override; -private: - class BufferMap final { - public: - constexpr BufferMap() = default; - - constexpr BufferMap(GPUVAddr start_addr_, std::size_t size_) - : start_addr{start_addr_}, end_addr{start_addr_ + size_} {} - - constexpr BufferMap(GPUVAddr start_addr_, std::size_t size_, VAddr cpu_addr_, - bool is_allocated_) - : start_addr{start_addr_}, end_addr{start_addr_ + size_}, cpu_addr{cpu_addr_}, - is_allocated{is_allocated_} {} - - constexpr VAddr StartAddr() const { - return start_addr; - } - - constexpr VAddr EndAddr() const { - return end_addr; - } - - constexpr std::size_t Size() const { - return end_addr - start_addr; - } - - constexpr VAddr CpuAddr() const { - return cpu_addr; - } - - constexpr bool IsAllocated() const { - return is_allocated; - } - - private: - GPUVAddr start_addr{}; - GPUVAddr end_addr{}; - VAddr cpu_addr{}; - bool is_allocated{}; + struct VaRegion { + u64 offset; + u32 page_size; + u32 _pad0_; + u64 pages; }; + static_assert(sizeof(VaRegion) == 0x18); +private: struct IoctlAllocAsEx { u32_le flags{}; // usually passes 1 s32_le as_fd{}; // ignored; passes 0 @@ -113,7 +83,7 @@ private: struct IoctlAllocSpace { u32_le pages{}; u32_le page_size{}; - AddressSpaceFlags flags{}; + MappingFlags flags{}; INSERT_PADDING_WORDS(1); union { u64_le offset; @@ -130,19 +100,19 @@ private: static_assert(sizeof(IoctlFreeSpace) == 16, "IoctlFreeSpace is incorrect size"); struct IoctlRemapEntry { - u16_le flags{}; - u16_le kind{}; - u32_le nvmap_handle{}; - u32_le map_offset{}; - u32_le offset{}; - u32_le pages{}; + u16 flags; + u16 kind; + NvCore::NvMap::Handle::Id handle; + u32 handle_offset_big_pages; + u32 as_offset_big_pages; + u32 big_pages; }; static_assert(sizeof(IoctlRemapEntry) == 20, "IoctlRemapEntry is incorrect size"); struct IoctlMapBufferEx { - AddressSpaceFlags flags{}; // bit0: fixed_offset, bit2: cacheable - u32_le kind{}; // -1 is default - u32_le nvmap_handle{}; + MappingFlags flags{}; // bit0: fixed_offset, bit2: cacheable + u32_le kind{}; // -1 is default + NvCore::NvMap::Handle::Id handle; u32_le page_size{}; // 0 means don't care s64_le buffer_offset{}; u64_le mapping_size{}; @@ -160,27 +130,15 @@ private: }; static_assert(sizeof(IoctlBindChannel) == 4, "IoctlBindChannel is incorrect size"); - struct IoctlVaRegion { - u64_le offset{}; - u32_le page_size{}; - INSERT_PADDING_WORDS(1); - u64_le pages{}; - }; - static_assert(sizeof(IoctlVaRegion) == 24, "IoctlVaRegion is incorrect size"); - struct IoctlGetVaRegions { u64_le buf_addr{}; // (contained output user ptr on linux, ignored) u32_le buf_size{}; // forced to 2*sizeof(struct va_region) u32_le reserved{}; - IoctlVaRegion small{}; - IoctlVaRegion big{}; + std::array regions{}; }; - static_assert(sizeof(IoctlGetVaRegions) == 16 + sizeof(IoctlVaRegion) * 2, + static_assert(sizeof(IoctlGetVaRegions) == 16 + sizeof(VaRegion) * 2, "IoctlGetVaRegions is incorrect size"); - s32 channel{}; - u32 big_page_size{DEFAULT_BIG_PAGE_SIZE}; - NvResult AllocAsEx(const std::vector& input, std::vector& output); NvResult AllocateSpace(const std::vector& input, std::vector& output); NvResult Remap(const std::vector& input, std::vector& output); @@ -189,23 +147,74 @@ private: NvResult FreeSpace(const std::vector& input, std::vector& output); NvResult BindChannel(const std::vector& input, std::vector& output); + void GetVARegionsImpl(IoctlGetVaRegions& params); NvResult GetVARegions(const std::vector& input, std::vector& output); NvResult GetVARegions(const std::vector& input, std::vector& output, std::vector& inline_output); - std::optional FindBufferMap(GPUVAddr gpu_addr) const; - void AddBufferMap(GPUVAddr gpu_addr, std::size_t size, VAddr cpu_addr, bool is_allocated); - std::optional RemoveBufferMap(GPUVAddr gpu_addr); + void FreeMappingLocked(u64 offset); Module& module; NvCore::Container& container; NvCore::NvMap& nvmap; + struct Mapping { + VAddr ptr; + u64 offset; + u64 size; + bool fixed; + bool big_page; // Only valid if fixed == false + bool sparse_alloc; + + Mapping(VAddr ptr_, u64 offset_, u64 size_, bool fixed_, bool big_page_, bool sparse_alloc_) + : ptr(ptr_), offset(offset_), size(size_), fixed(fixed_), big_page(big_page_), + sparse_alloc(sparse_alloc_) {} + }; + + struct Allocation { + u64 size; + std::list> mappings; + u32 page_size; + bool sparse; + }; + + std::map> + mapping_map; //!< This maps the base addresses of mapped buffers to their total sizes and + //!< mapping type, this is needed as what was originally a single buffer may + //!< have been split into multiple GPU side buffers with the remap flag. + std::map allocation_map; //!< Holds allocations created by AllocSpace from + //!< which fixed buffers can be mapped into + std::mutex mutex; //!< Locks all AS operations + + struct VM { + static constexpr u32 YUZU_PAGESIZE{0x1000}; + static constexpr u32 PAGE_SIZE_BITS{std::countr_zero(YUZU_PAGESIZE)}; + + static constexpr u32 SUPPORTED_BIG_PAGE_SIZES{0x30000}; + static constexpr u32 DEFAULT_BIG_PAGE_SIZE{0x20000}; + u32 big_page_size{DEFAULT_BIG_PAGE_SIZE}; + u32 big_page_size_bits{std::countr_zero(DEFAULT_BIG_PAGE_SIZE)}; + + static constexpr u32 VA_START_SHIFT{10}; + static constexpr u64 DEFAULT_VA_SPLIT{1ULL << 34}; + static constexpr u64 DEFAULT_VA_RANGE{1ULL << 37}; + u64 va_range_start{DEFAULT_BIG_PAGE_SIZE << VA_START_SHIFT}; + u64 va_range_split{DEFAULT_VA_SPLIT}; + u64 va_range_end{DEFAULT_VA_RANGE}; + + using Allocator = Common::FlatAllocator; + + std::unique_ptr big_page_allocator; + std::shared_ptr + small_page_allocator; //! Shared as this is also used by nvhost::GpuChannel + + bool initialised{}; + } vm; std::shared_ptr gmmu; - // This is expected to be ordered, therefore we must use a map, not unordered_map - std::map buffer_mappings; + // s32 channel{}; + // u32 big_page_size{VM::DEFAULT_BIG_PAGE_SIZE}; }; } // namespace Service::Nvidia::Devices -- cgit v1.2.3