From 22f3a7e94ce0e325a8c3dfeb0814c4b82a42649d Mon Sep 17 00:00:00 2001 From: tfarley Date: Sat, 16 Apr 2016 18:57:57 -0400 Subject: HWRasterizer: Texture forwarding --- src/core/hle/service/fs/archive.cpp | 1 + src/core/hle/service/gsp_gpu.cpp | 31 ++-- src/core/hle/service/y2r_u.cpp | 10 +- src/core/hw/gpu.cpp | 327 +++++++++++++++++++----------------- src/core/hw/gpu.h | 4 +- src/core/memory.cpp | 140 +++++++++++++++ src/core/memory.h | 16 ++ 7 files changed, 348 insertions(+), 181 deletions(-) (limited to 'src/core') diff --git a/src/core/hle/service/fs/archive.cpp b/src/core/hle/service/fs/archive.cpp index e9588cb72..cc51ede0c 100644 --- a/src/core/hle/service/fs/archive.cpp +++ b/src/core/hle/service/fs/archive.cpp @@ -114,6 +114,7 @@ ResultVal File::SyncRequest() { return read.Code(); } cmd_buff[2] = static_cast(*read); + Memory::RasterizerFlushAndInvalidateRegion(Memory::VirtualToPhysicalAddress(address), length); break; } diff --git a/src/core/hle/service/gsp_gpu.cpp b/src/core/hle/service/gsp_gpu.cpp index 0c655395e..211fcf599 100644 --- a/src/core/hle/service/gsp_gpu.cpp +++ b/src/core/hle/service/gsp_gpu.cpp @@ -4,6 +4,7 @@ #include "common/bit_field.h" #include "common/microprofile.h" +#include "common/profiler.h" #include "core/memory.h" #include "core/hle/kernel/event.h" @@ -15,8 +16,6 @@ #include "video_core/gpu_debugger.h" #include "video_core/debug_utils/debug_utils.h" -#include "video_core/renderer_base.h" -#include "video_core/video_core.h" #include "gsp_gpu.h" @@ -291,8 +290,6 @@ static void FlushDataCache(Service::Interface* self) { u32 size = cmd_buff[2]; u32 process = cmd_buff[4]; - VideoCore::g_renderer->Rasterizer()->InvalidateRegion(Memory::VirtualToPhysicalAddress(address), size); - // TODO(purpasmart96): Verify return header on HW cmd_buff[1] = RESULT_SUCCESS.raw; // No error @@ -408,6 +405,8 @@ void SignalInterrupt(InterruptId interrupt_id) { g_interrupt_event->Signal(); } +MICROPROFILE_DEFINE(GPU_GSP_DMA, "GPU", "GSP DMA", MP_RGB(100, 0, 255)); + /// Executes the next GSP command static void ExecuteCommand(const Command& command, u32 thread_id) { // Utility function to convert register ID to address @@ -419,18 +418,21 @@ static void ExecuteCommand(const Command& command, u32 thread_id) { // GX request DMA - typically used for copying memory from GSP heap to VRAM case CommandId::REQUEST_DMA: - VideoCore::g_renderer->Rasterizer()->FlushRegion(Memory::VirtualToPhysicalAddress(command.dma_request.source_address), - command.dma_request.size); + { + MICROPROFILE_SCOPE(GPU_GSP_DMA); + + // TODO: Consider attempting rasterizer-accelerated surface blit if that usage is ever possible/likely + Memory::RasterizerFlushRegion(Memory::VirtualToPhysicalAddress(command.dma_request.source_address), + command.dma_request.size); + Memory::RasterizerFlushAndInvalidateRegion(Memory::VirtualToPhysicalAddress(command.dma_request.dest_address), + command.dma_request.size); memcpy(Memory::GetPointer(command.dma_request.dest_address), Memory::GetPointer(command.dma_request.source_address), command.dma_request.size); SignalInterrupt(InterruptId::DMA); - - VideoCore::g_renderer->Rasterizer()->InvalidateRegion(Memory::VirtualToPhysicalAddress(command.dma_request.dest_address), - command.dma_request.size); break; - + } // TODO: This will need some rework in the future. (why?) case CommandId::SUBMIT_GPU_CMDLIST: { @@ -517,13 +519,8 @@ static void ExecuteCommand(const Command& command, u32 thread_id) { case CommandId::CACHE_FLUSH: { - for (auto& region : command.cache_flush.regions) { - if (region.size == 0) - break; - - VideoCore::g_renderer->Rasterizer()->InvalidateRegion( - Memory::VirtualToPhysicalAddress(region.address), region.size); - } + // NOTE: Rasterizer flushing handled elsewhere in CPU read/write and other GPU handlers + // Use command.cache_flush.regions to implement this handler break; } diff --git a/src/core/hle/service/y2r_u.cpp b/src/core/hle/service/y2r_u.cpp index 22f373adf..1672ad775 100644 --- a/src/core/hle/service/y2r_u.cpp +++ b/src/core/hle/service/y2r_u.cpp @@ -12,9 +12,6 @@ #include "core/hle/service/y2r_u.h" #include "core/hw/y2r.h" -#include "video_core/renderer_base.h" -#include "video_core/video_core.h" - //////////////////////////////////////////////////////////////////////////////////////////////////// // Namespace Y2R_U @@ -262,13 +259,12 @@ static void SetAlpha(Service::Interface* self) { static void StartConversion(Service::Interface* self) { u32* cmd_buff = Kernel::GetCommandBuffer(); - HW::Y2R::PerformConversion(conversion); - // dst_image_size would seem to be perfect for this, but it doesn't include the gap :( u32 total_output_size = conversion.input_lines * (conversion.dst.transfer_unit + conversion.dst.gap); - VideoCore::g_renderer->Rasterizer()->InvalidateRegion( - Memory::VirtualToPhysicalAddress(conversion.dst.address), total_output_size); + Memory::RasterizerFlushAndInvalidateRegion(Memory::VirtualToPhysicalAddress(conversion.dst.address), total_output_size); + + HW::Y2R::PerformConversion(conversion); LOG_DEBUG(Service_Y2R, "called"); completion_event->Signal(); diff --git a/src/core/hw/gpu.cpp b/src/core/hw/gpu.cpp index 7e2f9cdfa..2fe856293 100644 --- a/src/core/hw/gpu.cpp +++ b/src/core/hw/gpu.cpp @@ -115,21 +115,39 @@ inline void Write(u32 addr, const T data) { u8* start = Memory::GetPhysicalPointer(config.GetStartAddress()); u8* end = Memory::GetPhysicalPointer(config.GetEndAddress()); - if (config.fill_24bit) { - // fill with 24-bit values - for (u8* ptr = start; ptr < end; ptr += 3) { - ptr[0] = config.value_24bit_r; - ptr[1] = config.value_24bit_g; - ptr[2] = config.value_24bit_b; + // TODO: Consider always accelerating and returning vector of + // regions that the accelerated fill did not cover to + // reduce/eliminate the fill that the cpu has to do. + // This would also mean that the flush below is not needed. + // Fill should first flush all surfaces that touch but are + // not completely within the fill range. + // Then fill all completely covered surfaces, and return the + // regions that were between surfaces or within the touching + // ones for cpu to manually fill here. + if (!VideoCore::g_renderer->Rasterizer()->AccelerateFill(config)) { + Memory::RasterizerFlushAndInvalidateRegion(config.GetStartAddress(), config.GetEndAddress() - config.GetStartAddress()); + + if (config.fill_24bit) { + // fill with 24-bit values + for (u8* ptr = start; ptr < end; ptr += 3) { + ptr[0] = config.value_24bit_r; + ptr[1] = config.value_24bit_g; + ptr[2] = config.value_24bit_b; + } + } else if (config.fill_32bit) { + // fill with 32-bit values + if (end > start) { + u32 value = config.value_32bit; + size_t len = (end - start) / sizeof(u32); + for (size_t i = 0; i < len; ++i) + memcpy(&start[i * sizeof(u32)], &value, sizeof(u32)); + } + } else { + // fill with 16-bit values + u16 value_16bit = config.value_16bit.Value(); + for (u8* ptr = start; ptr < end; ptr += sizeof(u16)) + memcpy(ptr, &value_16bit, sizeof(u16)); } - } else if (config.fill_32bit) { - // fill with 32-bit values - for (u32* ptr = (u32*)start; ptr < (u32*)end; ++ptr) - *ptr = config.value_32bit; - } else { - // fill with 16-bit values - for (u16* ptr = (u16*)start; ptr < (u16*)end; ++ptr) - *ptr = config.value_16bit; } LOG_TRACE(HW_GPU, "MemoryFill from 0x%08x to 0x%08x", config.GetStartAddress(), config.GetEndAddress()); @@ -139,8 +157,6 @@ inline void Write(u32 addr, const T data) { } else { GSP_GPU::SignalInterrupt(GSP_GPU::InterruptId::PSC1); } - - VideoCore::g_renderer->Rasterizer()->InvalidateRegion(config.GetStartAddress(), config.GetEndAddress() - config.GetStartAddress()); } // Reset "trigger" flag and set the "finish" flag @@ -161,184 +177,185 @@ inline void Write(u32 addr, const T data) { if (Pica::g_debug_context) Pica::g_debug_context->OnEvent(Pica::DebugContext::Event::IncomingDisplayTransfer, nullptr); - u8* src_pointer = Memory::GetPhysicalPointer(config.GetPhysicalInputAddress()); - u8* dst_pointer = Memory::GetPhysicalPointer(config.GetPhysicalOutputAddress()); - - if (config.is_texture_copy) { - u32 input_width = config.texture_copy.input_width * 16; - u32 input_gap = config.texture_copy.input_gap * 16; - u32 output_width = config.texture_copy.output_width * 16; - u32 output_gap = config.texture_copy.output_gap * 16; - - size_t contiguous_input_size = config.texture_copy.size / input_width * (input_width + input_gap); - VideoCore::g_renderer->Rasterizer()->FlushRegion(config.GetPhysicalInputAddress(), contiguous_input_size); - - u32 remaining_size = config.texture_copy.size; - u32 remaining_input = input_width; - u32 remaining_output = output_width; - while (remaining_size > 0) { - u32 copy_size = std::min({ remaining_input, remaining_output, remaining_size }); + if (!VideoCore::g_renderer->Rasterizer()->AccelerateDisplayTransfer(config)) { + u8* src_pointer = Memory::GetPhysicalPointer(config.GetPhysicalInputAddress()); + u8* dst_pointer = Memory::GetPhysicalPointer(config.GetPhysicalOutputAddress()); - std::memcpy(dst_pointer, src_pointer, copy_size); - src_pointer += copy_size; - dst_pointer += copy_size; + if (config.is_texture_copy) { + u32 input_width = config.texture_copy.input_width * 16; + u32 input_gap = config.texture_copy.input_gap * 16; + u32 output_width = config.texture_copy.output_width * 16; + u32 output_gap = config.texture_copy.output_gap * 16; - remaining_input -= copy_size; - remaining_output -= copy_size; - remaining_size -= copy_size; + size_t contiguous_input_size = config.texture_copy.size / input_width * (input_width + input_gap); + Memory::RasterizerFlushRegion(config.GetPhysicalInputAddress(), contiguous_input_size); - if (remaining_input == 0) { - remaining_input = input_width; - src_pointer += input_gap; - } - if (remaining_output == 0) { - remaining_output = output_width; - dst_pointer += output_gap; - } - } + size_t contiguous_output_size = config.texture_copy.size / output_width * (output_width + output_gap); + Memory::RasterizerFlushAndInvalidateRegion(config.GetPhysicalOutputAddress(), contiguous_output_size); - LOG_TRACE(HW_GPU, "TextureCopy: 0x%X bytes from 0x%08X(%u+%u)-> 0x%08X(%u+%u), flags 0x%08X", - config.texture_copy.size, - config.GetPhysicalInputAddress(), input_width, input_gap, - config.GetPhysicalOutputAddress(), output_width, output_gap, - config.flags); + u32 remaining_size = config.texture_copy.size; + u32 remaining_input = input_width; + u32 remaining_output = output_width; + while (remaining_size > 0) { + u32 copy_size = std::min({ remaining_input, remaining_output, remaining_size }); - size_t contiguous_output_size = config.texture_copy.size / output_width * (output_width + output_gap); - VideoCore::g_renderer->Rasterizer()->InvalidateRegion(config.GetPhysicalOutputAddress(), contiguous_output_size); + std::memcpy(dst_pointer, src_pointer, copy_size); + src_pointer += copy_size; + dst_pointer += copy_size; - GSP_GPU::SignalInterrupt(GSP_GPU::InterruptId::PPF); - break; - } + remaining_input -= copy_size; + remaining_output -= copy_size; + remaining_size -= copy_size; - if (config.scaling > config.ScaleXY) { - LOG_CRITICAL(HW_GPU, "Unimplemented display transfer scaling mode %u", config.scaling.Value()); - UNIMPLEMENTED(); - break; - } + if (remaining_input == 0) { + remaining_input = input_width; + src_pointer += input_gap; + } + if (remaining_output == 0) { + remaining_output = output_width; + dst_pointer += output_gap; + } + } - if (config.input_linear && config.scaling != config.NoScale) { - LOG_CRITICAL(HW_GPU, "Scaling is only implemented on tiled input"); - UNIMPLEMENTED(); - break; - } + LOG_TRACE(HW_GPU, "TextureCopy: 0x%X bytes from 0x%08X(%u+%u)-> 0x%08X(%u+%u), flags 0x%08X", + config.texture_copy.size, + config.GetPhysicalInputAddress(), input_width, input_gap, + config.GetPhysicalOutputAddress(), output_width, output_gap, + config.flags); - bool horizontal_scale = config.scaling != config.NoScale; - bool vertical_scale = config.scaling == config.ScaleXY; + GSP_GPU::SignalInterrupt(GSP_GPU::InterruptId::PPF); + break; + } - u32 output_width = config.output_width >> horizontal_scale; - u32 output_height = config.output_height >> vertical_scale; + if (config.scaling > config.ScaleXY) { + LOG_CRITICAL(HW_GPU, "Unimplemented display transfer scaling mode %u", config.scaling.Value()); + UNIMPLEMENTED(); + break; + } - u32 input_size = config.input_width * config.input_height * GPU::Regs::BytesPerPixel(config.input_format); - u32 output_size = output_width * output_height * GPU::Regs::BytesPerPixel(config.output_format); + if (config.input_linear && config.scaling != config.NoScale) { + LOG_CRITICAL(HW_GPU, "Scaling is only implemented on tiled input"); + UNIMPLEMENTED(); + break; + } - VideoCore::g_renderer->Rasterizer()->FlushRegion(config.GetPhysicalInputAddress(), input_size); + int horizontal_scale = config.scaling != config.NoScale ? 1 : 0; + int vertical_scale = config.scaling == config.ScaleXY ? 1 : 0; - for (u32 y = 0; y < output_height; ++y) { - for (u32 x = 0; x < output_width; ++x) { - Math::Vec4 src_color; + u32 output_width = config.output_width >> horizontal_scale; + u32 output_height = config.output_height >> vertical_scale; - // Calculate the [x,y] position of the input image - // based on the current output position and the scale - u32 input_x = x << horizontal_scale; - u32 input_y = y << vertical_scale; + u32 input_size = config.input_width * config.input_height * GPU::Regs::BytesPerPixel(config.input_format); + u32 output_size = output_width * output_height * GPU::Regs::BytesPerPixel(config.output_format); - if (config.flip_vertically) { - // Flip the y value of the output data, - // we do this after calculating the [x,y] position of the input image - // to account for the scaling options. - y = output_height - y - 1; - } + Memory::RasterizerFlushRegion(config.GetPhysicalInputAddress(), input_size); + Memory::RasterizerFlushAndInvalidateRegion(config.GetPhysicalOutputAddress(), output_size); - u32 dst_bytes_per_pixel = GPU::Regs::BytesPerPixel(config.output_format); - u32 src_bytes_per_pixel = GPU::Regs::BytesPerPixel(config.input_format); - u32 src_offset; - u32 dst_offset; + for (u32 y = 0; y < output_height; ++y) { + for (u32 x = 0; x < output_width; ++x) { + Math::Vec4 src_color; - if (config.input_linear) { - if (!config.dont_swizzle) { - // Interpret the input as linear and the output as tiled - u32 coarse_y = y & ~7; - u32 stride = output_width * dst_bytes_per_pixel; + // Calculate the [x,y] position of the input image + // based on the current output position and the scale + u32 input_x = x << horizontal_scale; + u32 input_y = y << vertical_scale; - src_offset = (input_x + input_y * config.input_width) * src_bytes_per_pixel; - dst_offset = VideoCore::GetMortonOffset(x, y, dst_bytes_per_pixel) + coarse_y * stride; - } else { - // Both input and output are linear - src_offset = (input_x + input_y * config.input_width) * src_bytes_per_pixel; - dst_offset = (x + y * output_width) * dst_bytes_per_pixel; + if (config.flip_vertically) { + // Flip the y value of the output data, + // we do this after calculating the [x,y] position of the input image + // to account for the scaling options. + y = output_height - y - 1; } - } else { - if (!config.dont_swizzle) { - // Interpret the input as tiled and the output as linear - u32 coarse_y = input_y & ~7; - u32 stride = config.input_width * src_bytes_per_pixel; - src_offset = VideoCore::GetMortonOffset(input_x, input_y, src_bytes_per_pixel) + coarse_y * stride; - dst_offset = (x + y * output_width) * dst_bytes_per_pixel; + u32 dst_bytes_per_pixel = GPU::Regs::BytesPerPixel(config.output_format); + u32 src_bytes_per_pixel = GPU::Regs::BytesPerPixel(config.input_format); + u32 src_offset; + u32 dst_offset; + + if (config.input_linear) { + if (!config.dont_swizzle) { + // Interpret the input as linear and the output as tiled + u32 coarse_y = y & ~7; + u32 stride = output_width * dst_bytes_per_pixel; + + src_offset = (input_x + input_y * config.input_width) * src_bytes_per_pixel; + dst_offset = VideoCore::GetMortonOffset(x, y, dst_bytes_per_pixel) + coarse_y * stride; + } else { + // Both input and output are linear + src_offset = (input_x + input_y * config.input_width) * src_bytes_per_pixel; + dst_offset = (x + y * output_width) * dst_bytes_per_pixel; + } } else { - // Both input and output are tiled - u32 out_coarse_y = y & ~7; - u32 out_stride = output_width * dst_bytes_per_pixel; - - u32 in_coarse_y = input_y & ~7; - u32 in_stride = config.input_width * src_bytes_per_pixel; - - src_offset = VideoCore::GetMortonOffset(input_x, input_y, src_bytes_per_pixel) + in_coarse_y * in_stride; - dst_offset = VideoCore::GetMortonOffset(x, y, dst_bytes_per_pixel) + out_coarse_y * out_stride; + if (!config.dont_swizzle) { + // Interpret the input as tiled and the output as linear + u32 coarse_y = input_y & ~7; + u32 stride = config.input_width * src_bytes_per_pixel; + + src_offset = VideoCore::GetMortonOffset(input_x, input_y, src_bytes_per_pixel) + coarse_y * stride; + dst_offset = (x + y * output_width) * dst_bytes_per_pixel; + } else { + // Both input and output are tiled + u32 out_coarse_y = y & ~7; + u32 out_stride = output_width * dst_bytes_per_pixel; + + u32 in_coarse_y = input_y & ~7; + u32 in_stride = config.input_width * src_bytes_per_pixel; + + src_offset = VideoCore::GetMortonOffset(input_x, input_y, src_bytes_per_pixel) + in_coarse_y * in_stride; + dst_offset = VideoCore::GetMortonOffset(x, y, dst_bytes_per_pixel) + out_coarse_y * out_stride; + } } - } - const u8* src_pixel = src_pointer + src_offset; - src_color = DecodePixel(config.input_format, src_pixel); - if (config.scaling == config.ScaleX) { - Math::Vec4 pixel = DecodePixel(config.input_format, src_pixel + src_bytes_per_pixel); - src_color = ((src_color + pixel) / 2).Cast(); - } else if (config.scaling == config.ScaleXY) { - Math::Vec4 pixel1 = DecodePixel(config.input_format, src_pixel + 1 * src_bytes_per_pixel); - Math::Vec4 pixel2 = DecodePixel(config.input_format, src_pixel + 2 * src_bytes_per_pixel); - Math::Vec4 pixel3 = DecodePixel(config.input_format, src_pixel + 3 * src_bytes_per_pixel); - src_color = (((src_color + pixel1) + (pixel2 + pixel3)) / 4).Cast(); - } + const u8* src_pixel = src_pointer + src_offset; + src_color = DecodePixel(config.input_format, src_pixel); + if (config.scaling == config.ScaleX) { + Math::Vec4 pixel = DecodePixel(config.input_format, src_pixel + src_bytes_per_pixel); + src_color = ((src_color + pixel) / 2).Cast(); + } else if (config.scaling == config.ScaleXY) { + Math::Vec4 pixel1 = DecodePixel(config.input_format, src_pixel + 1 * src_bytes_per_pixel); + Math::Vec4 pixel2 = DecodePixel(config.input_format, src_pixel + 2 * src_bytes_per_pixel); + Math::Vec4 pixel3 = DecodePixel(config.input_format, src_pixel + 3 * src_bytes_per_pixel); + src_color = (((src_color + pixel1) + (pixel2 + pixel3)) / 4).Cast(); + } - u8* dst_pixel = dst_pointer + dst_offset; - switch (config.output_format) { - case Regs::PixelFormat::RGBA8: - Color::EncodeRGBA8(src_color, dst_pixel); - break; + u8* dst_pixel = dst_pointer + dst_offset; + switch (config.output_format) { + case Regs::PixelFormat::RGBA8: + Color::EncodeRGBA8(src_color, dst_pixel); + break; - case Regs::PixelFormat::RGB8: - Color::EncodeRGB8(src_color, dst_pixel); - break; + case Regs::PixelFormat::RGB8: + Color::EncodeRGB8(src_color, dst_pixel); + break; - case Regs::PixelFormat::RGB565: - Color::EncodeRGB565(src_color, dst_pixel); - break; + case Regs::PixelFormat::RGB565: + Color::EncodeRGB565(src_color, dst_pixel); + break; - case Regs::PixelFormat::RGB5A1: - Color::EncodeRGB5A1(src_color, dst_pixel); - break; + case Regs::PixelFormat::RGB5A1: + Color::EncodeRGB5A1(src_color, dst_pixel); + break; - case Regs::PixelFormat::RGBA4: - Color::EncodeRGBA4(src_color, dst_pixel); - break; + case Regs::PixelFormat::RGBA4: + Color::EncodeRGBA4(src_color, dst_pixel); + break; - default: - LOG_ERROR(HW_GPU, "Unknown destination framebuffer format %x", config.output_format.Value()); - break; + default: + LOG_ERROR(HW_GPU, "Unknown destination framebuffer format %x", config.output_format.Value()); + break; + } } } - } - LOG_TRACE(HW_GPU, "DisplayTriggerTransfer: 0x%08x bytes from 0x%08x(%ux%u)-> 0x%08x(%ux%u), dst format %x, flags 0x%08X", + LOG_TRACE(HW_GPU, "DisplayTriggerTransfer: 0x%08x bytes from 0x%08x(%ux%u)-> 0x%08x(%ux%u), dst format %x, flags 0x%08X", config.output_height * output_width * GPU::Regs::BytesPerPixel(config.output_format), config.GetPhysicalInputAddress(), config.input_width.Value(), config.input_height.Value(), config.GetPhysicalOutputAddress(), output_width, output_height, config.output_format.Value(), config.flags); + } g_regs.display_transfer_config.trigger = 0; GSP_GPU::SignalInterrupt(GSP_GPU::InterruptId::PPF); - - VideoCore::g_renderer->Rasterizer()->InvalidateRegion(config.GetPhysicalOutputAddress(), output_size); } break; } diff --git a/src/core/hw/gpu.h b/src/core/hw/gpu.h index a00adbf53..da4c345b4 100644 --- a/src/core/hw/gpu.h +++ b/src/core/hw/gpu.h @@ -78,7 +78,7 @@ struct Regs { INSERT_PADDING_WORDS(0x4); - struct { + struct MemoryFillConfig { u32 address_start; u32 address_end; @@ -165,7 +165,7 @@ struct Regs { INSERT_PADDING_WORDS(0x169); - struct { + struct DisplayTransferConfig { u32 input_address; u32 output_address; diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 7de5bd15d..ee9b69f81 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -15,6 +15,9 @@ #include "core/memory_setup.h" #include "core/mmio.h" +#include "video_core/renderer_base.h" +#include "video_core/video_core.h" + namespace Memory { enum class PageType { @@ -22,8 +25,12 @@ enum class PageType { Unmapped, /// Page is mapped to regular memory. This is the only type you can get pointers to. Memory, + /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and invalidation + RasterizerCachedMemory, /// Page is mapped to a I/O region. Writing and reading to this page is handled by functions. Special, + /// Page is mapped to a I/O region, but also needs to check for rasterizer cache flushing and invalidation + RasterizerCachedSpecial, }; struct SpecialRegion { @@ -57,6 +64,12 @@ struct PageTable { * the corresponding entry in `pointers` MUST be set to null. */ std::array attributes; + + /** + * Indicates the number of externally cached resources touching a page that should be + * flushed before the memory is accessed + */ + std::array cached_res_count; }; /// Singular page table used for the singleton process @@ -72,8 +85,15 @@ static void MapPages(u32 base, u32 size, u8* memory, PageType type) { while (base != end) { ASSERT_MSG(base < PageTable::NUM_ENTRIES, "out of range mapping at %08X", base); + // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be null here + if (current_page_table->attributes[base] == PageType::RasterizerCachedMemory || + current_page_table->attributes[base] == PageType::RasterizerCachedSpecial) { + RasterizerFlushAndInvalidateRegion(VirtualToPhysicalAddress(base << PAGE_BITS), PAGE_SIZE); + } + current_page_table->attributes[base] = type; current_page_table->pointers[base] = memory; + current_page_table->cached_res_count[base] = 0; base += 1; if (memory != nullptr) @@ -84,6 +104,7 @@ static void MapPages(u32 base, u32 size, u8* memory, PageType type) { void InitMemoryMap() { main_page_table.pointers.fill(nullptr); main_page_table.attributes.fill(PageType::Unmapped); + main_page_table.cached_res_count.fill(0); } void MapMemoryRegion(VAddr base, u32 size, u8* target) { @@ -106,6 +127,28 @@ void UnmapRegion(VAddr base, u32 size) { MapPages(base / PAGE_SIZE, size / PAGE_SIZE, nullptr, PageType::Unmapped); } +/** + * Gets a pointer to the exact memory at the virtual address (i.e. not page aligned) + * using a VMA from the current process + */ +static u8* GetPointerFromVMA(VAddr vaddr) { + u8* direct_pointer = nullptr; + + auto& vma = Kernel::g_current_process->vm_manager.FindVMA(vaddr)->second; + switch (vma.type) { + case Kernel::VMAType::AllocatedMemoryBlock: + direct_pointer = vma.backing_block->data() + vma.offset; + break; + case Kernel::VMAType::BackingMemory: + direct_pointer = vma.backing_memory; + break; + default: + UNREACHABLE(); + } + + return direct_pointer + (vaddr - vma.base); +} + /** * This function should only be called for virtual addreses with attribute `PageType::Special`. */ @@ -126,6 +169,7 @@ template T Read(const VAddr vaddr) { const u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; if (page_pointer) { + // NOTE: Avoid adding any extra logic to this fast-path block T value; std::memcpy(&value, &page_pointer[vaddr & PAGE_MASK], sizeof(T)); return value; @@ -139,8 +183,22 @@ T Read(const VAddr vaddr) { case PageType::Memory: ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr); break; + case PageType::RasterizerCachedMemory: + { + RasterizerFlushRegion(VirtualToPhysicalAddress(vaddr), sizeof(T)); + + T value; + std::memcpy(&value, GetPointerFromVMA(vaddr), sizeof(T)); + return value; + } case PageType::Special: return ReadMMIO(GetMMIOHandler(vaddr), vaddr); + case PageType::RasterizerCachedSpecial: + { + RasterizerFlushRegion(VirtualToPhysicalAddress(vaddr), sizeof(T)); + + return ReadMMIO(GetMMIOHandler(vaddr), vaddr); + } default: UNREACHABLE(); } @@ -153,6 +211,7 @@ template void Write(const VAddr vaddr, const T data) { u8* page_pointer = current_page_table->pointers[vaddr >> PAGE_BITS]; if (page_pointer) { + // NOTE: Avoid adding any extra logic to this fast-path block std::memcpy(&page_pointer[vaddr & PAGE_MASK], &data, sizeof(T)); return; } @@ -165,9 +224,23 @@ void Write(const VAddr vaddr, const T data) { case PageType::Memory: ASSERT_MSG(false, "Mapped memory page without a pointer @ %08X", vaddr); break; + case PageType::RasterizerCachedMemory: + { + RasterizerFlushAndInvalidateRegion(VirtualToPhysicalAddress(vaddr), sizeof(T)); + + std::memcpy(GetPointerFromVMA(vaddr), &data, sizeof(T)); + break; + } case PageType::Special: WriteMMIO(GetMMIOHandler(vaddr), vaddr, data); break; + case PageType::RasterizerCachedSpecial: + { + RasterizerFlushAndInvalidateRegion(VirtualToPhysicalAddress(vaddr), sizeof(T)); + + WriteMMIO(GetMMIOHandler(vaddr), vaddr, data); + break; + } default: UNREACHABLE(); } @@ -179,6 +252,10 @@ u8* GetPointer(const VAddr vaddr) { return page_pointer + (vaddr & PAGE_MASK); } + if (current_page_table->attributes[vaddr >> PAGE_BITS] == PageType::RasterizerCachedMemory) { + return GetPointerFromVMA(vaddr); + } + LOG_ERROR(HW_Memory, "unknown GetPointer @ 0x%08x", vaddr); return nullptr; } @@ -187,6 +264,69 @@ u8* GetPhysicalPointer(PAddr address) { return GetPointer(PhysicalToVirtualAddress(address)); } +void RasterizerMarkRegionCached(PAddr start, u32 size, int count_delta) { + if (start == 0) { + return; + } + + u32 num_pages = ((start + size - 1) >> PAGE_BITS) - (start >> PAGE_BITS) + 1; + PAddr paddr = start; + + for (unsigned i = 0; i < num_pages; ++i) { + VAddr vaddr = PhysicalToVirtualAddress(paddr); + u8& res_count = current_page_table->cached_res_count[vaddr >> PAGE_BITS]; + ASSERT_MSG(count_delta <= UINT8_MAX - res_count, "Rasterizer resource cache counter overflow!"); + ASSERT_MSG(count_delta >= -res_count, "Rasterizer resource cache counter underflow!"); + + // Switch page type to cached if now cached + if (res_count == 0) { + PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; + switch (page_type) { + case PageType::Memory: + page_type = PageType::RasterizerCachedMemory; + current_page_table->pointers[vaddr >> PAGE_BITS] = nullptr; + break; + case PageType::Special: + page_type = PageType::RasterizerCachedSpecial; + break; + default: + UNREACHABLE(); + } + } + + res_count += count_delta; + + // Switch page type to uncached if now uncached + if (res_count == 0) { + PageType& page_type = current_page_table->attributes[vaddr >> PAGE_BITS]; + switch (page_type) { + case PageType::RasterizerCachedMemory: + page_type = PageType::Memory; + current_page_table->pointers[vaddr >> PAGE_BITS] = GetPointerFromVMA(vaddr & ~PAGE_MASK); + break; + case PageType::RasterizerCachedSpecial: + page_type = PageType::Special; + break; + default: + UNREACHABLE(); + } + } + paddr += PAGE_SIZE; + } +} + +void RasterizerFlushRegion(PAddr start, u32 size) { + if (VideoCore::g_renderer != nullptr) { + VideoCore::g_renderer->Rasterizer()->FlushRegion(start, size); + } +} + +void RasterizerFlushAndInvalidateRegion(PAddr start, u32 size) { + if (VideoCore::g_renderer != nullptr) { + VideoCore::g_renderer->Rasterizer()->FlushAndInvalidateRegion(start, size); + } +} + u8 Read8(const VAddr addr) { return Read(addr); } diff --git a/src/core/memory.h b/src/core/memory.h index 5af72b7a7..9caa3c3f5 100644 --- a/src/core/memory.h +++ b/src/core/memory.h @@ -148,4 +148,20 @@ VAddr PhysicalToVirtualAddress(PAddr addr); */ u8* GetPhysicalPointer(PAddr address); +/** + * Adds the supplied value to the rasterizer resource cache counter of each + * page touching the region. + */ +void RasterizerMarkRegionCached(PAddr start, u32 size, int count_delta); + +/** + * Flushes any externally cached rasterizer resources touching the given region. + */ +void RasterizerFlushRegion(PAddr start, u32 size); + +/** + * Flushes and invalidates any externally cached rasterizer resources touching the given region. + */ +void RasterizerFlushAndInvalidateRegion(PAddr start, u32 size); + } -- cgit v1.2.3