diff options
Diffstat (limited to '')
-rw-r--r-- | src/video_core/texture_cache/image_base.h | 3 | ||||
-rw-r--r-- | src/video_core/texture_cache/image_info.cpp | 45 | ||||
-rw-r--r-- | src/video_core/texture_cache/image_info.h | 2 | ||||
-rw-r--r-- | src/video_core/texture_cache/texture_cache.h | 139 | ||||
-rw-r--r-- | src/video_core/texture_cache/texture_cache_base.h | 21 | ||||
-rw-r--r-- | src/video_core/texture_cache/types.h | 1 | ||||
-rw-r--r-- | src/video_core/texture_cache/util.cpp | 98 | ||||
-rw-r--r-- | src/video_core/texture_cache/util.h | 10 |
8 files changed, 307 insertions, 12 deletions
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 620565684..e8fa592d2 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -38,6 +38,9 @@ enum class ImageFlagBits : u32 { Rescaled = 1 << 13, CheckingRescalable = 1 << 14, IsRescalable = 1 << 15, + + AsynchronousDecode = 1 << 16, + IsDecoding = 1 << 17, ///< Is currently being decoded asynchornously. }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) diff --git a/src/video_core/texture_cache/image_info.cpp b/src/video_core/texture_cache/image_info.cpp index e9100091e..a1296b574 100644 --- a/src/video_core/texture_cache/image_info.cpp +++ b/src/video_core/texture_cache/image_info.cpp @@ -216,10 +216,51 @@ ImageInfo::ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept { .height = config.height, .depth = 1, }; - rescaleable = block.depth == 0; - rescaleable &= size.height > 256; + rescaleable = block.depth == 0 && size.height > 256; downscaleable = size.height > 512; } } +static PixelFormat ByteSizeToFormat(u32 bytes_per_pixel) { + switch (bytes_per_pixel) { + case 1: + return PixelFormat::R8_UINT; + case 2: + return PixelFormat::R8G8_UINT; + case 4: + return PixelFormat::A8B8G8R8_UINT; + case 8: + return PixelFormat::R16G16B16A16_UINT; + case 16: + return PixelFormat::R32G32B32A32_UINT; + default: + UNIMPLEMENTED(); + return PixelFormat::Invalid; + } +} + +ImageInfo::ImageInfo(const Tegra::DMA::ImageOperand& config) noexcept { + const u32 bytes_per_pixel = config.bytes_per_pixel; + format = ByteSizeToFormat(bytes_per_pixel); + type = config.params.block_size.depth > 0 ? ImageType::e3D : ImageType::e2D; + num_samples = 1; + block = Extent3D{ + .width = config.params.block_size.width, + .height = config.params.block_size.height, + .depth = config.params.block_size.depth, + }; + size = Extent3D{ + .width = config.params.width, + .height = config.params.height, + .depth = config.params.depth, + }; + tile_width_spacing = 0; + resources.levels = 1; + resources.layers = 1; + layer_stride = CalculateLayerStride(*this); + maybe_unaligned_layer_stride = CalculateLayerSize(*this); + rescaleable = block.depth == 0 && size.height > 256; + downscaleable = size.height > 512; +} + } // namespace VideoCommon diff --git a/src/video_core/texture_cache/image_info.h b/src/video_core/texture_cache/image_info.h index 93755e15e..a12f5b44f 100644 --- a/src/video_core/texture_cache/image_info.h +++ b/src/video_core/texture_cache/image_info.h @@ -5,6 +5,7 @@ #include "video_core/engines/fermi_2d.h" #include "video_core/engines/maxwell_3d.h" +#include "video_core/engines/maxwell_dma.h" #include "video_core/surface.h" #include "video_core/texture_cache/types.h" @@ -19,6 +20,7 @@ struct ImageInfo { explicit ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs, size_t index) noexcept; explicit ImageInfo(const Tegra::Engines::Maxwell3D::Regs& regs) noexcept; explicit ImageInfo(const Tegra::Engines::Fermi2D::Surface& config) noexcept; + explicit ImageInfo(const Tegra::DMA::ImageOperand& config) noexcept; PixelFormat format = PixelFormat::Invalid; ImageType type = ImageType::e1D; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 3e2cbb0b0..335338434 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -85,6 +85,11 @@ void TextureCache<P>::RunGarbageCollector() { } --num_iterations; auto& image = slot_images[image_id]; + if (True(image.flags & ImageFlagBits::IsDecoding)) { + // This image is still being decoded, deleting it will invalidate the slot + // used by the async decoder thread. + return false; + } const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); if (!high_priority_mode && @@ -133,6 +138,8 @@ void TextureCache<P>::TickFrame() { sentenced_images.Tick(); sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); + TickAsyncDecode(); + runtime.TickFrame(); critical_gc = 0; ++frame_tick; @@ -777,6 +784,10 @@ void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) { LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented"); return; } + if (True(image.flags & ImageFlagBits::AsynchronousDecode)) { + QueueAsyncDecode(image, image_id); + return; + } auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); @@ -990,6 +1001,65 @@ u64 TextureCache<P>::GetScaledImageSizeBytes(const ImageBase& image) { } template <class P> +void TextureCache<P>::QueueAsyncDecode(Image& image, ImageId image_id) { + UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted)); + LOG_INFO(HW_GPU, "Queuing async texture decode"); + + image.flags |= ImageFlagBits::IsDecoding; + auto decode = std::make_unique<AsyncDecodeContext>(); + auto* decode_ptr = decode.get(); + decode->image_id = image_id; + async_decodes.push_back(std::move(decode)); + + Common::ScratchBuffer<u8> local_unswizzle_data_buffer(image.unswizzled_size_bytes); + const size_t guest_size_bytes = image.guest_size_bytes; + swizzle_data_buffer.resize_destructive(guest_size_bytes); + gpu_memory->ReadBlockUnsafe(image.gpu_addr, swizzle_data_buffer.data(), guest_size_bytes); + auto copies = UnswizzleImage(*gpu_memory, image.gpu_addr, image.info, swizzle_data_buffer, + local_unswizzle_data_buffer); + const size_t out_size = MapSizeBytes(image); + + auto func = [out_size, copies, info = image.info, + input = std::move(local_unswizzle_data_buffer), + async_decode = decode_ptr]() mutable { + async_decode->decoded_data.resize_destructive(out_size); + std::span copies_span{copies.data(), copies.size()}; + ConvertImage(input, info, async_decode->decoded_data, copies_span); + + // TODO: Do we need this lock? + std::unique_lock lock{async_decode->mutex}; + async_decode->copies = std::move(copies); + async_decode->complete = true; + }; + texture_decode_worker.QueueWork(std::move(func)); +} + +template <class P> +void TextureCache<P>::TickAsyncDecode() { + bool has_uploads{}; + auto i = async_decodes.begin(); + while (i != async_decodes.end()) { + auto* async_decode = i->get(); + std::unique_lock lock{async_decode->mutex}; + if (!async_decode->complete) { + ++i; + continue; + } + Image& image = slot_images[async_decode->image_id]; + auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); + std::memcpy(staging.mapped_span.data(), async_decode->decoded_data.data(), + async_decode->decoded_data.size()); + image.UploadMemory(staging, async_decode->copies); + image.flags &= ~ImageFlagBits::IsDecoding; + has_uploads = true; + i = async_decodes.erase(i); + } + if (has_uploads) { + runtime.InsertUploadMemoryBarrier(); + } +} + +template <class P> bool TextureCache<P>::ScaleUp(Image& image) { const bool has_copy = image.HasScaled(); const bool rescaled = image.ScaleUp(); @@ -1289,6 +1359,75 @@ std::optional<typename TextureCache<P>::BlitImages> TextureCache<P>::GetBlitImag } template <class P> +ImageId TextureCache<P>::FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr) { + std::optional<VAddr> cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr); + if (!cpu_addr) { + cpu_addr = gpu_memory->GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info)); + if (!cpu_addr) { + return ImageId{}; + } + } + ImageId image_id{}; + boost::container::small_vector<ImageId, 1> image_ids; + const auto lambda = [&](ImageId existing_image_id, ImageBase& existing_image) { + if (True(existing_image.flags & ImageFlagBits::Remapped)) { + return false; + } + if (info.type == ImageType::Linear || existing_image.info.type == ImageType::Linear) + [[unlikely]] { + const bool strict_size = True(existing_image.flags & ImageFlagBits::Strong); + const ImageInfo& existing = existing_image.info; + if (existing_image.gpu_addr == gpu_addr && existing.type == info.type && + existing.pitch == info.pitch && + IsPitchLinearSameSize(existing, info, strict_size) && + IsViewCompatible(existing.format, info.format, false, true)) { + image_id = existing_image_id; + image_ids.push_back(existing_image_id); + return true; + } + } else if (IsSubCopy(info, existing_image, gpu_addr)) { + image_id = existing_image_id; + image_ids.push_back(existing_image_id); + return true; + } + return false; + }; + ForEachImageInRegion(*cpu_addr, CalculateGuestSizeInBytes(info), lambda); + if (image_ids.size() <= 1) [[likely]] { + return image_id; + } + auto image_ids_compare = [this](ImageId a, ImageId b) { + auto& image_a = slot_images[a]; + auto& image_b = slot_images[b]; + return image_a.modification_tick < image_b.modification_tick; + }; + return *std::ranges::max_element(image_ids, image_ids_compare); +} + +template <class P> +std::optional<std::pair<typename TextureCache<P>::Image*, std::pair<u32, u32>>> +TextureCache<P>::ObtainImage(const Tegra::DMA::ImageOperand& operand, bool mark_as_modified) { + ImageInfo dst_info(operand); + ImageId dst_id = FindDMAImage(dst_info, operand.address); + if (!dst_id) { + return std::nullopt; + } + auto& image = slot_images[dst_id]; + auto base = image.TryFindBase(operand.address); + if (!base) { + return std::nullopt; + } + if (False(image.flags & ImageFlagBits::GpuModified)) { + // No need to waste time on an image that's synced with guest + return std::nullopt; + } + PrepareImage(dst_id, mark_as_modified, false); + auto& new_image = slot_images[dst_id]; + lru_cache.Touch(new_image.lru_index, frame_tick); + return std::make_pair(&new_image, std::make_pair(base->level, base->layer)); +} + +template <class P> SamplerId TextureCache<P>::FindSampler(const TSCEntry& config) { if (std::ranges::all_of(config.raw, [](u64 value) { return value == 0; })) { return NULL_SAMPLER_ID; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 485eaabaa..848a5d9ea 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -3,6 +3,7 @@ #pragma once +#include <atomic> #include <deque> #include <limits> #include <mutex> @@ -18,6 +19,7 @@ #include "common/lru_cache.h" #include "common/polyfill_ranges.h" #include "common/scratch_buffer.h" +#include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" #include "video_core/delayed_destruction_ring.h" @@ -54,6 +56,14 @@ struct ImageViewInOut { ImageViewId id{}; }; +struct AsyncDecodeContext { + ImageId image_id; + Common::ScratchBuffer<u8> decoded_data; + std::vector<BufferImageCopy> copies; + std::mutex mutex; + std::atomic_bool complete; +}; + using TextureCacheGPUMap = std::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>; class TextureCacheChannelInfo : public ChannelInfo { @@ -199,6 +209,9 @@ public: /// Pop asynchronous downloads void PopAsyncFlushes(); + [[nodiscard]] std::optional<std::pair<Image*, std::pair<u32, u32>>> ObtainImage( + const Tegra::DMA::ImageOperand& operand, bool mark_as_modified); + /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size); @@ -290,6 +303,8 @@ private: /// Remove joined images from the cache [[nodiscard]] ImageId JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VAddr cpu_addr); + [[nodiscard]] ImageId FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr); + /// Return a blit image pair from the given guest blit parameters [[nodiscard]] std::optional<BlitImages> GetBlitImages( const Tegra::Engines::Fermi2D::Surface& dst, const Tegra::Engines::Fermi2D::Surface& src, @@ -377,6 +392,9 @@ private: bool ScaleDown(Image& image); u64 GetScaledImageSizeBytes(const ImageBase& image); + void QueueAsyncDecode(Image& image, ImageId image_id); + void TickAsyncDecode(); + Runtime& runtime; VideoCore::RasterizerInterface& rasterizer; @@ -430,6 +448,9 @@ private: u64 modification_tick = 0; u64 frame_tick = 0; + + Common::ThreadWorker texture_decode_worker{1, "TextureDecoder"}; + std::vector<std::unique_ptr<AsyncDecodeContext>> async_decodes; }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h index 0453456b4..a0e10643f 100644 --- a/src/video_core/texture_cache/types.h +++ b/src/video_core/texture_cache/types.h @@ -54,6 +54,7 @@ enum class RelaxedOptions : u32 { Format = 1 << 1, Samples = 1 << 2, ForceBrokenViews = 1 << 3, + FormatBpp = 1 << 4, }; DECLARE_ENUM_FLAG_OPERATORS(RelaxedOptions) diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index 697f86641..de37db684 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -743,6 +743,44 @@ std::vector<ImageCopy> MakeShrinkImageCopies(const ImageInfo& dst, const ImageIn return copies; } +std::vector<ImageCopy> MakeReinterpretImageCopies(const ImageInfo& src, u32 up_scale, + u32 down_shift) { + std::vector<ImageCopy> copies; + copies.reserve(src.resources.levels); + const bool is_3d = src.type == ImageType::e3D; + for (s32 level = 0; level < src.resources.levels; ++level) { + ImageCopy& copy = copies.emplace_back(); + copy.src_subresource = SubresourceLayers{ + .base_level = level, + .base_layer = 0, + .num_layers = src.resources.layers, + }; + copy.dst_subresource = SubresourceLayers{ + .base_level = level, + .base_layer = 0, + .num_layers = src.resources.layers, + }; + copy.src_offset = Offset3D{ + .x = 0, + .y = 0, + .z = 0, + }; + copy.dst_offset = Offset3D{ + .x = 0, + .y = 0, + .z = 0, + }; + const Extent3D mip_size = AdjustMipSize(src.size, level); + copy.extent = AdjustSamplesSize(mip_size, src.num_samples); + if (is_3d) { + copy.extent.depth = src.size.depth; + } + copy.extent.width = std::max<u32>((copy.extent.width * up_scale) >> down_shift, 1); + copy.extent.height = std::max<u32>((copy.extent.height * up_scale) >> down_shift, 1); + } + return copies; +} + bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config) { const GPUVAddr address = config.Address(); if (address == 0) { @@ -999,6 +1037,20 @@ bool IsBlockLinearSizeCompatible(const ImageInfo& lhs, const ImageInfo& rhs, u32 } } +bool IsBlockLinearSizeCompatibleBPPRelaxed(const ImageInfo& lhs, const ImageInfo& rhs, + u32 lhs_level, u32 rhs_level) noexcept { + ASSERT(lhs.type != ImageType::Linear); + ASSERT(rhs.type != ImageType::Linear); + const auto lhs_bpp = BytesPerBlock(lhs.format); + const auto rhs_bpp = BytesPerBlock(rhs.format); + const Extent3D lhs_size = AdjustMipSize(lhs.size, lhs_level); + const Extent3D rhs_size = AdjustMipSize(rhs.size, rhs_level); + return Common::AlignUpLog2(lhs_size.width * lhs_bpp, GOB_SIZE_X_SHIFT) == + Common::AlignUpLog2(rhs_size.width * rhs_bpp, GOB_SIZE_X_SHIFT) && + Common::AlignUpLog2(lhs_size.height, GOB_SIZE_Y_SHIFT) == + Common::AlignUpLog2(rhs_size.height, GOB_SIZE_Y_SHIFT); +} + bool IsPitchLinearSameSize(const ImageInfo& lhs, const ImageInfo& rhs, bool strict_size) noexcept { ASSERT(lhs.type == ImageType::Linear); ASSERT(rhs.type == ImageType::Linear); @@ -1073,7 +1125,8 @@ std::optional<SubresourceBase> FindSubresource(const ImageInfo& candidate, const // Format checking is relaxed, but we still have to check for matching bytes per block. // This avoids creating a view for blits on UE4 titles where formats with different bytes // per block are aliased. - if (BytesPerBlock(existing.format) != BytesPerBlock(candidate.format)) { + if (BytesPerBlock(existing.format) != BytesPerBlock(candidate.format) && + False(options & RelaxedOptions::FormatBpp)) { return std::nullopt; } } else { @@ -1088,10 +1141,8 @@ std::optional<SubresourceBase> FindSubresource(const ImageInfo& candidate, const if (existing.type != candidate.type) { return std::nullopt; } - if (False(options & RelaxedOptions::Samples)) { - if (existing.num_samples != candidate.num_samples) { - return std::nullopt; - } + if (False(options & RelaxedOptions::Samples) && existing.num_samples != candidate.num_samples) { + return std::nullopt; } if (existing.resources.levels < candidate.resources.levels + base->level) { return std::nullopt; @@ -1101,14 +1152,16 @@ std::optional<SubresourceBase> FindSubresource(const ImageInfo& candidate, const if (mip_depth < candidate.size.depth + base->layer) { return std::nullopt; } - } else { - if (existing.resources.layers < candidate.resources.layers + base->layer) { - return std::nullopt; - } + } else if (existing.resources.layers < candidate.resources.layers + base->layer) { + return std::nullopt; } const bool strict_size = False(options & RelaxedOptions::Size); if (!IsBlockLinearSizeCompatible(existing, candidate, base->level, 0, strict_size)) { - return std::nullopt; + if (False(options & RelaxedOptions::FormatBpp)) { + return std::nullopt; + } else if (!IsBlockLinearSizeCompatibleBPPRelaxed(existing, candidate, base->level, 0)) { + return std::nullopt; + } } // TODO: compare block sizes return base; @@ -1120,6 +1173,31 @@ bool IsSubresource(const ImageInfo& candidate, const ImageBase& image, GPUVAddr .has_value(); } +bool IsSubCopy(const ImageInfo& candidate, const ImageBase& image, GPUVAddr candidate_addr) { + const std::optional<SubresourceBase> base = image.TryFindBase(candidate_addr); + if (!base) { + return false; + } + const ImageInfo& existing = image.info; + if (existing.resources.levels < candidate.resources.levels + base->level) { + return false; + } + if (existing.type == ImageType::e3D) { + const u32 mip_depth = std::max(1U, existing.size.depth << base->level); + if (mip_depth < candidate.size.depth + base->layer) { + return false; + } + } else { + if (existing.resources.layers < candidate.resources.layers + base->layer) { + return false; + } + } + if (!IsBlockLinearSizeCompatibleBPPRelaxed(existing, candidate, base->level, 0)) { + return false; + } + return true; +} + void DeduceBlitImages(ImageInfo& dst_info, ImageInfo& src_info, const ImageBase* dst, const ImageBase* src) { const auto original_dst_format = dst_info.format; diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index d103db8ae..84aa6880d 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -56,6 +56,10 @@ struct OverlapResult { SubresourceBase base, u32 up_scale = 1, u32 down_shift = 0); +[[nodiscard]] std::vector<ImageCopy> MakeReinterpretImageCopies(const ImageInfo& src, + u32 up_scale = 1, + u32 down_shift = 0); + [[nodiscard]] bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config); [[nodiscard]] std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory, @@ -88,6 +92,9 @@ void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const Ima [[nodiscard]] bool IsPitchLinearSameSize(const ImageInfo& lhs, const ImageInfo& rhs, bool strict_size) noexcept; +[[nodiscard]] bool IsBlockLinearSizeCompatibleBPPRelaxed(const ImageInfo& lhs, const ImageInfo& rhs, + u32 lhs_level, u32 rhs_level) noexcept; + [[nodiscard]] std::optional<OverlapResult> ResolveOverlap(const ImageInfo& new_info, GPUVAddr gpu_addr, VAddr cpu_addr, const ImageBase& overlap, @@ -106,6 +113,9 @@ void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const Ima GPUVAddr candidate_addr, RelaxedOptions options, bool broken_views, bool native_bgr); +[[nodiscard]] bool IsSubCopy(const ImageInfo& candidate, const ImageBase& image, + GPUVAddr candidate_addr); + void DeduceBlitImages(ImageInfo& dst_info, ImageInfo& src_info, const ImageBase* dst, const ImageBase* src); |