From 38165fb7e3e486b5099cfa76f5a09ec9f3201acd Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Sat, 12 Jun 2021 15:52:27 +0200
Subject: Texture Cache: Initial Implementation of Sparse Textures.

---
 src/video_core/texture_cache/image_base.cpp  |   3 +
 src/video_core/texture_cache/image_base.h    |   8 +
 src/video_core/texture_cache/texture_cache.h | 248 +++++++++++++++++++++++++--
 src/video_core/texture_cache/types.h         |   1 +
 src/video_core/texture_cache/util.cpp        |  41 ++++-
 src/video_core/texture_cache/util.h          |   6 +-
 6 files changed, 285 insertions(+), 22 deletions(-)

(limited to 'src/video_core/texture_cache')

diff --git a/src/video_core/texture_cache/image_base.cpp b/src/video_core/texture_cache/image_base.cpp
index ad69d32d1..2aae338b6 100644
--- a/src/video_core/texture_cache/image_base.cpp
+++ b/src/video_core/texture_cache/image_base.cpp
@@ -69,6 +69,9 @@ ImageBase::ImageBase(const ImageInfo& info_, GPUVAddr gpu_addr_, VAddr cpu_addr_
     }
 }
 
+ImageMapView::ImageMapView(GPUVAddr gpu_addr_, VAddr cpu_addr_, size_t size_, ImageId image_id_)
+    : gpu_addr{gpu_addr_}, cpu_addr{cpu_addr_}, size{size_}, image_id{image_id_} {}
+
 std::optional<SubresourceBase> ImageBase::TryFindBase(GPUVAddr other_addr) const noexcept {
     if (other_addr < gpu_addr) {
         // Subresource address can't be lower than the base
diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index e326cab71..004ec23e4 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -57,6 +57,12 @@ struct ImageBase {
         return cpu_addr < overlap_end && overlap_cpu_addr < cpu_addr_end;
     }
 
+    [[nodiscard]] bool OverlapsGPU(GPUVAddr overlap_gpu_addr, size_t overlap_size) const noexcept {
+        const VAddr overlap_end = overlap_gpu_addr + overlap_size;
+        const GPUVAddr gpu_addr_end = gpu_addr + guest_size_bytes;
+        return gpu_addr < overlap_end && overlap_gpu_addr < gpu_addr_end;
+    }
+
     void CheckBadOverlapState();
     void CheckAliasState();
 
@@ -84,6 +90,8 @@ struct ImageBase {
 
     std::vector<AliasedImage> aliased_images;
     std::vector<ImageId> overlapping_images;
+    ImageMapId map_view_id{};
+    bool is_sparse{};
 };
 
 struct ImageAllocBase {
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index d8dbd3824..9f6410d58 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -152,6 +152,9 @@ public:
     /// Remove images in a region
     void UnmapMemory(VAddr cpu_addr, size_t size);
 
+    /// Remove images in a region
+    void UnmapGPUMemory(GPUVAddr gpu_addr, size_t size);
+
     /// Blit an image with the given parameters
     void BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
                    const Tegra::Engines::Fermi2D::Surface& src,
@@ -190,7 +193,22 @@ public:
 private:
     /// Iterate over all page indices in a range
     template <typename Func>
-    static void ForEachPage(VAddr addr, size_t size, Func&& func) {
+    static void ForEachCPUPage(VAddr addr, size_t size, Func&& func) {
+        static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
+        const u64 page_end = (addr + size - 1) >> PAGE_BITS;
+        for (u64 page = addr >> PAGE_BITS; page <= page_end; ++page) {
+            if constexpr (RETURNS_BOOL) {
+                if (func(page)) {
+                    break;
+                }
+            } else {
+                func(page);
+            }
+        }
+    }
+
+    template <typename Func>
+    static void ForEachGPUPage(GPUVAddr addr, size_t size, Func&& func) {
         static constexpr bool RETURNS_BOOL = std::is_same_v<std::invoke_result<Func, u64>, bool>;
         const u64 page_end = (addr + size - 1) >> PAGE_BITS;
         for (u64 page = addr >> PAGE_BITS; page <= page_end; ++page) {
@@ -269,6 +287,13 @@ private:
     template <typename Func>
     void ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& func);
 
+    template <typename Func>
+    void ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Func&& func);
+
+    /// Iterates over all the images in a region calling func
+    template <typename Func>
+    void ForEachSparseSegment(ImageBase& image, Func&& func);
+
     /// Find or create an image view in the given image with the passed parameters
     [[nodiscard]] ImageViewId FindOrEmplaceImageView(ImageId image_id, const ImageViewInfo& info);
 
@@ -340,7 +365,8 @@ private:
     std::unordered_map<TSCEntry, SamplerId> samplers;
     std::unordered_map<RenderTargets, FramebufferId> framebuffers;
 
-    std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> page_table;
+    std::unordered_map<u64, std::vector<ImageMapId>, IdentityHash<u64>> page_table;
+    std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> gpu_page_table;
 
     bool has_deleted_images = false;
     u64 total_used_memory = 0;
@@ -349,6 +375,7 @@ private:
     u64 critical_memory;
 
     SlotVector<Image> slot_images;
+    SlotVector<ImageMapView> slot_map_views;
     SlotVector<ImageView> slot_image_views;
     SlotVector<ImageAlloc> slot_image_allocs;
     SlotVector<Sampler> slot_samplers;
@@ -702,6 +729,21 @@ void TextureCache<P>::UnmapMemory(VAddr cpu_addr, size_t size) {
     }
 }
 
+template <class P>
+void TextureCache<P>::UnmapGPUMemory(GPUVAddr gpu_addr, size_t size) {
+    std::vector<ImageId> deleted_images;
+    ForEachImageInRegionGPU(gpu_addr, size,
+                            [&](ImageId id, Image&) { deleted_images.push_back(id); });
+    for (const ImageId id : deleted_images) {
+        Image& image = slot_images[id];
+        if (True(image.flags & ImageFlagBits::Tracked)) {
+            UntrackImage(image);
+        }
+        UnregisterImage(id);
+        DeleteImage(id);
+    }
+}
+
 template <class P>
 void TextureCache<P>::BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
                                 const Tegra::Engines::Fermi2D::Surface& src,
@@ -833,9 +875,10 @@ typename P::ImageView* TextureCache<P>::TryFindFramebufferImageView(VAddr cpu_ad
     if (it == page_table.end()) {
         return nullptr;
     }
-    const auto& image_ids = it->second;
-    for (const ImageId image_id : image_ids) {
-        const ImageBase& image = slot_images[image_id];
+    const auto& image_map_ids = it->second;
+    for (const ImageMapId map_id : image_map_ids) {
+        const ImageMapView& map = slot_map_views[map_id];
+        const ImageBase& image = slot_images[map.image_id];
         if (image.cpu_addr != cpu_addr) {
             continue;
         }
@@ -958,7 +1001,7 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
 
 template <class P>
 ImageViewId TextureCache<P>::FindImageView(const TICEntry& config) {
-    if (!IsValidAddress(gpu_memory, config)) {
+    if (!IsValidEntry(gpu_memory, config)) {
         return NULL_IMAGE_VIEW_ID;
     }
     const auto [pair, is_new] = image_views.try_emplace(config);
@@ -1026,7 +1069,7 @@ ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
         }
         return false;
     };
-    ForEachImageInRegion(*cpu_addr, CalculateGuestSizeInBytes(info), lambda);
+    ForEachImageInRegionGPU(gpu_addr, CalculateGuestSizeInBytes(info), lambda);
     return image_id;
 }
 
@@ -1056,7 +1099,7 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     std::vector<ImageId> left_aliased_ids;
     std::vector<ImageId> right_aliased_ids;
     std::vector<ImageId> bad_overlap_ids;
-    ForEachImageInRegion(cpu_addr, size_bytes, [&](ImageId overlap_id, ImageBase& overlap) {
+    ForEachImageInRegionGPU(gpu_addr, size_bytes, [&](ImageId overlap_id, ImageBase& overlap) {
         if (info.type == ImageType::Linear) {
             if (info.pitch == overlap.info.pitch && gpu_addr == overlap.gpu_addr) {
                 // Alias linear images with the same pitch
@@ -1091,6 +1134,24 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr);
     Image& new_image = slot_images[new_image_id];
 
+    new_image.is_sparse = false;
+    if (new_image.info.type != ImageType::Linear && new_image.info.type != ImageType::Buffer) {
+        const LevelArray offsets = CalculateMipLevelOffsets(new_image.info);
+        size_t level;
+        const size_t levels = static_cast<size_t>(new_image.info.resources.levels);
+        VAddr n_cpu_addr = new_image.cpu_addr;
+        GPUVAddr n_gpu_addr = new_image.gpu_addr;
+        for (level = 0; level < levels; level++) {
+            n_gpu_addr += offsets[level];
+            n_cpu_addr += offsets[level];
+            std::optional<VAddr> cpu_addr_opt = gpu_memory.GpuToCpuAddress(n_gpu_addr);
+            if (!cpu_addr_opt || *cpu_addr_opt == 0 || n_cpu_addr != *cpu_addr_opt) {
+                new_image.is_sparse = true;
+                break;
+            }
+        }
+    }
+
     // TODO: Only upload what we need
     RefreshContents(new_image);
 
@@ -1239,7 +1300,8 @@ void TextureCache<P>::ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& f
     using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
     static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
     boost::container::small_vector<ImageId, 32> images;
-    ForEachPage(cpu_addr, size, [this, &images, cpu_addr, size, func](u64 page) {
+    boost::container::small_vector<ImageMapId, 32> maps;
+    ForEachCPUPage(cpu_addr, size, [this, &images, &maps, cpu_addr, size, func](u64 page) {
         const auto it = page_table.find(page);
         if (it == page_table.end()) {
             if constexpr (BOOL_BREAK) {
@@ -1248,12 +1310,63 @@ void TextureCache<P>::ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& f
                 return;
             }
         }
+        for (const ImageMapId map_id : it->second) {
+            ImageMapView& map = slot_map_views[map_id];
+            if (map.picked) {
+                continue;
+            }
+            if (!map.Overlaps(cpu_addr, size)) {
+                continue;
+            }
+            map.picked = true;
+            maps.push_back(map_id);
+            Image& image = slot_images[map.image_id];
+            if (True(image.flags & ImageFlagBits::Picked)) {
+                continue;
+            }
+            image.flags |= ImageFlagBits::Picked;
+            images.push_back(map.image_id);
+            if constexpr (BOOL_BREAK) {
+                if (func(map.image_id, image)) {
+                    return true;
+                }
+            } else {
+                func(map.image_id, image);
+            }
+        }
+        if constexpr (BOOL_BREAK) {
+            return false;
+        }
+    });
+    for (const ImageId image_id : images) {
+        slot_images[image_id].flags &= ~ImageFlagBits::Picked;
+    }
+    for (const ImageMapId map_id : maps) {
+        slot_map_views[map_id].picked = false;
+    }
+}
+
+template <class P>
+template <typename Func>
+void TextureCache<P>::ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Func&& func) {
+    using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
+    static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
+    boost::container::small_vector<ImageId, 8> images;
+    ForEachGPUPage(gpu_addr, size, [this, &images, gpu_addr, size, func](u64 page) {
+        const auto it = gpu_page_table.find(page);
+        if (it == gpu_page_table.end()) {
+            if constexpr (BOOL_BREAK) {
+                return false;
+            } else {
+                return;
+            }
+        }
         for (const ImageId image_id : it->second) {
             Image& image = slot_images[image_id];
             if (True(image.flags & ImageFlagBits::Picked)) {
                 continue;
             }
-            if (!image.Overlaps(cpu_addr, size)) {
+            if (!image.OverlapsGPU(gpu_addr, size)) {
                 continue;
             }
             image.flags |= ImageFlagBits::Picked;
@@ -1275,6 +1388,30 @@ void TextureCache<P>::ForEachImageInRegion(VAddr cpu_addr, size_t size, Func&& f
     }
 }
 
+template <class P>
+template <typename Func>
+void TextureCache<P>::ForEachSparseSegment(ImageBase& image, Func&& func) {
+    using FuncReturn = typename std::invoke_result<Func, GPUVAddr, VAddr, size_t>::type;
+    static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
+    GPUVAddr gpu_addr = image.gpu_addr;
+    const size_t levels = image.info.resources.levels;
+    const auto mipmap_sizes = CalculateMipLevelSizes(image.info);
+    for (size_t level = 0; level < levels; level++) {
+        const size_t size = mipmap_sizes[level];
+        std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+        if (cpu_addr && *cpu_addr != 0) {
+            if constexpr (BOOL_BREAK) {
+                if (func(gpu_addr, *cpu_addr, size)) {
+                    return true;
+                }
+            } else {
+                func(gpu_addr, *cpu_addr, size);
+            }
+        }
+        gpu_addr += size;
+    }
+}
+
 template <class P>
 ImageViewId TextureCache<P>::FindOrEmplaceImageView(ImageId image_id, const ImageViewInfo& info) {
     Image& image = slot_images[image_id];
@@ -1292,8 +1429,6 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
     ASSERT_MSG(False(image.flags & ImageFlagBits::Registered),
                "Trying to register an already registered image");
     image.flags |= ImageFlagBits::Registered;
-    ForEachPage(image.cpu_addr, image.guest_size_bytes,
-                [this, image_id](u64 page) { page_table[page].push_back(image_id); });
     u64 tentative_size = std::max(image.guest_size_bytes, image.unswizzled_size_bytes);
     if ((IsPixelFormatASTC(image.info.format) &&
          True(image.flags & ImageFlagBits::AcceleratedUpload)) ||
@@ -1301,6 +1436,21 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
         tentative_size = EstimatedDecompressedSize(tentative_size, image.info.format);
     }
     total_used_memory += Common::AlignUp(tentative_size, 1024);
+    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
+                   [this, image_id](u64 page) { gpu_page_table[page].push_back(image_id); });
+    if (!image.is_sparse) {
+        auto map_id =
+            slot_map_views.insert(image.gpu_addr, image.cpu_addr, image.guest_size_bytes, image_id);
+        ForEachCPUPage(image.cpu_addr, image.guest_size_bytes,
+                       [this, map_id](u64 page) { page_table[page].push_back(map_id); });
+        image.map_view_id = map_id;
+        return;
+    }
+    ForEachSparseSegment(image, [this, image_id](GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
+        auto map_id = slot_map_views.insert(gpu_addr, cpu_addr, size, image_id);
+        ForEachCPUPage(cpu_addr, size,
+                       [this, map_id](u64 page) { page_table[page].push_back(map_id); });
+    });
 }
 
 template <class P>
@@ -1317,9 +1467,9 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
         tentative_size = EstimatedDecompressedSize(tentative_size, image.info.format);
     }
     total_used_memory -= Common::AlignUp(tentative_size, 1024);
-    ForEachPage(image.cpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
-        const auto page_it = page_table.find(page);
-        if (page_it == page_table.end()) {
+    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
+        const auto page_it = gpu_page_table.find(page);
+        if (page_it == gpu_page_table.end()) {
             UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
             return;
         }
@@ -1331,20 +1481,84 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
         }
         image_ids.erase(vector_it);
     });
+    if (!image.is_sparse) {
+        const auto map_id = image.map_view_id;
+        ForEachCPUPage(image.cpu_addr, image.guest_size_bytes, [this, map_id](u64 page) {
+            const auto page_it = page_table.find(page);
+            if (page_it == page_table.end()) {
+                UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
+                return;
+            }
+            std::vector<ImageMapId>& image_map_ids = page_it->second;
+            const auto vector_it = std::ranges::find(image_map_ids, map_id);
+            if (vector_it == image_map_ids.end()) {
+                UNREACHABLE_MSG("Unregistering unregistered image in page=0x{:x}",
+                                page << PAGE_BITS);
+                return;
+            }
+            image_map_ids.erase(vector_it);
+        });
+        slot_map_views.erase(map_id);
+        return;
+    }
+    boost::container::small_vector<ImageMapId, 8> maps_to_delete;
+    ForEachSparseSegment(
+        image, [this, image_id, &maps_to_delete]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr,
+                                                 size_t size) {
+            ForEachCPUPage(cpu_addr, size, [this, image_id, &maps_to_delete](u64 page) {
+                const auto page_it = page_table.find(page);
+                if (page_it == page_table.end()) {
+                    UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
+                    return;
+                }
+                std::vector<ImageMapId>& image_map_ids = page_it->second;
+                auto vector_it = image_map_ids.begin();
+                while (vector_it != image_map_ids.end()) {
+                    ImageMapView& map = slot_map_views[*vector_it];
+                    if (map.image_id != image_id) {
+                        vector_it++;
+                        continue;
+                    }
+                    if (!map.picked) {
+                        maps_to_delete.push_back(*vector_it);
+                        map.picked = true;
+                    }
+                    vector_it = image_map_ids.erase(vector_it);
+                }
+            });
+        });
+
+    for (const ImageMapId map_id : maps_to_delete) {
+        slot_map_views.erase(map_id);
+    }
 }
 
 template <class P>
 void TextureCache<P>::TrackImage(ImageBase& image) {
     ASSERT(False(image.flags & ImageFlagBits::Tracked));
     image.flags |= ImageFlagBits::Tracked;
-    rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
+    if (!image.is_sparse) {
+        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
+        return;
+    }
+    ForEachSparseSegment(image,
+                         [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
+                             rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
+                         });
 }
 
 template <class P>
 void TextureCache<P>::UntrackImage(ImageBase& image) {
     ASSERT(True(image.flags & ImageFlagBits::Tracked));
     image.flags &= ~ImageFlagBits::Tracked;
-    rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
+    if (!image.is_sparse) {
+        rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
+        return;
+    }
+    ForEachSparseSegment(image,
+                         [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
+                             rasterizer.UpdatePagesCachedCount(cpu_addr, size, -1);
+                         });
 }
 
 template <class P>
diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h
index c9571f7e4..9fbdc1ac6 100644
--- a/src/video_core/texture_cache/types.h
+++ b/src/video_core/texture_cache/types.h
@@ -16,6 +16,7 @@ constexpr size_t MAX_MIP_LEVELS = 14;
 constexpr SlotId CORRUPT_ID{0xfffffffe};
 
 using ImageId = SlotId;
+using ImageMapId = SlotId;
 using ImageViewId = SlotId;
 using ImageAllocId = SlotId;
 using SamplerId = SlotId;
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index 4efe042b6..96bf8f8d9 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -664,6 +664,16 @@ LevelArray CalculateMipLevelOffsets(const ImageInfo& info) noexcept {
     return offsets;
 }
 
+LevelArray CalculateMipLevelSizes(const ImageInfo& info) noexcept {
+    const u32 num_levels = info.resources.levels;
+    const LevelInfo level_info = MakeLevelInfo(info);
+    LevelArray sizes{};
+    for (u32 level = 0; level < num_levels; ++level) {
+        sizes[level] = CalculateLevelSize(level_info, level);
+    }
+    return sizes;
+}
+
 std::vector<u32> CalculateSliceOffsets(const ImageInfo& info) {
     ASSERT(info.type == ImageType::e3D);
     std::vector<u32> offsets;
@@ -776,14 +786,37 @@ std::vector<ImageCopy> MakeShrinkImageCopies(const ImageInfo& dst, const ImageIn
     return copies;
 }
 
-bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, const TICEntry& config) {
-    if (config.Address() == 0) {
+bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr) {
+    if (gpu_addr == 0) {
         return false;
     }
-    if (config.Address() > (u64(1) << 48)) {
+    if (gpu_addr > (u64(1) << 48)) {
         return false;
     }
-    return gpu_memory.GpuToCpuAddress(config.Address()).has_value();
+    const auto cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+    return cpu_addr.has_value() && *cpu_addr != 0;
+}
+
+bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config) {
+    const GPUVAddr gpu_addr = config.Address();
+    if (IsValidAddress(gpu_memory, gpu_addr)) {
+        return true;
+    }
+    if (!config.IsBlockLinear()) {
+        return false;
+    }
+    const size_t levels = config.max_mip_level + 1;
+    if (levels <= 1) {
+        return false;
+    }
+    const ImageInfo info{config};
+    const LevelArray offsets = CalculateMipLevelOffsets(info);
+    for (size_t level = 1; level < levels; level++) {
+        if (IsValidAddress(gpu_memory, static_cast<GPUVAddr>(gpu_addr + offsets[level]))) {
+            return true;
+        }
+    }
+    return false;
 }
 
 std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h
index cdc5cbc75..b73361484 100644
--- a/src/video_core/texture_cache/util.h
+++ b/src/video_core/texture_cache/util.h
@@ -40,6 +40,8 @@ struct OverlapResult {
 
 [[nodiscard]] LevelArray CalculateMipLevelOffsets(const ImageInfo& info) noexcept;
 
+[[nodiscard]] LevelArray CalculateMipLevelSizes(const ImageInfo& info) noexcept;
+
 [[nodiscard]] std::vector<u32> CalculateSliceOffsets(const ImageInfo& info);
 
 [[nodiscard]] std::vector<SubresourceBase> CalculateSliceSubresources(const ImageInfo& info);
@@ -55,7 +57,9 @@ struct OverlapResult {
                                                            const ImageInfo& src,
                                                            SubresourceBase base);
 
-[[nodiscard]] bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, const TICEntry& config);
+[[nodiscard]] bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr);
+
+[[nodiscard]] bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config);
 
 [[nodiscard]] std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory,
                                                           GPUVAddr gpu_addr, const ImageInfo& info,
-- 
cgit v1.2.3


From fd98fcf7f00d096322ccfaa1e35a314b4d698efd Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Sun, 13 Jun 2021 03:34:06 +0200
Subject: Texture Cache: Improve accuracy of sparse texture detection.

---
 src/video_core/texture_cache/image_base.h    |   5 +-
 src/video_core/texture_cache/texture_cache.h | 310 ++++++++++++++++++---------
 src/video_core/texture_cache/util.cpp        |  31 +--
 src/video_core/texture_cache/util.h          |   2 -
 4 files changed, 222 insertions(+), 126 deletions(-)

(limited to 'src/video_core/texture_cache')

diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 004ec23e4..fd14a3980 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -25,11 +25,12 @@ enum class ImageFlagBits : u32 {
     Strong = 1 << 5,      ///< Exists in the image table, the dimensions are can be trusted
     Registered = 1 << 6,  ///< True when the image is registered
     Picked = 1 << 7,      ///< Temporary flag to mark the image as picked
+    Remapped = 1 << 8,    ///< Image has been remapped.
 
     // Garbage Collection Flags
-    BadOverlap = 1 << 8, ///< This image overlaps other but doesn't fit, has higher
+    BadOverlap = 1 << 9, ///< This image overlaps other but doesn't fit, has higher
                          ///< garbage collection priority
-    Alias = 1 << 9,      ///< This image has aliases and has priority on garbage
+    Alias = 1 << 10,     ///< This image has aliases and has priority on garbage
                          ///< collection
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 9f6410d58..1704fc48c 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -13,6 +13,7 @@
 #include <span>
 #include <type_traits>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
@@ -155,6 +156,9 @@ public:
     /// Remove images in a region
     void UnmapGPUMemory(GPUVAddr gpu_addr, size_t size);
 
+    /// Used when GPU memory changes layout on sparse textures.
+    // void CheckRemaps();
+
     /// Blit an image with the given parameters
     void BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
                    const Tegra::Engines::Fermi2D::Surface& src,
@@ -238,7 +242,7 @@ private:
     FramebufferId GetFramebufferId(const RenderTargets& key);
 
     /// Refresh the contents (pixel data) of an image
-    void RefreshContents(Image& image);
+    void RefreshContents(Image& image, ImageId image_id);
 
     /// Upload data from guest to an image
     template <typename StagingBuffer>
@@ -290,6 +294,9 @@ private:
     template <typename Func>
     void ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Func&& func);
 
+    template <typename Func>
+    void ForEachSparseImageInRegion(GPUVAddr gpu_addr, size_t size, Func&& func);
+
     /// Iterates over all the images in a region calling func
     template <typename Func>
     void ForEachSparseSegment(ImageBase& image, Func&& func);
@@ -304,10 +311,10 @@ private:
     void UnregisterImage(ImageId image);
 
     /// Track CPU reads and writes for image
-    void TrackImage(ImageBase& image);
+    void TrackImage(ImageBase& image, ImageId image_id);
 
     /// Stop tracking CPU reads and writes for image
-    void UntrackImage(ImageBase& image);
+    void UntrackImage(ImageBase& image, ImageId image_id);
 
     /// Delete image from the cache
     void DeleteImage(ImageId image);
@@ -367,6 +374,11 @@ private:
 
     std::unordered_map<u64, std::vector<ImageMapId>, IdentityHash<u64>> page_table;
     std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> gpu_page_table;
+    std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>> sparse_page_table;
+
+    std::unordered_map<ImageId, std::vector<ImageViewId>> sparse_views;
+
+    VAddr virtual_invalid_space{};
 
     bool has_deleted_images = false;
     u64 total_used_memory = 0;
@@ -685,7 +697,9 @@ void TextureCache<P>::WriteMemory(VAddr cpu_addr, size_t size) {
             return;
         }
         image.flags |= ImageFlagBits::CpuModified;
-        UntrackImage(image);
+        if (True(image.flags & ImageFlagBits::Tracked)) {
+            UntrackImage(image, image_id);
+        }
     });
 }
 
@@ -722,7 +736,7 @@ void TextureCache<P>::UnmapMemory(VAddr cpu_addr, size_t size) {
     for (const ImageId id : deleted_images) {
         Image& image = slot_images[id];
         if (True(image.flags & ImageFlagBits::Tracked)) {
-            UntrackImage(image);
+            UntrackImage(image, id);
         }
         UnregisterImage(id);
         DeleteImage(id);
@@ -736,11 +750,13 @@ void TextureCache<P>::UnmapGPUMemory(GPUVAddr gpu_addr, size_t size) {
                             [&](ImageId id, Image&) { deleted_images.push_back(id); });
     for (const ImageId id : deleted_images) {
         Image& image = slot_images[id];
+        if (True(image.flags & ImageFlagBits::Remapped)) {
+            continue;
+        }
+        image.flags |= ImageFlagBits::Remapped;
         if (True(image.flags & ImageFlagBits::Tracked)) {
-            UntrackImage(image);
+            UntrackImage(image, id);
         }
-        UnregisterImage(id);
-        DeleteImage(id);
     }
 }
 
@@ -958,13 +974,13 @@ bool TextureCache<P>::IsRegionGpuModified(VAddr addr, size_t size) {
 }
 
 template <class P>
-void TextureCache<P>::RefreshContents(Image& image) {
+void TextureCache<P>::RefreshContents(Image& image, ImageId image_id) {
     if (False(image.flags & ImageFlagBits::CpuModified)) {
         // Only upload modified images
         return;
     }
     image.flags &= ~ImageFlagBits::CpuModified;
-    TrackImage(image);
+    TrackImage(image, image_id);
 
     if (image.info.num_samples > 1) {
         LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented");
@@ -1043,14 +1059,20 @@ ImageId TextureCache<P>::FindOrInsertImage(const ImageInfo& info, GPUVAddr gpu_a
 template <class P>
 ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
                                    RelaxedOptions options) {
-    const std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+    std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
     if (!cpu_addr) {
-        return ImageId{};
+        cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr, CalculateGuestSizeInBytes(info));
+        if (!cpu_addr) {
+            return ImageId{};
+        }
     }
     const bool broken_views = runtime.HasBrokenTextureViewFormats();
     const bool native_bgr = runtime.HasNativeBgr();
     ImageId image_id;
     const auto lambda = [&](ImageId existing_image_id, ImageBase& existing_image) {
+        if (True(existing_image.flags & ImageFlagBits::Remapped)) {
+            return false;
+        }
         if (info.type == ImageType::Linear || existing_image.info.type == ImageType::Linear) {
             const bool strict_size = False(options & RelaxedOptions::Size) &&
                                      True(existing_image.flags & ImageFlagBits::Strong);
@@ -1069,14 +1091,23 @@ ImageId TextureCache<P>::FindImage(const ImageInfo& info, GPUVAddr gpu_addr,
         }
         return false;
     };
-    ForEachImageInRegionGPU(gpu_addr, CalculateGuestSizeInBytes(info), lambda);
+    ForEachImageInRegion(*cpu_addr, CalculateGuestSizeInBytes(info), lambda);
     return image_id;
 }
 
 template <class P>
 ImageId TextureCache<P>::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr,
                                      RelaxedOptions options) {
-    const std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+    std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+    if (!cpu_addr) {
+        const auto size = CalculateGuestSizeInBytes(info);
+        cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr, size);
+        if (!cpu_addr) {
+            const VAddr fake_addr = ~(1ULL << 40ULL) + virtual_invalid_space;
+            virtual_invalid_space += Common::AlignUp(size, 32);
+            cpu_addr = std::optional<VAddr>(fake_addr);
+        }
+    }
     ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr=0x{:x}", gpu_addr);
     const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr);
     const Image& image = slot_images[image_id];
@@ -1096,10 +1127,16 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     const bool broken_views = runtime.HasBrokenTextureViewFormats();
     const bool native_bgr = runtime.HasNativeBgr();
     std::vector<ImageId> overlap_ids;
+    std::unordered_set<ImageId> overlaps_found;
     std::vector<ImageId> left_aliased_ids;
     std::vector<ImageId> right_aliased_ids;
+    std::unordered_set<ImageId> ignore_textures;
     std::vector<ImageId> bad_overlap_ids;
-    ForEachImageInRegionGPU(gpu_addr, size_bytes, [&](ImageId overlap_id, ImageBase& overlap) {
+    const auto region_check = [&](ImageId overlap_id, ImageBase& overlap) {
+        if (True(overlap.flags & ImageFlagBits::Remapped)) {
+            ignore_textures.insert(overlap_id);
+            return;
+        }
         if (info.type == ImageType::Linear) {
             if (info.pitch == overlap.info.pitch && gpu_addr == overlap.gpu_addr) {
                 // Alias linear images with the same pitch
@@ -1107,6 +1144,7 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
             }
             return;
         }
+        overlaps_found.insert(overlap_id);
         static constexpr bool strict_size = true;
         const std::optional<OverlapResult> solution = ResolveOverlap(
             new_info, gpu_addr, cpu_addr, overlap, strict_size, broken_views, native_bgr);
@@ -1130,30 +1168,34 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
             bad_overlap_ids.push_back(overlap_id);
             overlap.flags |= ImageFlagBits::BadOverlap;
         }
-    });
+    };
+    ForEachImageInRegion(cpu_addr, size_bytes, region_check);
+    const auto region_check_gpu = [&](ImageId overlap_id, ImageBase& overlap) {
+        if (!overlaps_found.contains(overlap_id)) {
+            ignore_textures.insert(overlap_id);
+        }
+    };
+    ForEachSparseImageInRegion(gpu_addr, size_bytes, region_check_gpu);
     const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr);
     Image& new_image = slot_images[new_image_id];
 
-    new_image.is_sparse = false;
-    if (new_image.info.type != ImageType::Linear && new_image.info.type != ImageType::Buffer) {
-        const LevelArray offsets = CalculateMipLevelOffsets(new_image.info);
-        size_t level;
-        const size_t levels = static_cast<size_t>(new_image.info.resources.levels);
-        VAddr n_cpu_addr = new_image.cpu_addr;
-        GPUVAddr n_gpu_addr = new_image.gpu_addr;
-        for (level = 0; level < levels; level++) {
-            n_gpu_addr += offsets[level];
-            n_cpu_addr += offsets[level];
-            std::optional<VAddr> cpu_addr_opt = gpu_memory.GpuToCpuAddress(n_gpu_addr);
-            if (!cpu_addr_opt || *cpu_addr_opt == 0 || n_cpu_addr != *cpu_addr_opt) {
-                new_image.is_sparse = true;
-                break;
-            }
+    new_image.is_sparse =
+        !gpu_memory.IsContinousRange(new_image.gpu_addr, new_image.guest_size_bytes);
+
+    for (const ImageId overlap_id : ignore_textures) {
+        Image& overlap = slot_images[overlap_id];
+        if (True(overlap.flags & ImageFlagBits::GpuModified)) {
+            UNIMPLEMENTED();
+        }
+        if (True(overlap.flags & ImageFlagBits::Tracked)) {
+            UntrackImage(overlap, overlap_id);
         }
+        UnregisterImage(overlap_id);
+        DeleteImage(overlap_id);
     }
 
     // TODO: Only upload what we need
-    RefreshContents(new_image);
+    RefreshContents(new_image, new_image_id);
 
     for (const ImageId overlap_id : overlap_ids) {
         Image& overlap = slot_images[overlap_id];
@@ -1165,7 +1207,7 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
             runtime.CopyImage(new_image, overlap, copies);
         }
         if (True(overlap.flags & ImageFlagBits::Tracked)) {
-            UntrackImage(overlap);
+            UntrackImage(overlap, overlap_id);
         }
         UnregisterImage(overlap_id);
         DeleteImage(overlap_id);
@@ -1390,25 +1432,64 @@ void TextureCache<P>::ForEachImageInRegionGPU(GPUVAddr gpu_addr, size_t size, Fu
 
 template <class P>
 template <typename Func>
-void TextureCache<P>::ForEachSparseSegment(ImageBase& image, Func&& func) {
-    using FuncReturn = typename std::invoke_result<Func, GPUVAddr, VAddr, size_t>::type;
+void TextureCache<P>::ForEachSparseImageInRegion(GPUVAddr gpu_addr, size_t size, Func&& func) {
+    using FuncReturn = typename std::invoke_result<Func, ImageId, Image&>::type;
     static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
-    GPUVAddr gpu_addr = image.gpu_addr;
-    const size_t levels = image.info.resources.levels;
-    const auto mipmap_sizes = CalculateMipLevelSizes(image.info);
-    for (size_t level = 0; level < levels; level++) {
-        const size_t size = mipmap_sizes[level];
-        std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
-        if (cpu_addr && *cpu_addr != 0) {
+    boost::container::small_vector<ImageId, 8> images;
+    ForEachGPUPage(gpu_addr, size, [this, &images, gpu_addr, size, func](u64 page) {
+        const auto it = sparse_page_table.find(page);
+        if (it == sparse_page_table.end()) {
+            if constexpr (BOOL_BREAK) {
+                return false;
+            } else {
+                return;
+            }
+        }
+        for (const ImageId image_id : it->second) {
+            Image& image = slot_images[image_id];
+            if (True(image.flags & ImageFlagBits::Picked)) {
+                continue;
+            }
+            if (!image.OverlapsGPU(gpu_addr, size)) {
+                continue;
+            }
+            image.flags |= ImageFlagBits::Picked;
+            images.push_back(image_id);
             if constexpr (BOOL_BREAK) {
-                if (func(gpu_addr, *cpu_addr, size)) {
+                if (func(image_id, image)) {
                     return true;
                 }
             } else {
-                func(gpu_addr, *cpu_addr, size);
+                func(image_id, image);
+            }
+        }
+        if constexpr (BOOL_BREAK) {
+            return false;
+        }
+    });
+    for (const ImageId image_id : images) {
+        slot_images[image_id].flags &= ~ImageFlagBits::Picked;
+    }
+}
+
+template <class P>
+template <typename Func>
+void TextureCache<P>::ForEachSparseSegment(ImageBase& image, Func&& func) {
+    using FuncReturn = typename std::invoke_result<Func, GPUVAddr, VAddr, size_t>::type;
+    static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
+    const auto segments = gpu_memory.GetSubmappedRange(image.gpu_addr, image.guest_size_bytes);
+    for (auto& segment : segments) {
+        const auto gpu_addr = segment.first;
+        const auto size = segment.second;
+        std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
+        ASSERT(cpu_addr);
+        if constexpr (BOOL_BREAK) {
+            if (func(gpu_addr, *cpu_addr, size)) {
+                return true;
             }
+        } else {
+            func(gpu_addr, *cpu_addr, size);
         }
-        gpu_addr += size;
     }
 }
 
@@ -1446,11 +1527,17 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
         image.map_view_id = map_id;
         return;
     }
-    ForEachSparseSegment(image, [this, image_id](GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
-        auto map_id = slot_map_views.insert(gpu_addr, cpu_addr, size, image_id);
-        ForEachCPUPage(cpu_addr, size,
-                       [this, map_id](u64 page) { page_table[page].push_back(map_id); });
-    });
+    std::vector<ImageViewId> sparse_maps{};
+    ForEachSparseSegment(
+        image, [this, image_id, &sparse_maps](GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
+            auto map_id = slot_map_views.insert(gpu_addr, cpu_addr, size, image_id);
+            ForEachCPUPage(cpu_addr, size,
+                           [this, map_id](u64 page) { page_table[page].push_back(map_id); });
+            sparse_maps.push_back(map_id);
+        });
+    sparse_views.emplace(image_id, std::move(sparse_maps));
+    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
+                   [this, image_id](u64 page) { sparse_page_table[page].push_back(image_id); });
 }
 
 template <class P>
@@ -1467,20 +1554,26 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
         tentative_size = EstimatedDecompressedSize(tentative_size, image.info.format);
     }
     total_used_memory -= Common::AlignUp(tentative_size, 1024);
-    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) {
-        const auto page_it = gpu_page_table.find(page);
-        if (page_it == gpu_page_table.end()) {
-            UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
-            return;
-        }
-        std::vector<ImageId>& image_ids = page_it->second;
-        const auto vector_it = std::ranges::find(image_ids, image_id);
-        if (vector_it == image_ids.end()) {
-            UNREACHABLE_MSG("Unregistering unregistered image in page=0x{:x}", page << PAGE_BITS);
-            return;
-        }
-        image_ids.erase(vector_it);
-    });
+    const auto& clear_page_table =
+        [this, image_id](
+            u64 page,
+            std::unordered_map<u64, std::vector<ImageId>, IdentityHash<u64>>& selected_page_table) {
+            const auto page_it = selected_page_table.find(page);
+            if (page_it == selected_page_table.end()) {
+                UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
+                return;
+            }
+            std::vector<ImageId>& image_ids = page_it->second;
+            const auto vector_it = std::ranges::find(image_ids, image_id);
+            if (vector_it == image_ids.end()) {
+                UNREACHABLE_MSG("Unregistering unregistered image in page=0x{:x}",
+                                page << PAGE_BITS);
+                return;
+            }
+            image_ids.erase(vector_it);
+        };
+    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
+                   [this, &clear_page_table](u64 page) { clear_page_table(page, gpu_page_table); });
     if (!image.is_sparse) {
         const auto map_id = image.map_view_id;
         ForEachCPUPage(image.cpu_addr, image.guest_size_bytes, [this, map_id](u64 page) {
@@ -1501,46 +1594,61 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
         slot_map_views.erase(map_id);
         return;
     }
-    boost::container::small_vector<ImageMapId, 8> maps_to_delete;
-    ForEachSparseSegment(
-        image, [this, image_id, &maps_to_delete]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr,
-                                                 size_t size) {
-            ForEachCPUPage(cpu_addr, size, [this, image_id, &maps_to_delete](u64 page) {
-                const auto page_it = page_table.find(page);
-                if (page_it == page_table.end()) {
-                    UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
-                    return;
+    ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, &clear_page_table](u64 page) {
+        clear_page_table(page, sparse_page_table);
+    });
+    auto it = sparse_views.find(image_id);
+    ASSERT(it != sparse_views.end());
+    auto& sparse_maps = it->second;
+    for (auto& map_view_id : sparse_maps) {
+        const auto& map = slot_map_views[map_view_id];
+        const VAddr cpu_addr = map.cpu_addr;
+        const std::size_t size = map.size;
+        ForEachCPUPage(cpu_addr, size, [this, image_id](u64 page) {
+            const auto page_it = page_table.find(page);
+            if (page_it == page_table.end()) {
+                UNREACHABLE_MSG("Unregistering unregistered page=0x{:x}", page << PAGE_BITS);
+                return;
+            }
+            std::vector<ImageMapId>& image_map_ids = page_it->second;
+            auto vector_it = image_map_ids.begin();
+            while (vector_it != image_map_ids.end()) {
+                ImageMapView& map = slot_map_views[*vector_it];
+                if (map.image_id != image_id) {
+                    vector_it++;
+                    continue;
                 }
-                std::vector<ImageMapId>& image_map_ids = page_it->second;
-                auto vector_it = image_map_ids.begin();
-                while (vector_it != image_map_ids.end()) {
-                    ImageMapView& map = slot_map_views[*vector_it];
-                    if (map.image_id != image_id) {
-                        vector_it++;
-                        continue;
-                    }
-                    if (!map.picked) {
-                        maps_to_delete.push_back(*vector_it);
-                        map.picked = true;
-                    }
-                    vector_it = image_map_ids.erase(vector_it);
+                if (!map.picked) {
+                    map.picked = true;
                 }
-            });
+                vector_it = image_map_ids.erase(vector_it);
+            }
         });
-
-    for (const ImageMapId map_id : maps_to_delete) {
-        slot_map_views.erase(map_id);
+        slot_map_views.erase(map_view_id);
     }
+    sparse_views.erase(it);
 }
 
 template <class P>
-void TextureCache<P>::TrackImage(ImageBase& image) {
+void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(False(image.flags & ImageFlagBits::Tracked));
     image.flags |= ImageFlagBits::Tracked;
     if (!image.is_sparse) {
         rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
         return;
     }
+    if (True(image.flags & ImageFlagBits::Registered)) {
+        auto it = sparse_views.find(image_id);
+        ASSERT(it != sparse_views.end());
+        auto& sparse_maps = it->second;
+        for (auto& map_view_id : sparse_maps) {
+            const auto& map = slot_map_views[map_view_id];
+            const VAddr cpu_addr = map.cpu_addr;
+            const std::size_t size = map.size;
+            rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
+        }
+        return;
+    }
     ForEachSparseSegment(image,
                          [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
                              rasterizer.UpdatePagesCachedCount(cpu_addr, size, 1);
@@ -1548,17 +1656,23 @@ void TextureCache<P>::TrackImage(ImageBase& image) {
 }
 
 template <class P>
-void TextureCache<P>::UntrackImage(ImageBase& image) {
+void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(True(image.flags & ImageFlagBits::Tracked));
     image.flags &= ~ImageFlagBits::Tracked;
     if (!image.is_sparse) {
         rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
         return;
     }
-    ForEachSparseSegment(image,
-                         [this]([[maybe_unused]] GPUVAddr gpu_addr, VAddr cpu_addr, size_t size) {
-                             rasterizer.UpdatePagesCachedCount(cpu_addr, size, -1);
-                         });
+    ASSERT(True(image.flags & ImageFlagBits::Registered));
+    auto it = sparse_views.find(image_id);
+    ASSERT(it != sparse_views.end());
+    auto& sparse_maps = it->second;
+    for (auto& map_view_id : sparse_maps) {
+        const auto& map = slot_map_views[map_view_id];
+        const VAddr cpu_addr = map.cpu_addr;
+        const std::size_t size = map.size;
+        rasterizer.UpdatePagesCachedCount(cpu_addr, size, -1);
+    }
 }
 
 template <class P>
@@ -1700,10 +1814,10 @@ void TextureCache<P>::PrepareImage(ImageId image_id, bool is_modification, bool
     if (invalidate) {
         image.flags &= ~(ImageFlagBits::CpuModified | ImageFlagBits::GpuModified);
         if (False(image.flags & ImageFlagBits::Tracked)) {
-            TrackImage(image);
+            TrackImage(image, image_id);
         }
     } else {
-        RefreshContents(image);
+        RefreshContents(image, image_id);
         SynchronizeAliases(image_id);
     }
     if (is_modification) {
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index 96bf8f8d9..10093a11d 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -786,37 +786,20 @@ std::vector<ImageCopy> MakeShrinkImageCopies(const ImageInfo& dst, const ImageIn
     return copies;
 }
 
-bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr) {
-    if (gpu_addr == 0) {
+bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config) {
+    const GPUVAddr address = config.Address();
+    if (address == 0) {
         return false;
     }
-    if (gpu_addr > (u64(1) << 48)) {
+    if (address > (1ULL << 48)) {
         return false;
     }
-    const auto cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
-    return cpu_addr.has_value() && *cpu_addr != 0;
-}
-
-bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config) {
-    const GPUVAddr gpu_addr = config.Address();
-    if (IsValidAddress(gpu_memory, gpu_addr)) {
+    if (gpu_memory.GpuToCpuAddress(address).has_value()) {
         return true;
     }
-    if (!config.IsBlockLinear()) {
-        return false;
-    }
-    const size_t levels = config.max_mip_level + 1;
-    if (levels <= 1) {
-        return false;
-    }
     const ImageInfo info{config};
-    const LevelArray offsets = CalculateMipLevelOffsets(info);
-    for (size_t level = 1; level < levels; level++) {
-        if (IsValidAddress(gpu_memory, static_cast<GPUVAddr>(gpu_addr + offsets[level]))) {
-            return true;
-        }
-    }
-    return false;
+    const size_t guest_size_bytes = CalculateGuestSizeInBytes(info);
+    return gpu_memory.GpuToCpuAddress(address, guest_size_bytes).has_value();
 }
 
 std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h
index b73361484..766502908 100644
--- a/src/video_core/texture_cache/util.h
+++ b/src/video_core/texture_cache/util.h
@@ -57,8 +57,6 @@ struct OverlapResult {
                                                            const ImageInfo& src,
                                                            SubresourceBase base);
 
-[[nodiscard]] bool IsValidAddress(const Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr);
-
 [[nodiscard]] bool IsValidEntry(const Tegra::MemoryManager& gpu_memory, const TICEntry& config);
 
 [[nodiscard]] std::vector<BufferImageCopy> UnswizzleImage(Tegra::MemoryManager& gpu_memory,
-- 
cgit v1.2.3


From 8f9f142956bb5de205cad2237f0cf60770796867 Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Sun, 20 Jun 2021 12:25:59 +0200
Subject: Texture Cache: Address feedback.

---
 src/video_core/texture_cache/image_base.h    | 28 +++++++++++++++++++++++++---
 src/video_core/texture_cache/texture_cache.h | 16 +++++++---------
 2 files changed, 32 insertions(+), 12 deletions(-)

(limited to 'src/video_core/texture_cache')

diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index fd14a3980..65e68cbb3 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -26,11 +26,12 @@ enum class ImageFlagBits : u32 {
     Registered = 1 << 6,  ///< True when the image is registered
     Picked = 1 << 7,      ///< Temporary flag to mark the image as picked
     Remapped = 1 << 8,    ///< Image has been remapped.
+    Sparse = 1 << 9,      ///< Image has non continous submemory.
 
     // Garbage Collection Flags
-    BadOverlap = 1 << 9, ///< This image overlaps other but doesn't fit, has higher
+    BadOverlap = 1 << 10,///< This image overlaps other but doesn't fit, has higher
                          ///< garbage collection priority
-    Alias = 1 << 10,     ///< This image has aliases and has priority on garbage
+    Alias = 1 << 11,     ///< This image has aliases and has priority on garbage
                          ///< collection
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
@@ -92,7 +93,28 @@ struct ImageBase {
     std::vector<AliasedImage> aliased_images;
     std::vector<ImageId> overlapping_images;
     ImageMapId map_view_id{};
-    bool is_sparse{};
+};
+
+struct ImageMapView {
+    explicit ImageMapView(GPUVAddr gpu_addr, VAddr cpu_addr, size_t size, ImageId image_id);
+
+    [[nodiscard]] bool Overlaps(VAddr overlap_cpu_addr, size_t overlap_size) const noexcept {
+        const VAddr overlap_end = overlap_cpu_addr + overlap_size;
+        const VAddr cpu_addr_end = cpu_addr + size;
+        return cpu_addr < overlap_end && overlap_cpu_addr < cpu_addr_end;
+    }
+
+    [[nodiscard]] bool OverlapsGPU(GPUVAddr overlap_gpu_addr, size_t overlap_size) const noexcept {
+        const GPUVAddr overlap_end = overlap_gpu_addr + overlap_size;
+        const GPUVAddr gpu_addr_end = gpu_addr + size;
+        return gpu_addr < overlap_end && overlap_gpu_addr < gpu_addr_end;
+    }
+
+    GPUVAddr gpu_addr;
+    VAddr cpu_addr;
+    size_t size;
+    ImageId image_id;
+    bool picked{};
 };
 
 struct ImageAllocBase {
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 1704fc48c..500c4dd52 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -156,9 +156,6 @@ public:
     /// Remove images in a region
     void UnmapGPUMemory(GPUVAddr gpu_addr, size_t size);
 
-    /// Used when GPU memory changes layout on sparse textures.
-    // void CheckRemaps();
-
     /// Blit an image with the given parameters
     void BlitImage(const Tegra::Engines::Fermi2D::Surface& dst,
                    const Tegra::Engines::Fermi2D::Surface& src,
@@ -1179,8 +1176,9 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr);
     Image& new_image = slot_images[new_image_id];
 
-    new_image.is_sparse =
-        !gpu_memory.IsContinousRange(new_image.gpu_addr, new_image.guest_size_bytes);
+    if (!gpu_memory.IsContinousRange(new_image.gpu_addr, new_image.guest_size_bytes)) {
+        new_image.flags |= ImageFlagBits::Sparse;
+    }
 
     for (const ImageId overlap_id : ignore_textures) {
         Image& overlap = slot_images[overlap_id];
@@ -1519,7 +1517,7 @@ void TextureCache<P>::RegisterImage(ImageId image_id) {
     total_used_memory += Common::AlignUp(tentative_size, 1024);
     ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
                    [this, image_id](u64 page) { gpu_page_table[page].push_back(image_id); });
-    if (!image.is_sparse) {
+    if (False(image.flags & ImageFlagBits::Sparse)) {
         auto map_id =
             slot_map_views.insert(image.gpu_addr, image.cpu_addr, image.guest_size_bytes, image_id);
         ForEachCPUPage(image.cpu_addr, image.guest_size_bytes,
@@ -1574,7 +1572,7 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
         };
     ForEachGPUPage(image.gpu_addr, image.guest_size_bytes,
                    [this, &clear_page_table](u64 page) { clear_page_table(page, gpu_page_table); });
-    if (!image.is_sparse) {
+    if (False(image.flags & ImageFlagBits::Sparse)) {
         const auto map_id = image.map_view_id;
         ForEachCPUPage(image.cpu_addr, image.guest_size_bytes, [this, map_id](u64 page) {
             const auto page_it = page_table.find(page);
@@ -1633,7 +1631,7 @@ template <class P>
 void TextureCache<P>::TrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(False(image.flags & ImageFlagBits::Tracked));
     image.flags |= ImageFlagBits::Tracked;
-    if (!image.is_sparse) {
+    if (False(image.flags & ImageFlagBits::Sparse)) {
         rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1);
         return;
     }
@@ -1659,7 +1657,7 @@ template <class P>
 void TextureCache<P>::UntrackImage(ImageBase& image, ImageId image_id) {
     ASSERT(True(image.flags & ImageFlagBits::Tracked));
     image.flags &= ~ImageFlagBits::Tracked;
-    if (!image.is_sparse) {
+    if (False(image.flags & ImageFlagBits::Sparse)) {
         rasterizer.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1);
         return;
     }
-- 
cgit v1.2.3


From a8a0927d424815d01782c6872a4c9ab605a87dbe Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Sun, 20 Jun 2021 19:14:40 +0200
Subject: Texture Cache: Fix GCC & Clang.

---
 src/video_core/texture_cache/image_base.h    |  8 ++++----
 src/video_core/texture_cache/texture_cache.h | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

(limited to 'src/video_core/texture_cache')

diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h
index 65e68cbb3..ff1feda9b 100644
--- a/src/video_core/texture_cache/image_base.h
+++ b/src/video_core/texture_cache/image_base.h
@@ -29,10 +29,10 @@ enum class ImageFlagBits : u32 {
     Sparse = 1 << 9,      ///< Image has non continous submemory.
 
     // Garbage Collection Flags
-    BadOverlap = 1 << 10,///< This image overlaps other but doesn't fit, has higher
-                         ///< garbage collection priority
-    Alias = 1 << 11,     ///< This image has aliases and has priority on garbage
-                         ///< collection
+    BadOverlap = 1 << 10, ///< This image overlaps other but doesn't fit, has higher
+                          ///< garbage collection priority
+    Alias = 1 << 11,      ///< This image has aliases and has priority on garbage
+                          ///< collection
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
 
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 500c4dd52..8b1aa8122 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -495,7 +495,7 @@ void TextureCache<P>::RunGarbageCollector() {
                 }
             }
             if (True(image->flags & ImageFlagBits::Tracked)) {
-                UntrackImage(*image);
+                UntrackImage(*image, image_id);
             }
             UnregisterImage(image_id);
             DeleteImage(image_id);
@@ -1474,16 +1474,16 @@ template <class P>
 template <typename Func>
 void TextureCache<P>::ForEachSparseSegment(ImageBase& image, Func&& func) {
     using FuncReturn = typename std::invoke_result<Func, GPUVAddr, VAddr, size_t>::type;
-    static constexpr bool BOOL_BREAK = std::is_same_v<FuncReturn, bool>;
+    static constexpr bool RETURNS_BOOL = std::is_same_v<FuncReturn, bool>;
     const auto segments = gpu_memory.GetSubmappedRange(image.gpu_addr, image.guest_size_bytes);
     for (auto& segment : segments) {
         const auto gpu_addr = segment.first;
         const auto size = segment.second;
         std::optional<VAddr> cpu_addr = gpu_memory.GpuToCpuAddress(gpu_addr);
         ASSERT(cpu_addr);
-        if constexpr (BOOL_BREAK) {
+        if constexpr (RETURNS_BOOL) {
             if (func(gpu_addr, *cpu_addr, size)) {
-                return true;
+                break;
             }
         } else {
             func(gpu_addr, *cpu_addr, size);
@@ -1599,9 +1599,9 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) {
     ASSERT(it != sparse_views.end());
     auto& sparse_maps = it->second;
     for (auto& map_view_id : sparse_maps) {
-        const auto& map = slot_map_views[map_view_id];
-        const VAddr cpu_addr = map.cpu_addr;
-        const std::size_t size = map.size;
+        const auto& map_range = slot_map_views[map_view_id];
+        const VAddr cpu_addr = map_range.cpu_addr;
+        const std::size_t size = map_range.size;
         ForEachCPUPage(cpu_addr, size, [this, image_id](u64 page) {
             const auto page_it = page_table.find(page);
             if (page_it == page_table.end()) {
-- 
cgit v1.2.3


From c6a9e91784957877d86c50b1cf1807265fafe65a Mon Sep 17 00:00:00 2001
From: Fernando Sahmkow <fsahmkow27@gmail.com>
Date: Sun, 4 Jul 2021 16:36:31 +0200
Subject: Texture Cache: Fix collision with multiple overlaps of the same
 sparse texture.

---
 src/video_core/texture_cache/texture_cache.h | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'src/video_core/texture_cache')

diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 8b1aa8122..e3542301e 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -1169,7 +1169,12 @@ ImageId TextureCache<P>::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, VA
     ForEachImageInRegion(cpu_addr, size_bytes, region_check);
     const auto region_check_gpu = [&](ImageId overlap_id, ImageBase& overlap) {
         if (!overlaps_found.contains(overlap_id)) {
-            ignore_textures.insert(overlap_id);
+            if (True(overlap.flags & ImageFlagBits::Remapped)) {
+                ignore_textures.insert(overlap_id);
+            }
+            if (overlap.gpu_addr == gpu_addr && overlap.guest_size_bytes == size_bytes) {
+                ignore_textures.insert(overlap_id);
+            }
         }
     };
     ForEachSparseImageInRegion(gpu_addr, size_bytes, region_check_gpu);
-- 
cgit v1.2.3