// Copyright 2020 yuzu Emulator Project // Licensed under GPLv2 or any later version // Refer to the license.txt file included. #pragma once #include #include #include #include #include #include #include "common/assert.h" #include "common/common_types.h" #include "video_core/rasterizer_interface.h" namespace VideoCommon { template class ShaderCache { static constexpr u64 PAGE_BITS = 14; static constexpr u64 PAGE_SIZE = u64(1) << PAGE_BITS; struct Entry { VAddr addr_start; VAddr addr_end; T* data; bool is_memory_marked = true; constexpr bool Overlaps(VAddr start, VAddr end) const noexcept { return start < addr_end && addr_start < end; } }; public: virtual ~ShaderCache() = default; /// @brief Removes shaders inside a given region /// @note Checks for ranges /// @param addr Start address of the invalidation /// @param size Number of bytes of the invalidation void InvalidateRegion(VAddr addr, std::size_t size) { std::scoped_lock lock{invalidation_mutex}; InvalidatePagesInRegion(addr, size); RemovePendingShaders(); } /// @brief Unmarks a memory region as cached and marks it for removal /// @param addr Start address of the CPU write operation /// @param size Number of bytes of the CPU write operation void OnCPUWrite(VAddr addr, std::size_t size) { std::lock_guard lock{invalidation_mutex}; InvalidatePagesInRegion(addr, size); } /// @brief Flushes delayed removal operations void SyncGuestHost() { std::scoped_lock lock{invalidation_mutex}; RemovePendingShaders(); } /// @brief Tries to obtain a cached shader starting in a given address /// @note Doesn't check for ranges, the given address has to be the start of the shader /// @param addr Start address of the shader, this doesn't cache for region /// @return Pointer to a valid shader, nullptr when nothing is found T* TryGet(VAddr addr) const { std::scoped_lock lock{lookup_mutex}; const auto it = lookup_cache.find(addr); if (it == lookup_cache.end()) { return nullptr; } return it->second->data; } protected: explicit ShaderCache(VideoCore::RasterizerInterface& rasterizer_) : rasterizer{rasterizer_} {} /// @brief Register in the cache a given entry /// @param data Shader to store in the cache /// @param addr Start address of the shader that will be registered /// @param size Size in bytes of the shader void Register(std::unique_ptr data, VAddr addr, std::size_t size) { std::scoped_lock lock{invalidation_mutex, lookup_mutex}; const VAddr addr_end = addr + size; Entry* const entry = NewEntry(addr, addr_end, data.get()); const u64 page_end = (addr_end + PAGE_SIZE - 1) >> PAGE_BITS; for (u64 page = addr >> PAGE_BITS; page < page_end; ++page) { invalidation_cache[page].push_back(entry); } storage.push_back(std::move(data)); rasterizer.UpdatePagesCachedCount(addr, size, 1); } /// @brief Called when a shader is going to be removed /// @param shader Shader that will be removed /// @pre invalidation_cache is locked /// @pre lookup_mutex is locked virtual void OnShaderRemoval([[maybe_unused]] T* shader) {} private: /// @brief Invalidate pages in a given region /// @pre invalidation_mutex is locked void InvalidatePagesInRegion(VAddr addr, std::size_t size) { const VAddr addr_end = addr + size; const u64 page_end = (addr_end + PAGE_SIZE - 1) >> PAGE_BITS; for (u64 page = addr >> PAGE_BITS; page < page_end; ++page) { auto it = invalidation_cache.find(page); if (it == invalidation_cache.end()) { continue; } InvalidatePageEntries(it->second, addr, addr_end); } } /// @brief Remove shaders marked for deletion /// @pre invalidation_mutex is locked void RemovePendingShaders() { if (marked_for_removal.empty()) { return; } // Remove duplicates std::sort(marked_for_removal.begin(), marked_for_removal.end()); marked_for_removal.erase(std::unique(marked_for_removal.begin(), marked_for_removal.end()), marked_for_removal.end()); std::vector removed_shaders; removed_shaders.reserve(marked_for_removal.size()); std::scoped_lock lock{lookup_mutex}; for (Entry* const entry : marked_for_removal) { removed_shaders.push_back(entry->data); const auto it = lookup_cache.find(entry->addr_start); ASSERT(it != lookup_cache.end()); lookup_cache.erase(it); } marked_for_removal.clear(); if (!removed_shaders.empty()) { RemoveShadersFromStorage(std::move(removed_shaders)); } } /// @brief Invalidates entries in a given range for the passed page /// @param entries Vector of entries in the page, it will be modified on overlaps /// @param addr Start address of the invalidation /// @param addr_end Non-inclusive end address of the invalidation /// @pre invalidation_mutex is locked void InvalidatePageEntries(std::vector& entries, VAddr addr, VAddr addr_end) { std::size_t index = 0; while (index < entries.size()) { Entry* const entry = entries[index]; if (!entry->Overlaps(addr, addr_end)) { ++index; continue; } UnmarkMemory(entry); RemoveEntryFromInvalidationCache(entry); marked_for_removal.push_back(entry); } } /// @brief Removes all references to an entry in the invalidation cache /// @param entry Entry to remove from the invalidation cache /// @pre invalidation_mutex is locked void RemoveEntryFromInvalidationCache(const Entry* entry) { const u64 page_end = (entry->addr_end + PAGE_SIZE - 1) >> PAGE_BITS; for (u64 page = entry->addr_start >> PAGE_BITS; page < page_end; ++page) { const auto entries_it = invalidation_cache.find(page); ASSERT(entries_it != invalidation_cache.end()); std::vector& entries = entries_it->second; const auto entry_it = std::find(entries.begin(), entries.end(), entry); ASSERT(entry_it != entries.end()); entries.erase(entry_it); } } /// @brief Unmarks an entry from the rasterizer cache /// @param entry Entry to unmark from memory void UnmarkMemory(Entry* entry) { if (!entry->is_memory_marked) { return; } entry->is_memory_marked = false; const VAddr addr = entry->addr_start; const std::size_t size = entry->addr_end - addr; rasterizer.UpdatePagesCachedCount(addr, size, -1); } /// @brief Removes a vector of shaders from a list /// @param removed_shaders Shaders to be removed from the storage /// @pre invalidation_mutex is locked /// @pre lookup_mutex is locked void RemoveShadersFromStorage(std::vector removed_shaders) { // Notify removals for (T* const shader : removed_shaders) { OnShaderRemoval(shader); } // Remove them from the cache const auto is_removed = [&removed_shaders](const std::unique_ptr& shader) { return std::find(removed_shaders.begin(), removed_shaders.end(), shader.get()) != removed_shaders.end(); }; std::erase_if(storage, is_removed); } /// @brief Creates a new entry in the lookup cache and returns its pointer /// @pre lookup_mutex is locked Entry* NewEntry(VAddr addr, VAddr addr_end, T* data) { auto entry = std::make_unique(Entry{addr, addr_end, data}); Entry* const entry_pointer = entry.get(); lookup_cache.emplace(addr, std::move(entry)); return entry_pointer; } VideoCore::RasterizerInterface& rasterizer; mutable std::mutex lookup_mutex; std::mutex invalidation_mutex; std::unordered_map> lookup_cache; std::unordered_map> invalidation_cache; std::vector> storage; std::vector marked_for_removal; }; } // namespace VideoCommon