10 files changed, 200 insertions, 52 deletions

diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp
index 51c40f620..122c1d5e1 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp
@@ -3,9 +3,11 @@
 // SPDX-License-Identifier: GPL-3.0-or-later Licensed under GPLv3
 // or any later version Refer to the license.txt file included.
 
+#include <bit>
 #include <cstdlib>
 #include <cstring>
 
+#include <fmt/format.h>
 #include "common/assert.h"
 #include "common/logging/log.h"
 #include "common/scope_exit.h"
@@ -22,8 +24,19 @@ namespace Service::Nvidia::Devices {
 nvhost_ctrl::nvhost_ctrl(Core::System& system_, EventInterface& events_interface_,
                          NvCore::Container& core_)
     : nvdevice{system_}, events_interface{events_interface_}, core{core_},
-      syncpoint_manager{core_.GetSyncpointManager()} {}
-nvhost_ctrl::~nvhost_ctrl() = default;
+      syncpoint_manager{core_.GetSyncpointManager()} {
+    events_interface.RegisterForSignal(this);
+}
+
+nvhost_ctrl::~nvhost_ctrl() {
+    events_interface.UnregisterForSignal(this);
+    for (auto& event : events) {
+        if (!event.registered) {
+            continue;
+        }
+        events_interface.FreeEvent(event.kevent);
+    }
+}
 
 NvResult nvhost_ctrl::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input,
                              std::vector<u8>& output) {
@@ -87,7 +100,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
     SCOPE_EXIT({
         std::memcpy(output.data(), &params, sizeof(params));
         if (must_unmark_fail) {
-            events_interface.fails[event_id] = 0;
+            events[event_id].fails = 0;
         }
     });
 
@@ -116,12 +129,12 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
     auto& gpu = system.GPU();
     const u32 target_value = params.fence.value;
 
-    auto lock = events_interface.Lock();
+    auto lock = NvEventsLock();
 
     u32 slot = [&]() {
         if (is_allocation) {
             params.value.raw = 0;
-            return events_interface.FindFreeEvent(fence_id);
+            return FindFreeNvEvent(fence_id);
         } else {
             return params.value.raw;
         }
@@ -130,7 +143,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
     must_unmark_fail = true;
 
     const auto check_failing = [&]() {
-        if (events_interface.fails[slot] > 2) {
+        if (events[slot].fails > 2) {
             {
                 auto lk = system.StallProcesses();
                 gpu.WaitFence(fence_id, target_value);
@@ -142,6 +155,10 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
         return false;
     };
 
+    if (slot >= MaxNvEvents) {
+        return NvResult::BadParameter;
+    }
+
     if (params.timeout == 0) {
         if (check_failing()) {
             return NvResult::Success;
@@ -149,17 +166,13 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
         return NvResult::Timeout;
     }
 
-    if (slot >= MaxNvEvents) {
-        return NvResult::BadParameter;
-    }
-
-    auto* event = events_interface.events[slot];
+    auto& event = events[slot];
 
-    if (!event) {
+    if (!event.registered) {
         return NvResult::BadParameter;
     }
 
-    if (events_interface.IsBeingUsed(slot)) {
+    if (event.IsBeingUsed()) {
         return NvResult::BadParameter;
     }
 
@@ -169,9 +182,9 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector
 
     params.value.raw = 0;
 
-    events_interface.status[slot].store(EventState::Waiting, std::memory_order_release);
-    events_interface.assigned_syncpt[slot] = fence_id;
-    events_interface.assigned_value[slot] = target_value;
+    event.status.store(EventState::Waiting, std::memory_order_release);
+    event.assigned_syncpt = fence_id;
+    event.assigned_value = target_value;
     if (is_allocation) {
         params.value.syncpoint_id_for_allocation.Assign(static_cast<u16>(fence_id));
         params.value.event_allocated.Assign(1);
@@ -189,15 +202,17 @@ NvResult nvhost_ctrl::FreeEvent(u32 slot) {
         return NvResult::BadParameter;
     }
 
-    if (!events_interface.registered[slot]) {
+    auto& event = events[slot];
+
+    if (!event.registered) {
         return NvResult::Success;
     }
 
-    if (events_interface.IsBeingUsed(slot)) {
+    if (event.IsBeingUsed()) {
         return NvResult::Busy;
     }
 
-    events_interface.Free(slot);
+    FreeNvEvent(slot);
     return NvResult::Success;
 }
 
@@ -210,15 +225,15 @@ NvResult nvhost_ctrl::IocCtrlEventRegister(const std::vector<u8>& input, std::ve
         return NvResult::BadParameter;
     }
 
-    auto lock = events_interface.Lock();
+    auto lock = NvEventsLock();
 
-    if (events_interface.registered[event_id]) {
+    if (events[event_id].registered) {
         const auto result = FreeEvent(event_id);
         if (result != NvResult::Success) {
             return result;
         }
     }
-    events_interface.Create(event_id);
+    CreateNvEvent(event_id);
     return NvResult::Success;
 }
 
@@ -229,7 +244,7 @@ NvResult nvhost_ctrl::IocCtrlEventUnregister(const std::vector<u8>& input,
     const u32 event_id = params.user_event_id & 0x00FF;
     LOG_DEBUG(Service_NVDRV, " called, user_event_id: {:X}", event_id);
 
-    auto lock = events_interface.Lock();
+    auto lock = NvEventsLock();
     return FreeEvent(event_id);
 }
 
@@ -244,44 +259,121 @@ NvResult nvhost_ctrl::IocCtrlClearEventWait(const std::vector<u8>& input, std::v
         return NvResult::BadParameter;
     }
 
-    auto lock = events_interface.Lock();
+    auto lock = NvEventsLock();
 
-    if (events_interface.status[event_id].exchange(
-            EventState::Cancelling, std::memory_order_acq_rel) == EventState::Waiting) {
-        system.GPU().CancelSyncptInterrupt(events_interface.assigned_syncpt[event_id],
-                                           events_interface.assigned_value[event_id]);
-        syncpoint_manager.RefreshSyncpoint(events_interface.assigned_syncpt[event_id]);
+    auto& event = events[event_id];
+    if (event.status.exchange(EventState::Cancelling, std::memory_order_acq_rel) ==
+        EventState::Waiting) {
+        system.GPU().CancelSyncptInterrupt(event.assigned_syncpt, event.assigned_value);
+        syncpoint_manager.RefreshSyncpoint(event.assigned_syncpt);
     }
-    events_interface.fails[event_id]++;
-    events_interface.status[event_id].store(EventState::Cancelled, std::memory_order_release);
-    events_interface.events[event_id]->GetWritableEvent().Clear();
+    event.fails++;
+    event.status.store(EventState::Cancelled, std::memory_order_release);
+    event.kevent->GetWritableEvent().Clear();
 
     return NvResult::Success;
 }
 
 Kernel::KEvent* nvhost_ctrl::QueryEvent(u32 event_id) {
-    const auto event = SyncpointEventValue{.raw = event_id};
+    const auto desired_event = SyncpointEventValue{.raw = event_id};
 
-    const bool allocated = event.event_allocated.Value() != 0;
-    const u32 slot{allocated ? event.partial_slot.Value() : static_cast<u32>(event.slot)};
+    const bool allocated = desired_event.event_allocated.Value() != 0;
+    const u32 slot{allocated ? desired_event.partial_slot.Value()
+                             : static_cast<u32>(desired_event.slot)};
     if (slot >= MaxNvEvents) {
         ASSERT(false);
         return nullptr;
     }
 
-    const u32 syncpoint_id{allocated ? event.syncpoint_id_for_allocation.Value()
-                                     : event.syncpoint_id.Value()};
+    const u32 syncpoint_id{allocated ? desired_event.syncpoint_id_for_allocation.Value()
+                                     : desired_event.syncpoint_id.Value()};
 
-    auto lock = events_interface.Lock();
+    auto lock = NvEventsLock();
 
-    if (events_interface.registered[slot] &&
-        events_interface.assigned_syncpt[slot] == syncpoint_id) {
-        ASSERT(events_interface.events[slot]);
-        return events_interface.events[slot];
+    auto& event = events[slot];
+    if (event.registered && event.assigned_syncpt == syncpoint_id) {
+        ASSERT(event.kevent);
+        return event.kevent;
     }
     // Is this possible in hardware?
     ASSERT_MSG(false, "Slot:{}, SyncpointID:{}, requested", slot, syncpoint_id);
     return nullptr;
 }
 
+std::unique_lock<std::mutex> nvhost_ctrl::NvEventsLock() {
+    return std::unique_lock<std::mutex>(events_mutex);
+}
+
+void nvhost_ctrl::CreateNvEvent(u32 event_id) {
+    auto& event = events[event_id];
+    ASSERT(!event.kevent);
+    ASSERT(!event.registered);
+    ASSERT(!event.IsBeingUsed());
+    event.kevent = events_interface.CreateEvent(fmt::format("NVCTRL::NvEvent_{}", event_id));
+    event.status = EventState::Available;
+    event.registered = true;
+    const u64 mask = 1ULL << event_id;
+    event.fails = 0;
+    events_mask |= mask;
+    event.assigned_syncpt = 0;
+}
+
+void nvhost_ctrl::FreeNvEvent(u32 event_id) {
+    auto& event = events[event_id];
+    ASSERT(event.kevent);
+    ASSERT(event.registered);
+    ASSERT(!event.IsBeingUsed());
+    events_interface.FreeEvent(event.kevent);
+    event.kevent = nullptr;
+    event.status = EventState::Available;
+    event.registered = false;
+    const u64 mask = ~(1ULL << event_id);
+    events_mask &= mask;
+}
+
+u32 nvhost_ctrl::FindFreeNvEvent(u32 syncpoint_id) {
+    u32 slot{MaxNvEvents};
+    u32 free_slot{MaxNvEvents};
+    for (u32 i = 0; i < MaxNvEvents; i++) {
+        auto& event = events[i];
+        if (event.registered) {
+            if (!event.IsBeingUsed()) {
+                slot = i;
+                if (event.assigned_syncpt == syncpoint_id) {
+                    return slot;
+                }
+            }
+        } else if (free_slot == MaxNvEvents) {
+            free_slot = i;
+        }
+    }
+    if (free_slot < MaxNvEvents) {
+        CreateNvEvent(free_slot);
+        return free_slot;
+    }
+
+    if (slot < MaxNvEvents) {
+        return slot;
+    }
+
+    LOG_CRITICAL(Service_NVDRV, "Failed to allocate an event");
+    return 0;
+}
+
+void nvhost_ctrl::SignalNvEvent(u32 syncpoint_id, u32 value) {
+    const u32 max = MaxNvEvents - std::countl_zero(events_mask);
+    const u32 min = std::countr_zero(events_mask);
+    for (u32 i = min; i < max; i++) {
+        auto& event = events[i];
+        if (event.assigned_syncpt != syncpoint_id || event.assigned_value != value) {
+            continue;
+        }
+        if (event.status.exchange(EventState::Signalling, std::memory_order_acq_rel) ==
+            EventState::Waiting) {
+            event.kevent->GetWritableEvent().Signal();
+        }
+        event.status.store(EventState::Signalled, std::memory_order_release);
+    }
+}
+
 } // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h
index 9fd46ea5f..f2fc5d047 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h
@@ -53,7 +53,49 @@ public:
     };
     static_assert(sizeof(SyncpointEventValue) == sizeof(u32));
 
+    void SignalNvEvent(u32 syncpoint_id, u32 value);
+
 private:
+    struct InternalEvent {
+        // Mask representing registered events
+
+        // Each kernel event associated to an NV event
+        Kernel::KEvent* kevent{};
+        // The status of the current NVEvent
+        std::atomic<EventState> status{};
+
+        // Tells the NVEvent that it has failed.
+        u32 fails{};
+        // When an NVEvent is waiting on GPU interrupt, this is the sync_point
+        // associated with it.
+        u32 assigned_syncpt{};
+        // This is the value of the GPU interrupt for which the NVEvent is waiting
+        // for.
+        u32 assigned_value{};
+
+        // Tells if an NVEvent is registered or not
+        bool registered{};
+
+        bool IsBeingUsed() {
+            const auto current_status = status.load(std::memory_order_acquire);
+            return current_status == EventState::Waiting ||
+                   current_status == EventState::Cancelling ||
+                   current_status == EventState::Signalling;
+        }
+    };
+
+    std::unique_lock<std::mutex> NvEventsLock();
+
+    void CreateNvEvent(u32 event_id);
+
+    void FreeNvEvent(u32 event_id);
+
+    u32 FindFreeNvEvent(u32 syncpoint_id);
+
+    std::array<InternalEvent, MaxNvEvents> events{};
+    std::mutex events_mutex;
+    u64 events_mask{};
+
     struct IocSyncptReadParams {
         u32_le id{};
         u32_le value{};
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
index e353408eb..ced57dfe6 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
@@ -13,10 +13,13 @@ namespace Service::Nvidia::Devices {
 
 nvhost_ctrl_gpu::nvhost_ctrl_gpu(Core::System& system_, EventInterface& events_interface_)
     : nvdevice{system_}, events_interface{events_interface_} {
-    error_notifier_event = events_interface.CreateNonCtrlEvent("CtrlGpuErrorNotifier");
-    unknown_event = events_interface.CreateNonCtrlEvent("CtrlGpuUknownEvent");
+    error_notifier_event = events_interface.CreateEvent("CtrlGpuErrorNotifier");
+    unknown_event = events_interface.CreateEvent("CtrlGpuUknownEvent");
+}
+nvhost_ctrl_gpu::~nvhost_ctrl_gpu() {
+    events_interface.FreeEvent(error_notifier_event);
+    events_interface.FreeEvent(unknown_event);
 }
-nvhost_ctrl_gpu::~nvhost_ctrl_gpu() = default;
 
 NvResult nvhost_ctrl_gpu::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input,
                                  std::vector<u8>& output) {
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
index e7921ade2..cb54ee5a4 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp
@@ -30,13 +30,17 @@ nvhost_gpu::nvhost_gpu(Core::System& system_, EventInterface& events_interface_,
     channel_fence.id = syncpoint_manager.AllocateSyncpoint();
     channel_fence.value = system_.GPU().GetSyncpointValue(channel_fence.id);
     sm_exception_breakpoint_int_report_event =
-        events_interface.CreateNonCtrlEvent("GpuChannelSMExceptionBreakpointInt");
+        events_interface.CreateEvent("GpuChannelSMExceptionBreakpointInt");
     sm_exception_breakpoint_pause_report_event =
-        events_interface.CreateNonCtrlEvent("GpuChannelSMExceptionBreakpointPause");
-    error_notifier_event = events_interface.CreateNonCtrlEvent("GpuChannelErrorNotifier");
+        events_interface.CreateEvent("GpuChannelSMExceptionBreakpointPause");
+    error_notifier_event = events_interface.CreateEvent("GpuChannelErrorNotifier");
 }
 
-nvhost_gpu::~nvhost_gpu() = default;
+nvhost_gpu::~nvhost_gpu() {
+    events_interface.FreeEvent(sm_exception_breakpoint_int_report_event);
+    events_interface.FreeEvent(sm_exception_breakpoint_pause_report_event);
+    events_interface.FreeEvent(error_notifier_event);
+}
 
 NvResult nvhost_gpu::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input,
                             std::vector<u8>& output) {
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp
index aa1a00832..00947ea19 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp
@@ -10,6 +10,8 @@
 
 namespace Service::Nvidia::Devices {
 
+u32 nvhost_nvdec::next_id{};
+
 nvhost_nvdec::nvhost_nvdec(Core::System& system_, NvCore::Container& core)
     : nvhost_nvdec_common{system_, core} {}
 nvhost_nvdec::~nvhost_nvdec() = default;
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h
index fef4b3216..3261ce1d4 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h
@@ -24,7 +24,7 @@ public:
     void OnClose(DeviceFD fd) override;
 
 private:
-    u32 next_id{};
+    static u32 next_id;
 };
 
 } // namespace Service::Nvidia::Devices
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp
index e76c9e5ed..77e6a1cd6 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp
@@ -45,6 +45,8 @@ std::size_t WriteVectors(std::vector<u8>& dst, const std::vector<T>& src, std::s
 }
 } // Anonymous namespace
 
+std::unordered_map<DeviceFD, u32> nvhost_nvdec_common::fd_to_id{};
+
 nvhost_nvdec_common::nvhost_nvdec_common(Core::System& system_, NvCore::Container& core_)
     : nvdevice{system_}, core{core_},
       syncpoint_manager{core.GetSyncpointManager()}, nvmap{core.GetNvMapFile()} {}
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h
index 74231d5c5..53029af6a 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h
@@ -115,7 +115,7 @@ protected:
 
     Kernel::KEvent* QueryEvent(u32 event_id) override;
 
-    std::unordered_map<DeviceFD, u32> fd_to_id{};
+    static std::unordered_map<DeviceFD, u32> fd_to_id;
     s32_le nvmap_fd{};
     u32_le submit_timeout{};
     NvCore::Container& core;
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp b/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp
index 358e89aa8..c89ff6b27 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp
+++ b/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp
@@ -8,6 +8,9 @@
 #include "video_core/renderer_base.h"
 
 namespace Service::Nvidia::Devices {
+
+u32 nvhost_vic::next_id{};
+
 nvhost_vic::nvhost_vic(Core::System& system_, NvCore::Container& core)
     : nvhost_nvdec_common{system_, core} {}
 
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_vic.h b/src/core/hle/service/nvdrv/devices/nvhost_vic.h
index 252b1e6f2..59e23b41e 100644
--- a/src/core/hle/service/nvdrv/devices/nvhost_vic.h
+++ b/src/core/hle/service/nvdrv/devices/nvhost_vic.h
@@ -23,6 +23,6 @@ public:
     void OnClose(DeviceFD fd) override;
 
 private:
-    u32 next_id{};
+    static u32 next_id;
 };
 } // namespace Service::Nvidia::Devices