path: root/src/core/hle/kernel/k_priority_queue.h

// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
// SPDX-License-Identifier: GPL-2.0-or-later

#pragma once

#include <array>
#include <bit>
#include <concepts>

#include "common/assert.h"
#include "common/bit_set.h"
#include "common/common_types.h"
#include "common/concepts.h"

namespace Kernel {

class KThread;

template <typename T>
concept KPriorityQueueAffinityMask = !
std::is_reference_v<T>&& requires(T& t) {
                             { t.GetAffinityMask() } -> Common::ConvertibleTo<u64>;
                             { t.SetAffinityMask(0) };

                             { t.GetAffinity(0) } -> std::same_as<bool>;
                             { t.SetAffinity(0, false) };
                             { t.SetAll() };
                         };

template <typename T>
concept KPriorityQueueMember = !
std::is_reference_v<T>&& requires(T& t) {
                             { typename T::QueueEntry() };
                             { (typename T::QueueEntry()).Initialize() };
                             { (typename T::QueueEntry()).SetPrev(std::addressof(t)) };
                             { (typename T::QueueEntry()).SetNext(std::addressof(t)) };
                             { (typename T::QueueEntry()).GetNext() } -> std::same_as<T*>;
                             { (typename T::QueueEntry()).GetPrev() } -> std::same_as<T*>;
                             {
                                 t.GetPriorityQueueEntry(0)
                                 } -> std::same_as<typename T::QueueEntry&>;

                             { t.GetAffinityMask() };
                             {
                                 std::remove_cvref_t<decltype(t.GetAffinityMask())>()
                                 } -> KPriorityQueueAffinityMask;

                             { t.GetActiveCore() } -> Common::ConvertibleTo<s32>;
                             { t.GetPriority() } -> Common::ConvertibleTo<s32>;
                             { t.IsDummyThread() } -> Common::ConvertibleTo<bool>;
                         };

template <typename Member, size_t NumCores_, int LowestPriority, int HighestPriority>
    requires KPriorityQueueMember<Member>
class KPriorityQueue {
public:
    using AffinityMaskType = std::remove_cv_t<
        std::remove_reference_t<decltype(std::declval<Member>().GetAffinityMask())>>;

    static_assert(LowestPriority >= 0);
    static_assert(HighestPriority >= 0);
    static_assert(LowestPriority >= HighestPriority);
    static constexpr size_t NumPriority = LowestPriority - HighestPriority + 1;
    static constexpr size_t NumCores = NumCores_;

    static constexpr bool IsValidCore(s32 core) {
        return 0 <= core && core < static_cast<s32>(NumCores);
    }

    static constexpr bool IsValidPriority(s32 priority) {
        return HighestPriority <= priority && priority <= LowestPriority + 1;
    }

private:
    using Entry = typename Member::QueueEntry;

public:
    class KPerCoreQueue {
    private:
        std::array<Entry, NumCores> m_root{};

    public:
        constexpr KPerCoreQueue() {
            for (auto& per_core_root : m_root) {
                per_core_root.Initialize();
            }
        }

        constexpr bool PushBack(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entry associated with the end of the queue.
            Member* tail = m_root[core].GetPrev();
            Entry& tail_entry =
                (tail != nullptr) ? tail->GetPriorityQueueEntry(core) : m_root[core];

            // Link the entries.
            member_entry.SetPrev(tail);
            member_entry.SetNext(nullptr);
            tail_entry.SetNext(member);
            m_root[core].SetPrev(member);

            return tail == nullptr;
        }

        constexpr bool PushFront(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entry associated with the front of the queue.
            Member* head = m_root[core].GetNext();
            Entry& head_entry =
                (head != nullptr) ? head->GetPriorityQueueEntry(core) : m_root[core];

            // Link the entries.
            member_entry.SetPrev(nullptr);
            member_entry.SetNext(head);
            head_entry.SetPrev(member);
            m_root[core].SetNext(member);

            return (head == nullptr);
        }

        constexpr bool Remove(s32 core, Member* member) {
            // Get the entry associated with the member.
            Entry& member_entry = member->GetPriorityQueueEntry(core);

            // Get the entries associated with next and prev.
            Member* prev = member_entry.GetPrev();
            Member* next = member_entry.GetNext();
            Entry& prev_entry =
                (prev != nullptr) ? prev->GetPriorityQueueEntry(core) : m_root[core];
            Entry& next_entry =
                (next != nullptr) ? next->GetPriorityQueueEntry(core) : m_root[core];

            // Unlink.
            prev_entry.SetNext(next);
            next_entry.SetPrev(prev);

            return (this->GetFront(core) == nullptr);
        }

        constexpr Member* GetFront(s32 core) const {
            return m_root[core].GetNext();
        }
    };

    class KPriorityQueueImpl {
    public:
        constexpr KPriorityQueueImpl() = default;

        constexpr void PushBack(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].PushBack(core, member)) {
                m_available_priorities[core].SetBit(priority);
            }
        }

        constexpr void PushFront(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].PushFront(core, member)) {
                m_available_priorities[core].SetBit(priority);
            }
        }

        constexpr void Remove(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority > LowestPriority) {
                return;
            }

            if (m_queues[priority].Remove(core, member)) {
                m_available_priorities[core].ClearBit(priority);
            }
        }

        constexpr Member* GetFront(s32 core) const {
            ASSERT(IsValidCore(core));

            const s32 priority = static_cast<s32>(m_available_priorities[core].CountLeadingZero());
            if (priority <= LowestPriority) {
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

        constexpr Member* GetFront(s32 priority, s32 core) const {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

        constexpr Member* GetNext(s32 core, const Member* member) const {
            ASSERT(IsValidCore(core));

            Member* next = member->GetPriorityQueueEntry(core).GetNext();
            if (next == nullptr) {
                const s32 priority = static_cast<s32>(
                    m_available_priorities[core].GetNextSet(member->GetPriority()));
                if (priority <= LowestPriority) {
                    next = m_queues[priority].GetFront(core);
                }
            }
            return next;
        }

        constexpr void MoveToFront(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                m_queues[priority].Remove(core, member);
                m_queues[priority].PushFront(core, member);
            }
        }

        constexpr Member* MoveToBack(s32 priority, s32 core, Member* member) {
            ASSERT(IsValidCore(core));
            ASSERT(IsValidPriority(priority));

            if (priority <= LowestPriority) {
                m_queues[priority].Remove(core, member);
                m_queues[priority].PushBack(core, member);
                return m_queues[priority].GetFront(core);
            } else {
                return nullptr;
            }
        }

    private:
        std::array<KPerCoreQueue, NumPriority> m_queues{};
        std::array<Common::BitSet64<NumPriority>, NumCores> m_available_priorities{};
    };

private:
    KPriorityQueueImpl m_scheduled_queue;
    KPriorityQueueImpl m_suggested_queue;

private:
    static constexpr void ClearAffinityBit(u64& affinity, s32 core) {
        affinity &= ~(UINT64_C(1) << core);
    }

    static constexpr s32 GetNextCore(u64& affinity) {
        const s32 core = std::countr_zero(affinity);
        ClearAffinityBit(affinity, core);
        return core;
    }

    constexpr void PushBack(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Push onto the scheduled queue for its core, if we can.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.PushBack(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // And suggest the thread for all other cores.
        while (affinity) {
            m_suggested_queue.PushBack(priority, GetNextCore(affinity), member);
        }
    }

    constexpr void PushFront(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Push onto the scheduled queue for its core, if we can.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.PushFront(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // And suggest the thread for all other cores.
        // Note: Nintendo pushes onto the back of the suggested queue, not the front.
        while (affinity) {
            m_suggested_queue.PushBack(priority, GetNextCore(affinity), member);
        }
    }

    constexpr void Remove(s32 priority, Member* member) {
        ASSERT(IsValidPriority(priority));

        // Remove from the scheduled queue for its core.
        u64 affinity = member->GetAffinityMask().GetAffinityMask();
        if (const s32 core = member->GetActiveCore(); core >= 0) {
            m_scheduled_queue.Remove(priority, core, member);
            ClearAffinityBit(affinity, core);
        }

        // Remove from the suggested queue for all other cores.
        while (affinity) {
            m_suggested_queue.Remove(priority, GetNextCore(affinity), member);
        }
    }

public:
    constexpr KPriorityQueue() = default;

    // Getters.
    constexpr Member* GetScheduledFront(s32 core) const {
        return m_scheduled_queue.GetFront(core);
    }

    constexpr Member* GetScheduledFront(s32 core, s32 priority) const {
        return m_scheduled_queue.GetFront(priority, core);
    }

    constexpr Member* GetSuggestedFront(s32 core) const {
        return m_suggested_queue.GetFront(core);
    }

    constexpr Member* GetSuggestedFront(s32 core, s32 priority) const {
        return m_suggested_queue.GetFront(priority, core);
    }

    constexpr Member* GetScheduledNext(s32 core, const Member* member) const {
        return m_scheduled_queue.GetNext(core, member);
    }

    constexpr Member* GetSuggestedNext(s32 core, const Member* member) const {
        return m_suggested_queue.GetNext(core, member);
    }

    constexpr Member* GetSamePriorityNext(s32 core, const Member* member) const {
        return member->GetPriorityQueueEntry(core).GetNext();
    }

    // Mutators.
    constexpr void PushBack(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        this->PushBack(member->GetPriority(), member);
    }

    constexpr void Remove(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        this->Remove(member->GetPriority(), member);
    }

    constexpr void MoveToScheduledFront(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        m_scheduled_queue.MoveToFront(member->GetPriority(), member->GetActiveCore(), member);
    }

    constexpr KThread* MoveToScheduledBack(Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return {};
        }

        return m_scheduled_queue.MoveToBack(member->GetPriority(), member->GetActiveCore(), member);
    }

    // First class fancy operations.
    constexpr void ChangePriority(s32 prev_priority, bool is_running, Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        ASSERT(IsValidPriority(prev_priority));

        // Remove the member from the queues.
        const s32 new_priority = member->GetPriority();
        this->Remove(prev_priority, member);

        // And enqueue. If the member is running, we want to keep it running.
        if (is_running) {
            this->PushFront(new_priority, member);
        } else {
            this->PushBack(new_priority, member);
        }
    }

    constexpr void ChangeAffinityMask(s32 prev_core, const AffinityMaskType& prev_affinity,
                                      Member* member) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        // Get the new information.
        const s32 priority = member->GetPriority();
        const AffinityMaskType& new_affinity = member->GetAffinityMask();
        const s32 new_core = member->GetActiveCore();

        // Remove the member from all queues it was in before.
        for (s32 core = 0; core < static_cast<s32>(NumCores); core++) {
            if (prev_affinity.GetAffinity(core)) {
                if (core == prev_core) {
                    m_scheduled_queue.Remove(priority, core, member);
                } else {
                    m_suggested_queue.Remove(priority, core, member);
                }
            }
        }

        // And add the member to all queues it should be in now.
        for (s32 core = 0; core < static_cast<s32>(NumCores); core++) {
            if (new_affinity.GetAffinity(core)) {
                if (core == new_core) {
                    m_scheduled_queue.PushBack(priority, core, member);
                } else {
                    m_suggested_queue.PushBack(priority, core, member);
                }
            }
        }
    }

    constexpr void ChangeCore(s32 prev_core, Member* member, bool to_front = false) {
        // This is for host (dummy) threads that we do not want to enter the priority queue.
        if (member->IsDummyThread()) {
            return;
        }

        // Get the new information.
        const s32 new_core = member->GetActiveCore();
        const s32 priority = member->GetPriority();

        // We don't need to do anything if the core is the same.
        if (prev_core != new_core) {
            // Remove from the scheduled queue for the previous core.
            if (prev_core >= 0) {
                m_scheduled_queue.Remove(priority, prev_core, member);
            }

            // Remove from the suggested queue and add to the scheduled queue for the new core.
            if (new_core >= 0) {
                m_suggested_queue.Remove(priority, new_core, member);
                if (to_front) {
                    m_scheduled_queue.PushFront(priority, new_core, member);
                } else {
                    m_scheduled_queue.PushBack(priority, new_core, member);
                }
            }

            // Add to the suggested queue for the previous core.
            if (prev_core >= 0) {
                m_suggested_queue.PushBack(priority, prev_core, member);
            }
        }
    }
};

} // namespace Kernel