1 files changed, 131 insertions, 154 deletions

diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/k_scheduler.h
index 68db4a5ef..535ee34b9 100644
--- a/src/core/hle/kernel/scheduler.h
+++ b/src/core/hle/kernel/k_scheduler.h
@@ -1,7 +1,10 @@
-// Copyright 2018 yuzu emulator team
+// Copyright 2020 yuzu Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
+// This file references various implementation details from Atmosphere, an open-source firmware for
+// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX.
+
 #pragma once
 
 #include <atomic>
@@ -11,8 +14,12 @@
 
 #include "common/common_types.h"
 #include "common/multi_level_queue.h"
+#include "common/scope_exit.h"
 #include "common/spin_lock.h"
+#include "core/core_timing.h"
 #include "core/hardware_properties.h"
+#include "core/hle/kernel/k_priority_queue.h"
+#include "core/hle/kernel/k_scheduler_lock.h"
 #include "core/hle/kernel/thread.h"
 
 namespace Common {
@@ -30,10 +37,16 @@ class KernelCore;
 class Process;
 class SchedulerLock;
 
-class GlobalScheduler final {
+using KSchedulerPriorityQueue =
+    KPriorityQueue<Thread, Core::Hardware::NUM_CPU_CORES, THREADPRIO_LOWEST, THREADPRIO_HIGHEST>;
+static constexpr s32 HighestCoreMigrationAllowedPriority = 2;
+
+class GlobalSchedulerContext final {
+    friend class KScheduler;
+
 public:
-    explicit GlobalScheduler(KernelCore& kernel);
-    ~GlobalScheduler();
+    explicit GlobalSchedulerContext(KernelCore& kernel);
+    ~GlobalSchedulerContext();
 
     /// Adds a new thread to the scheduler
     void AddThread(std::shared_ptr<Thread> thread);
@@ -46,60 +59,6 @@ public:
         return thread_list;
     }
 
-    /// Notify the scheduler a thread's status has changed.
-    void AdjustSchedulingOnStatus(Thread* thread, u32 old_flags);
-
-    /// Notify the scheduler a thread's priority has changed.
-    void AdjustSchedulingOnPriority(Thread* thread, u32 old_priority);
-
-    /// Notify the scheduler a thread's core and/or affinity mask has changed.
-    void AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, s32 old_core);
-
-    /**
-     * Takes care of selecting the new scheduled threads in three steps:
-     *
-     * 1. First a thread is selected from the top of the priority queue. If no thread
-     *    is obtained then we move to step two, else we are done.
-     *
-     * 2. Second we try to get a suggested thread that's not assigned to any core or
-     *    that is not the top thread in that core.
-     *
-     * 3. Third is no suggested thread is found, we do a second pass and pick a running
-     *    thread in another core and swap it with its current thread.
-     *
-     * returns the cores needing scheduling.
-     */
-    u32 SelectThreads();
-
-    bool HaveReadyThreads(std::size_t core_id) const {
-        return !scheduled_queue[core_id].empty();
-    }
-
-    /**
-     * Takes a thread and moves it to the back of the it's priority list.
-     *
-     * @note This operation can be redundant and no scheduling is changed if marked as so.
-     */
-    bool YieldThread(Thread* thread);
-
-    /**
-     * Takes a thread and moves it to the back of the it's priority list.
-     * Afterwards, tries to pick a suggested thread from the suggested queue that has worse time or
-     * a better priority than the next thread in the core.
-     *
-     * @note This operation can be redundant and no scheduling is changed if marked as so.
-     */
-    bool YieldThreadAndBalanceLoad(Thread* thread);
-
-    /**
-     * Takes a thread and moves it out of the scheduling queue.
-     * and into the suggested queue. If no thread can be scheduled afterwards in that core,
-     * a suggested thread is obtained instead.
-     *
-     * @note This operation can be redundant and no scheduling is changed if marked as so.
-     */
-    bool YieldThreadAndWaitForLoadBalancing(Thread* thread);
-
     /**
      * Rotates the scheduling queues of threads at a preemption priority and then does
      * some core rebalancing. Preemption priorities can be found in the array
@@ -113,15 +72,7 @@ public:
         return Core::Hardware::NUM_CPU_CORES;
     }
 
-    void SetReselectionPending() {
-        is_reselection_pending.store(true, std::memory_order_release);
-    }
-
-    bool IsReselectionPending() const {
-        return is_reselection_pending.load(std::memory_order_acquire);
-    }
-
-    void Shutdown();
+    bool IsLocked() const;
 
 private:
     friend class SchedulerLock;
@@ -133,109 +84,50 @@ private:
     /// and reschedules current core if needed.
     void Unlock();
 
-    void EnableInterruptAndSchedule(u32 cores_pending_reschedule,
-                                    Core::EmuThreadHandle global_thread);
-
-    /**
-     * Add a thread to the suggested queue of a cpu core. Suggested threads may be
-     * picked if no thread is scheduled to run on the core.
-     */
-    void Suggest(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Remove a thread to the suggested queue of a cpu core. Suggested threads may be
-     * picked if no thread is scheduled to run on the core.
-     */
-    void Unsuggest(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
-     * back the queue in its priority level.
-     */
-    void Schedule(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Add a thread to the scheduling queue of a cpu core. The thread is added at the
-     * front the queue in its priority level.
-     */
-    void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
-
-    /// Reschedule an already scheduled thread based on a new priority
-    void Reschedule(u32 priority, std::size_t core, Thread* thread);
-
-    /// Unschedules a thread.
-    void Unschedule(u32 priority, std::size_t core, Thread* thread);
-
-    /**
-     * Transfers a thread into an specific core. If the destination_core is -1
-     * it will be unscheduled from its source code and added into its suggested
-     * queue.
-     */
-    void TransferToCore(u32 priority, s32 destination_core, Thread* thread);
-
-    bool AskForReselectionOrMarkRedundant(Thread* current_thread, const Thread* winner);
-
-    static constexpr u32 min_regular_priority = 2;
-    std::array<Common::MultiLevelQueue<Thread*, THREADPRIO_COUNT>, Core::Hardware::NUM_CPU_CORES>
-        scheduled_queue;
-    std::array<Common::MultiLevelQueue<Thread*, THREADPRIO_COUNT>, Core::Hardware::NUM_CPU_CORES>
-        suggested_queue;
-    std::atomic<bool> is_reselection_pending{false};
-
-    // The priority levels at which the global scheduler preempts threads every 10 ms. They are
-    // ordered from Core 0 to Core 3.
-    std::array<u32, Core::Hardware::NUM_CPU_CORES> preemption_priorities = {59, 59, 59, 62};
+    using LockType = KAbstractSchedulerLock<KScheduler>;
 
-    /// Scheduler lock mechanisms.
-    bool is_locked{};
-    std::mutex inner_lock;
-    std::atomic<s64> scope_lock{};
-    Core::EmuThreadHandle current_owner{Core::EmuThreadHandle::InvalidHandle()};
+    KernelCore& kernel;
 
-    Common::SpinLock global_list_guard{};
+    std::atomic_bool scheduler_update_needed{};
+    KSchedulerPriorityQueue priority_queue;
+    LockType scheduler_lock;
 
     /// Lists all thread ids that aren't deleted/etc.
     std::vector<std::shared_ptr<Thread>> thread_list;
-    KernelCore& kernel;
+    Common::SpinLock global_list_guard{};
 };
 
-class Scheduler final {
+class KScheduler final {
 public:
-    explicit Scheduler(Core::System& system, std::size_t core_id);
-    ~Scheduler();
-
-    /// Returns whether there are any threads that are ready to run.
-    bool HaveReadyThreads() const;
+    explicit KScheduler(Core::System& system, std::size_t core_id);
+    ~KScheduler();
 
     /// Reschedules to the next available thread (call after current thread is suspended)
-    void TryDoContextSwitch();
+    void RescheduleCurrentCore();
+
+    /// Reschedules cores pending reschedule, to be called on EnableScheduling.
+    static void RescheduleCores(KernelCore& kernel, u64 cores_pending_reschedule,
+                                Core::EmuThreadHandle global_thread);
 
     /// The next two are for SingleCore Only.
     /// Unload current thread before preempting core.
     void Unload(Thread* thread);
-    void Unload();
+
     /// Reload current thread after core preemption.
     void Reload(Thread* thread);
-    void Reload();
 
     /// Gets the current running thread
     Thread* GetCurrentThread() const;
 
-    /// Gets the currently selected thread from the top of the multilevel queue
-    Thread* GetSelectedThread() const;
-
     /// Gets the timestamp for the last context switch in ticks.
     u64 GetLastContextSwitchTicks() const;
 
     bool ContextSwitchPending() const {
-        return is_context_switch_pending;
+        return this->state.needs_scheduling;
     }
 
     void Initialize();
 
-    /// Shutdowns the scheduler.
-    void Shutdown();
-
     void OnThreadStart();
 
     std::shared_ptr<Common::Fiber>& ControlContext() {
@@ -246,11 +138,90 @@ public:
         return switch_fiber;
     }
 
+    std::size_t CurrentCoreId() const {
+        return core_id;
+    }
+
+    u64 UpdateHighestPriorityThread(Thread* highest_thread);
+
+    /**
+     * Takes a thread and moves it to the back of the it's priority list.
+     *
+     * @note This operation can be redundant and no scheduling is changed if marked as so.
+     */
+    void YieldWithoutCoreMigration();
+
+    /**
+     * Takes a thread and moves it to the back of the it's priority list.
+     * Afterwards, tries to pick a suggested thread from the suggested queue that has worse time or
+     * a better priority than the next thread in the core.
+     *
+     * @note This operation can be redundant and no scheduling is changed if marked as so.
+     */
+    void YieldWithCoreMigration();
+
+    /**
+     * Takes a thread and moves it out of the scheduling queue.
+     * and into the suggested queue. If no thread can be scheduled afterwards in that core,
+     * a suggested thread is obtained instead.
+     *
+     * @note This operation can be redundant and no scheduling is changed if marked as so.
+     */
+    void YieldToAnyThread();
+
+    /// Notify the scheduler a thread's status has changed.
+    static void OnThreadStateChanged(KernelCore& kernel, Thread* thread, u32 old_state);
+
+    /// Notify the scheduler a thread's priority has changed.
+    static void OnThreadPriorityChanged(KernelCore& kernel, Thread* thread, Thread* current_thread,
+                                        u32 old_priority);
+
+    /// Notify the scheduler a thread's core and/or affinity mask has changed.
+    static void OnThreadAffinityMaskChanged(KernelCore& kernel, Thread* thread,
+                                            const KAffinityMask& old_affinity, s32 old_core);
+
+private:
+    /**
+     * Takes care of selecting the new scheduled threads in three steps:
+     *
+     * 1. First a thread is selected from the top of the priority queue. If no thread
+     *    is obtained then we move to step two, else we are done.
+     *
+     * 2. Second we try to get a suggested thread that's not assigned to any core or
+     *    that is not the top thread in that core.
+     *
+     * 3. Third is no suggested thread is found, we do a second pass and pick a running
+     *    thread in another core and swap it with its current thread.
+     *
+     * returns the cores needing scheduling.
+     */
+    static u64 UpdateHighestPriorityThreadsImpl(KernelCore& kernel);
+
+    void RotateScheduledQueue(s32 core_id, s32 priority);
+
+public:
+    static bool CanSchedule(KernelCore& kernel);
+    static bool IsSchedulerUpdateNeeded(const KernelCore& kernel);
+    static void SetSchedulerUpdateNeeded(KernelCore& kernel);
+    static void ClearSchedulerUpdateNeeded(KernelCore& kernel);
+    static void DisableScheduling(KernelCore& kernel);
+    static void EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduling,
+                                 Core::EmuThreadHandle global_thread);
+    static u64 UpdateHighestPriorityThreads(KernelCore& kernel);
+
 private:
-    friend class GlobalScheduler;
+    friend class GlobalSchedulerContext;
+
+    static KSchedulerPriorityQueue& GetPriorityQueue(KernelCore& kernel);
+
+    void Schedule() {
+        ASSERT(GetCurrentThread()->GetDisableDispatchCount() == 1);
+        this->ScheduleImpl();
+    }
 
     /// Switches the CPU's active thread context to that of the specified thread
-    void SwitchContext();
+    void ScheduleImpl();
+    void SwitchThread(Thread* next_thread);
 
     /// When a thread wakes up, it must run this through it's new scheduler
     void SwitchContextStep2();
@@ -271,22 +242,28 @@ private:
     static void OnSwitch(void* this_scheduler);
     void SwitchToCurrent();
 
-    std::shared_ptr<Thread> current_thread = nullptr;
-    std::shared_ptr<Thread> selected_thread = nullptr;
-    std::shared_ptr<Thread> current_thread_prev = nullptr;
-    std::shared_ptr<Thread> selected_thread_set = nullptr;
-    std::shared_ptr<Thread> idle_thread = nullptr;
+private:
+    Thread* current_thread{};
+    Thread* idle_thread{};
+
+    std::shared_ptr<Common::Fiber> switch_fiber{};
 
-    std::shared_ptr<Common::Fiber> switch_fiber = nullptr;
+    struct SchedulingState {
+        std::atomic<bool> needs_scheduling;
+        bool interrupt_task_thread_runnable{};
+        bool should_count_idle{};
+        u64 idle_count{};
+        Thread* highest_priority_thread{};
+        void* idle_thread_stack{};
+    };
+
+    SchedulingState state;
 
     Core::System& system;
-    u64 last_context_switch_time = 0;
-    u64 idle_selection_count = 0;
+    u64 last_context_switch_time{};
     const std::size_t core_id;
 
     Common::SpinLock guard{};
-
-    bool is_context_switch_pending = false;
 };
 
 class SchedulerLock {