diff options
Diffstat (limited to 'src/core/hle/kernel/kernel.cpp')
| -rw-r--r-- | src/core/hle/kernel/kernel.cpp | 442 |
1 files changed, 229 insertions, 213 deletions
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 49c0714ed..9251f29ad 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -1,6 +1,5 @@ -// Copyright 2021 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. +// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later #include <array> #include <atomic> @@ -18,13 +17,10 @@ #include "common/thread.h" #include "common/thread_worker.h" #include "core/arm/arm_interface.h" -#include "core/arm/cpu_interrupt_handler.h" #include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/core_timing.h" -#include "core/core_timing_util.h" #include "core/cpu_manager.h" -#include "core/device_memory.h" #include "core/hardware_properties.h" #include "core/hle/kernel/init/init_slab_setup.h" #include "core/hle/kernel/k_client_port.h" @@ -35,7 +31,6 @@ #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_shared_memory.h" -#include "core/hle/kernel/k_slab_heap.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_worker_task_manager.h" #include "core/hle/kernel/kernel.h" @@ -52,42 +47,41 @@ namespace Kernel { struct KernelCore::Impl { explicit Impl(Core::System& system_, KernelCore& kernel_) - : time_manager{system_}, object_list_container{kernel_}, - service_threads_manager{1, "yuzu:ServiceThreadsManager"}, system{system_} {} + : time_manager{system_}, + service_threads_manager{1, "ServiceThreadsManager"}, system{system_} {} void SetMulticore(bool is_multi) { is_multicore = is_multi; } void Initialize(KernelCore& kernel) { + global_object_list_container = std::make_unique<KAutoObjectWithListContainer>(kernel); global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel); global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel); global_handle_table->Initialize(KHandleTable::MaxTableSize); + default_service_thread = CreateServiceThread(kernel, "DefaultServiceThread"); is_phantom_mode_for_singlecore = false; - InitializePhysicalCores(); - // Derive the initial memory layout from the emulated board Init::InitializeSlabResourceCounts(kernel); - KMemoryLayout memory_layout; - DeriveInitialMemoryLayout(memory_layout); - Init::InitializeSlabHeaps(system, memory_layout); + DeriveInitialMemoryLayout(); + Init::InitializeSlabHeaps(system, *memory_layout); // Initialize kernel memory and resources. - InitializeSystemResourceLimit(kernel, system.CoreTiming(), memory_layout); - InitializeMemoryLayout(memory_layout); - InitializePageSlab(); - InitializeSchedulers(); - InitializeSuspendThreads(); + InitializeSystemResourceLimit(kernel, system.CoreTiming()); + InitializeMemoryLayout(); + Init::InitializeKPageBufferSlabHeap(system); + InitializeShutdownThreads(); InitializePreemption(kernel); + InitializePhysicalCores(); RegisterHostThread(); } void InitializeCores() { for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { - cores[core_id].Initialize(current_process->Is64BitProcess()); + cores[core_id]->Initialize((*current_process).Is64BitProcess()); system.Memory().SetCurrentPageTable(*current_process, core_id); } } @@ -98,39 +92,16 @@ struct KernelCore::Impl { process_list.clear(); - // Close all open server ports. - std::unordered_set<KServerPort*> server_ports_; - { - std::lock_guard lk(server_ports_lock); - server_ports_ = server_ports; - server_ports.clear(); - } - for (auto* server_port : server_ports_) { - server_port->Close(); - } - // Close all open server sessions. - std::unordered_set<KServerSession*> server_sessions_; - { - std::lock_guard lk(server_sessions_lock); - server_sessions_ = server_sessions; - server_sessions.clear(); - } - for (auto* server_session : server_sessions_) { - server_session->Close(); - } - - // Ensure that the object list container is finalized and properly shutdown. - object_list_container.Finalize(); - - // Ensures all service threads gracefully shutdown. - ClearServiceThreads(); + CloseServices(); next_object_id = 0; next_kernel_process_id = KProcess::InitialKIPIDMin; next_user_process_id = KProcess::ProcessIDMin; next_thread_id = 1; - cores.clear(); + for (auto& core : cores) { + core = nullptr; + } global_handle_table->Finalize(); global_handle_table.reset(); @@ -155,15 +126,15 @@ struct KernelCore::Impl { CleanupObject(font_shared_mem); CleanupObject(irs_shared_mem); CleanupObject(time_shared_mem); + CleanupObject(hidbus_shared_mem); CleanupObject(system_resource_limit); for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { - if (suspend_threads[core_id]) { - suspend_threads[core_id]->Close(); - suspend_threads[core_id] = nullptr; + if (shutdown_threads[core_id]) { + shutdown_threads[core_id]->Close(); + shutdown_threads[core_id] = nullptr; } - schedulers[core_id]->Finalize(); schedulers[core_id].reset(); } @@ -172,7 +143,7 @@ struct KernelCore::Impl { // Close kernel objects that were not freed on shutdown { - std::lock_guard lk(registered_in_use_objects_lock); + std::scoped_lock lk{registered_in_use_objects_lock}; if (registered_in_use_objects.size()) { for (auto& object : registered_in_use_objects) { object->Close(); @@ -183,48 +154,76 @@ struct KernelCore::Impl { // Shutdown all processes. if (current_process) { - current_process->Finalize(); + (*current_process).Finalize(); // current_process->Close(); // TODO: The current process should be destroyed based on accurate ref counting after // calling Close(). Adding a manual Destroy() call instead to avoid a memory leak. - current_process->Destroy(); + (*current_process).Destroy(); current_process = nullptr; } // Track kernel objects that were not freed on shutdown { - std::lock_guard lk(registered_objects_lock); + std::scoped_lock lk{registered_objects_lock}; if (registered_objects.size()) { - LOG_WARNING(Kernel, "{} kernel objects were dangling on shutdown!", - registered_objects.size()); + LOG_DEBUG(Kernel, "{} kernel objects were dangling on shutdown!", + registered_objects.size()); registered_objects.clear(); } } + + // Ensure that the object list container is finalized and properly shutdown. + global_object_list_container->Finalize(); + global_object_list_container.reset(); + } + + void CloseServices() { + // Close all open server sessions and ports. + std::unordered_set<KAutoObject*> server_objects_; + { + std::scoped_lock lk(server_objects_lock); + server_objects_ = server_objects; + server_objects.clear(); + } + for (auto* server_object : server_objects_) { + server_object->Close(); + } + + // Ensures all service threads gracefully shutdown. + ClearServiceThreads(); } void InitializePhysicalCores() { exclusive_monitor = Core::MakeExclusiveMonitor(system.Memory(), Core::Hardware::NUM_CPU_CORES); for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { - schedulers[i] = std::make_unique<Kernel::KScheduler>(system, i); - cores.emplace_back(i, system, *schedulers[i], interrupts); - } - } + const s32 core{static_cast<s32>(i)}; - void InitializeSchedulers() { - for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { - cores[i].Scheduler().Initialize(); + schedulers[i] = std::make_unique<Kernel::KScheduler>(system.Kernel()); + cores[i] = std::make_unique<Kernel::PhysicalCore>(i, system, *schedulers[i]); + + auto* main_thread{Kernel::KThread::Create(system.Kernel())}; + main_thread->SetName(fmt::format("MainThread:{}", core)); + main_thread->SetCurrentCore(core); + ASSERT(Kernel::KThread::InitializeMainThread(system, main_thread, core).IsSuccess()); + + auto* idle_thread{Kernel::KThread::Create(system.Kernel())}; + idle_thread->SetCurrentCore(core); + ASSERT(Kernel::KThread::InitializeIdleThread(system, idle_thread, core).IsSuccess()); + + schedulers[i]->Initialize(main_thread, idle_thread, core); } } // Creates the default system resource limit void InitializeSystemResourceLimit(KernelCore& kernel, - const Core::Timing::CoreTiming& core_timing, - const KMemoryLayout& memory_layout) { + const Core::Timing::CoreTiming& core_timing) { system_resource_limit = KResourceLimit::Create(system.Kernel()); system_resource_limit->Initialize(&core_timing); - const auto [total_size, kernel_size] = memory_layout.GetTotalAndKernelMemorySizes(); + const auto sizes{memory_layout->GetTotalAndKernelMemorySizes()}; + const auto total_size{sizes.first}; + const auto kernel_size{sizes.second}; // If setting the default system values fails, then something seriously wrong has occurred. ASSERT(system_resource_limit->SetLimitValue(LimitableResource::PhysicalMemory, total_size) @@ -240,37 +239,31 @@ struct KernelCore::Impl { constexpr u64 secure_applet_memory_size{4_MiB}; ASSERT(system_resource_limit->Reserve(LimitableResource::PhysicalMemory, secure_applet_memory_size)); - - // This memory seems to be reserved on hardware, but is not reserved/used by yuzu. - // Likely Horizon OS reserved memory - // TODO(ameerj): Derive the memory rather than hardcode it. - constexpr u64 unknown_reserved_memory{0x2f896000}; - ASSERT(system_resource_limit->Reserve(LimitableResource::PhysicalMemory, - unknown_reserved_memory)); } void InitializePreemption(KernelCore& kernel) { preemption_event = Core::Timing::CreateEvent( - "PreemptionCallback", [this, &kernel](std::uintptr_t, std::chrono::nanoseconds) { + "PreemptionCallback", + [this, &kernel](std::uintptr_t, s64 time, + std::chrono::nanoseconds) -> std::optional<std::chrono::nanoseconds> { { KScopedSchedulerLock lock(kernel); global_scheduler_context->PreemptThreads(); } - const auto time_interval = std::chrono::nanoseconds{std::chrono::milliseconds(10)}; - system.CoreTiming().ScheduleEvent(time_interval, preemption_event); + return std::nullopt; }); const auto time_interval = std::chrono::nanoseconds{std::chrono::milliseconds(10)}; - system.CoreTiming().ScheduleEvent(time_interval, preemption_event); + system.CoreTiming().ScheduleLoopingEvent(time_interval, time_interval, preemption_event); } - void InitializeSuspendThreads() { + void InitializeShutdownThreads() { for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { - suspend_threads[core_id] = KThread::Create(system.Kernel()); - ASSERT(KThread::InitializeHighPriorityThread(system, suspend_threads[core_id], {}, {}, + shutdown_threads[core_id] = KThread::Create(system.Kernel()); + ASSERT(KThread::InitializeHighPriorityThread(system, shutdown_threads[core_id], {}, {}, core_id) .IsSuccess()); - suspend_threads[core_id]->SetName(fmt::format("SuspendThread:{}", core_id)); + shutdown_threads[core_id]->SetName(fmt::format("SuspendThread:{}", core_id)); } } @@ -300,15 +293,16 @@ struct KernelCore::Impl { // Gets the dummy KThread for the caller, allocating a new one if this is the first time KThread* GetHostDummyThread() { - auto make_thread = [this]() { - KThread* thread = KThread::Create(system.Kernel()); + auto initialize = [this](KThread* thread) { ASSERT(KThread::InitializeDummyThread(thread).IsSuccess()); thread->SetName(fmt::format("DummyThread:{}", GetHostThreadId())); return thread; }; - thread_local KThread* saved_thread = make_thread(); - return saved_thread; + thread_local auto raw_thread = KThread(system.Kernel()); + thread_local auto thread = initialize(&raw_thread); + + return thread; } /// Registers a CPU core thread by allocating a host thread ID for it @@ -347,6 +341,8 @@ struct KernelCore::Impl { return is_shutting_down.load(std::memory_order_relaxed); } + static inline thread_local KThread* current_thread{nullptr}; + KThread* GetCurrentEmuThread() { // If we are shutting down the kernel, none of this is relevant anymore. if (IsShuttingDown()) { @@ -357,19 +353,26 @@ struct KernelCore::Impl { if (thread_id >= Core::Hardware::NUM_CPU_CORES) { return GetHostDummyThread(); } - return schedulers[thread_id]->GetCurrentThread(); + + return current_thread; + } + + void SetCurrentEmuThread(KThread* thread) { + current_thread = thread; } - void DeriveInitialMemoryLayout(KMemoryLayout& memory_layout) { + void DeriveInitialMemoryLayout() { + memory_layout = std::make_unique<KMemoryLayout>(); + // Insert the root region for the virtual memory tree, from which all other regions will // derive. - memory_layout.GetVirtualMemoryRegionTree().InsertDirectly( + memory_layout->GetVirtualMemoryRegionTree().InsertDirectly( KernelVirtualAddressSpaceBase, KernelVirtualAddressSpaceBase + KernelVirtualAddressSpaceSize - 1); // Insert the root region for the physical memory tree, from which all other regions will // derive. - memory_layout.GetPhysicalMemoryRegionTree().InsertDirectly( + memory_layout->GetPhysicalMemoryRegionTree().InsertDirectly( KernelPhysicalAddressSpaceBase, KernelPhysicalAddressSpaceBase + KernelPhysicalAddressSpaceSize - 1); @@ -386,7 +389,7 @@ struct KernelCore::Impl { if (!(kernel_region_start + KernelRegionSize - 1 <= KernelVirtualAddressSpaceLast)) { kernel_region_size = KernelVirtualAddressSpaceEnd - kernel_region_start; } - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( kernel_region_start, kernel_region_size, KMemoryRegionType_Kernel)); // Setup the code region. @@ -395,11 +398,11 @@ struct KernelCore::Impl { Common::AlignDown(code_start_virt_addr, CodeRegionAlign); constexpr VAddr code_region_end = Common::AlignUp(code_end_virt_addr, CodeRegionAlign); constexpr size_t code_region_size = code_region_end - code_region_start; - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( code_region_start, code_region_size, KMemoryRegionType_KernelCode)); // Setup board-specific device physical regions. - Init::SetupDevicePhysicalMemoryRegions(memory_layout); + Init::SetupDevicePhysicalMemoryRegions(*memory_layout); // Determine the amount of space needed for the misc region. size_t misc_region_needed_size; @@ -408,7 +411,7 @@ struct KernelCore::Impl { misc_region_needed_size = Core::Hardware::NUM_CPU_CORES * (3 * (PageSize + PageSize)); // Account for each auto-map device. - for (const auto& region : memory_layout.GetPhysicalMemoryRegionTree()) { + for (const auto& region : memory_layout->GetPhysicalMemoryRegionTree()) { if (region.HasTypeAttribute(KMemoryRegionAttr_ShouldKernelMap)) { // Check that the region is valid. ASSERT(region.GetEndAddress() != 0); @@ -433,22 +436,22 @@ struct KernelCore::Impl { // Setup the misc region. const VAddr misc_region_start = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegion( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion( misc_region_size, MiscRegionAlign, KMemoryRegionType_Kernel); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( misc_region_start, misc_region_size, KMemoryRegionType_KernelMisc)); // Setup the stack region. constexpr size_t StackRegionSize = 14_MiB; constexpr size_t StackRegionAlign = KernelAslrAlignment; const VAddr stack_region_start = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegion( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion( StackRegionSize, StackRegionAlign, KMemoryRegionType_Kernel); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( stack_region_start, StackRegionSize, KMemoryRegionType_KernelStack)); // Determine the size of the resource region. - const size_t resource_region_size = memory_layout.GetResourceRegionSizeForInit(); + const size_t resource_region_size = memory_layout->GetResourceRegionSizeForInit(); // Determine the size of the slab region. const size_t slab_region_size = @@ -465,23 +468,23 @@ struct KernelCore::Impl { Common::AlignUp(code_end_phys_addr + slab_region_size, SlabRegionAlign) - Common::AlignDown(code_end_phys_addr, SlabRegionAlign); const VAddr slab_region_start = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegion( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion( slab_region_needed_size, SlabRegionAlign, KMemoryRegionType_Kernel) + (code_end_phys_addr % SlabRegionAlign); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( slab_region_start, slab_region_size, KMemoryRegionType_KernelSlab)); // Setup the temp region. constexpr size_t TempRegionSize = 128_MiB; constexpr size_t TempRegionAlign = KernelAslrAlignment; const VAddr temp_region_start = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegion( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegion( TempRegionSize, TempRegionAlign, KMemoryRegionType_Kernel); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert(temp_region_start, TempRegionSize, - KMemoryRegionType_KernelTemp)); + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert(temp_region_start, TempRegionSize, + KMemoryRegionType_KernelTemp)); // Automatically map in devices that have auto-map attributes. - for (auto& region : memory_layout.GetPhysicalMemoryRegionTree()) { + for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) { // We only care about kernel regions. if (!region.IsDerivedFrom(KMemoryRegionType_Kernel)) { continue; @@ -508,21 +511,21 @@ struct KernelCore::Impl { const size_t map_size = Common::AlignUp(region.GetEndAddress(), PageSize) - map_phys_addr; const VAddr map_virt_addr = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard( map_size, PageSize, KMemoryRegionType_KernelMisc, PageSize); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( map_virt_addr, map_size, KMemoryRegionType_KernelMiscMappedDevice)); region.SetPairAddress(map_virt_addr + region.GetAddress() - map_phys_addr); } - Init::SetupDramPhysicalMemoryRegions(memory_layout); + Init::SetupDramPhysicalMemoryRegions(*memory_layout); // Insert a physical region for the kernel code region. - ASSERT(memory_layout.GetPhysicalMemoryRegionTree().Insert( + ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert( code_start_phys_addr, code_region_size, KMemoryRegionType_DramKernelCode)); // Insert a physical region for the kernel slab region. - ASSERT(memory_layout.GetPhysicalMemoryRegionTree().Insert( + ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert( slab_start_phys_addr, slab_region_size, KMemoryRegionType_DramKernelSlab)); // Determine size available for kernel page table heaps, requiring > 8 MB. @@ -531,12 +534,12 @@ struct KernelCore::Impl { ASSERT(page_table_heap_size / 4_MiB > 2); // Insert a physical region for the kernel page table heap region - ASSERT(memory_layout.GetPhysicalMemoryRegionTree().Insert( + ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert( slab_end_phys_addr, page_table_heap_size, KMemoryRegionType_DramKernelPtHeap)); // All DRAM regions that we haven't tagged by this point will be mapped under the linear // mapping. Tag them. - for (auto& region : memory_layout.GetPhysicalMemoryRegionTree()) { + for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) { if (region.GetType() == KMemoryRegionType_Dram) { // Check that the region is valid. ASSERT(region.GetEndAddress() != 0); @@ -548,7 +551,7 @@ struct KernelCore::Impl { // Get the linear region extents. const auto linear_extents = - memory_layout.GetPhysicalMemoryRegionTree().GetDerivedRegionExtents( + memory_layout->GetPhysicalMemoryRegionTree().GetDerivedRegionExtents( KMemoryRegionAttr_LinearMapped); ASSERT(linear_extents.GetEndAddress() != 0); @@ -560,7 +563,7 @@ struct KernelCore::Impl { Common::AlignUp(linear_extents.GetEndAddress(), LinearRegionAlign) - aligned_linear_phys_start; const VAddr linear_region_start = - memory_layout.GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard( + memory_layout->GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard( linear_region_size, LinearRegionAlign, KMemoryRegionType_None, LinearRegionAlign); const u64 linear_region_phys_to_virt_diff = linear_region_start - aligned_linear_phys_start; @@ -569,7 +572,7 @@ struct KernelCore::Impl { { PAddr cur_phys_addr = 0; u64 cur_size = 0; - for (auto& region : memory_layout.GetPhysicalMemoryRegionTree()) { + for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) { if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) { continue; } @@ -588,55 +591,49 @@ struct KernelCore::Impl { const VAddr region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff; - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType()))); region.SetPairAddress(region_virt_addr); KMemoryRegion* virt_region = - memory_layout.GetVirtualMemoryRegionTree().FindModifiable(region_virt_addr); + memory_layout->GetVirtualMemoryRegionTree().FindModifiable(region_virt_addr); ASSERT(virt_region != nullptr); virt_region->SetPairAddress(region.GetAddress()); } } // Insert regions for the initial page table region. - ASSERT(memory_layout.GetPhysicalMemoryRegionTree().Insert( + ASSERT(memory_layout->GetPhysicalMemoryRegionTree().Insert( resource_end_phys_addr, KernelPageTableHeapSize, KMemoryRegionType_DramKernelInitPt)); - ASSERT(memory_layout.GetVirtualMemoryRegionTree().Insert( + ASSERT(memory_layout->GetVirtualMemoryRegionTree().Insert( resource_end_phys_addr + linear_region_phys_to_virt_diff, KernelPageTableHeapSize, KMemoryRegionType_VirtualDramKernelInitPt)); // All linear-mapped DRAM regions that we haven't tagged by this point will be allocated to // some pool partition. Tag them. - for (auto& region : memory_layout.GetPhysicalMemoryRegionTree()) { + for (auto& region : memory_layout->GetPhysicalMemoryRegionTree()) { if (region.GetType() == (KMemoryRegionType_Dram | KMemoryRegionAttr_LinearMapped)) { region.SetType(KMemoryRegionType_DramPoolPartition); } } // Setup all other memory regions needed to arrange the pool partitions. - Init::SetupPoolPartitionMemoryRegions(memory_layout); + Init::SetupPoolPartitionMemoryRegions(*memory_layout); // Cache all linear regions in their own trees for faster access, later. - memory_layout.InitializeLinearMemoryRegionTrees(aligned_linear_phys_start, - linear_region_start); + memory_layout->InitializeLinearMemoryRegionTrees(aligned_linear_phys_start, + linear_region_start); } - void InitializeMemoryLayout(const KMemoryLayout& memory_layout) { - const auto system_pool = memory_layout.GetKernelSystemPoolRegionPhysicalExtents(); - const auto applet_pool = memory_layout.GetKernelAppletPoolRegionPhysicalExtents(); - const auto application_pool = memory_layout.GetKernelApplicationPoolRegionPhysicalExtents(); + void InitializeMemoryLayout() { + const auto system_pool = memory_layout->GetKernelSystemPoolRegionPhysicalExtents(); - // Initialize memory managers + // Initialize the memory manager. memory_manager = std::make_unique<KMemoryManager>(system); - memory_manager->InitializeManager(KMemoryManager::Pool::Application, - application_pool.GetAddress(), - application_pool.GetEndAddress()); - memory_manager->InitializeManager(KMemoryManager::Pool::Applet, applet_pool.GetAddress(), - applet_pool.GetEndAddress()); - memory_manager->InitializeManager(KMemoryManager::Pool::System, system_pool.GetAddress(), - system_pool.GetEndAddress()); + const auto& management_region = memory_layout->GetPoolManagementRegion(); + ASSERT(management_region.GetEndAddress() != 0); + memory_manager->Initialize(management_region.GetAddress(), management_region.GetSize()); // Setup memory regions for emulated processes // TODO(bunnei): These should not be hardcoded regions initialized within the kernel @@ -644,16 +641,20 @@ struct KernelCore::Impl { constexpr std::size_t font_size{0x1100000}; constexpr std::size_t irs_size{0x8000}; constexpr std::size_t time_size{0x1000}; + constexpr std::size_t hidbus_size{0x1000}; const PAddr hid_phys_addr{system_pool.GetAddress()}; const PAddr font_phys_addr{system_pool.GetAddress() + hid_size}; const PAddr irs_phys_addr{system_pool.GetAddress() + hid_size + font_size}; const PAddr time_phys_addr{system_pool.GetAddress() + hid_size + font_size + irs_size}; + const PAddr hidbus_phys_addr{system_pool.GetAddress() + hid_size + font_size + irs_size + + time_size}; hid_shared_mem = KSharedMemory::Create(system.Kernel()); font_shared_mem = KSharedMemory::Create(system.Kernel()); irs_shared_mem = KSharedMemory::Create(system.Kernel()); time_shared_mem = KSharedMemory::Create(system.Kernel()); + hidbus_shared_mem = KSharedMemory::Create(system.Kernel()); hid_shared_mem->Initialize(system.DeviceMemory(), nullptr, {hid_phys_addr, hid_size / PageSize}, @@ -671,22 +672,10 @@ struct KernelCore::Impl { {time_phys_addr, time_size / PageSize}, Svc::MemoryPermission::None, Svc::MemoryPermission::Read, time_phys_addr, time_size, "Time:SharedMemory"); - } - - void InitializePageSlab() { - // Allocate slab heaps - user_slab_heap_pages = - std::make_unique<KSlabHeap<Page>>(KSlabHeap<Page>::AllocationType::Guest); - - // TODO(ameerj): This should be derived, not hardcoded within the kernel - constexpr u64 user_slab_heap_size{0x3de000}; - // Reserve slab heaps - ASSERT( - system_resource_limit->Reserve(LimitableResource::PhysicalMemory, user_slab_heap_size)); - // Initialize slab heap - user_slab_heap_pages->Initialize( - system.DeviceMemory().GetPointer(Core::DramMemoryMap::SlabHeapBase), - user_slab_heap_size); + hidbus_shared_mem->Initialize(system.DeviceMemory(), nullptr, + {hidbus_phys_addr, hidbus_size / PageSize}, + Svc::MemoryPermission::None, Svc::MemoryPermission::Read, + hidbus_phys_addr, hidbus_size, "HidBus:SharedMemory"); } KClientPort* CreateNamedServicePort(std::string name) { @@ -697,13 +686,20 @@ struct KernelCore::Impl { } KClientPort* port = &search->second(system.ServiceManager(), system); - { - std::lock_guard lk(server_ports_lock); - server_ports.insert(&port->GetParent()->GetServerPort()); - } + RegisterServerObject(&port->GetParent()->GetServerPort()); return port; } + void RegisterServerObject(KAutoObject* server_object) { + std::scoped_lock lk(server_objects_lock); + server_objects.insert(server_object); + } + + void UnregisterServerObject(KAutoObject* server_object) { + std::scoped_lock lk(server_objects_lock); + server_objects.erase(server_object); + } + std::weak_ptr<Kernel::ServiceThread> CreateServiceThread(KernelCore& kernel, const std::string& name) { auto service_thread = std::make_shared<Kernel::ServiceThread>(kernel, 1, name); @@ -716,6 +712,12 @@ struct KernelCore::Impl { void ReleaseServiceThread(std::weak_ptr<Kernel::ServiceThread> service_thread) { if (auto strong_ptr = service_thread.lock()) { + if (strong_ptr == default_service_thread.lock()) { + // Nothing to do here, the service is using default_service_thread, which will be + // released on shutdown. + return; + } + service_threads_manager.QueueWork( [this, strong_ptr{std::move(strong_ptr)}]() { service_threads.erase(strong_ptr); }); } @@ -725,8 +727,7 @@ struct KernelCore::Impl { service_threads_manager.QueueWork([this]() { service_threads.clear(); }); } - std::mutex server_ports_lock; - std::mutex server_sessions_lock; + std::mutex server_objects_lock; std::mutex registered_objects_lock; std::mutex registered_in_use_objects_lock; @@ -737,7 +738,7 @@ struct KernelCore::Impl { // Lists all processes that exist in the current session. std::vector<KProcess*> process_list; - KProcess* current_process{}; + std::atomic<KProcess*> current_process{}; std::unique_ptr<Kernel::GlobalSchedulerContext> global_scheduler_context; Kernel::TimeManager time_manager; @@ -750,39 +751,41 @@ struct KernelCore::Impl { // stores all the objects in place. std::unique_ptr<KHandleTable> global_handle_table; - KAutoObjectWithListContainer object_list_container; + std::unique_ptr<KAutoObjectWithListContainer> global_object_list_container; /// Map of named ports managed by the kernel, which can be retrieved using /// the ConnectToPort SVC. std::unordered_map<std::string, ServiceInterfaceFactory> service_interface_factory; NamedPortTable named_ports; - std::unordered_set<KServerPort*> server_ports; - std::unordered_set<KServerSession*> server_sessions; + std::unordered_set<KAutoObject*> server_objects; std::unordered_set<KAutoObject*> registered_objects; std::unordered_set<KAutoObject*> registered_in_use_objects; std::unique_ptr<Core::ExclusiveMonitor> exclusive_monitor; - std::vector<Kernel::PhysicalCore> cores; + std::array<std::unique_ptr<Kernel::PhysicalCore>, Core::Hardware::NUM_CPU_CORES> cores; // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others std::atomic<u32> next_host_thread_id{Core::Hardware::NUM_CPU_CORES}; // Kernel memory management std::unique_ptr<KMemoryManager> memory_manager; - std::unique_ptr<KSlabHeap<Page>> user_slab_heap_pages; // Shared memory for services Kernel::KSharedMemory* hid_shared_mem{}; Kernel::KSharedMemory* font_shared_mem{}; Kernel::KSharedMemory* irs_shared_mem{}; Kernel::KSharedMemory* time_shared_mem{}; + Kernel::KSharedMemory* hidbus_shared_mem{}; + + // Memory layout + std::unique_ptr<KMemoryLayout> memory_layout; // Threads used for services - std::unordered_set<std::shared_ptr<Kernel::ServiceThread>> service_threads; + std::unordered_set<std::shared_ptr<ServiceThread>> service_threads; + std::weak_ptr<ServiceThread> default_service_thread; Common::ThreadWorker service_threads_manager; - std::array<KThread*, Core::Hardware::NUM_CPU_CORES> suspend_threads; - std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES> interrupts{}; + std::array<KThread*, Core::Hardware::NUM_CPU_CORES> shutdown_threads; std::array<std::unique_ptr<Kernel::KScheduler>, Core::Hardware::NUM_CPU_CORES> schedulers{}; bool is_multicore{}; @@ -818,6 +821,10 @@ void KernelCore::Shutdown() { impl->Shutdown(); } +void KernelCore::CloseServices() { + impl->CloseServices(); +} + const KResourceLimit* KernelCore::GetSystemResourceLimit() const { return impl->system_resource_limit; } @@ -867,11 +874,11 @@ const Kernel::KScheduler& KernelCore::Scheduler(std::size_t id) const { } Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) { - return impl->cores[id]; + return *impl->cores[id]; } const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const { - return impl->cores[id]; + return *impl->cores[id]; } size_t KernelCore::CurrentPhysicalCoreIndex() const { @@ -883,11 +890,11 @@ size_t KernelCore::CurrentPhysicalCoreIndex() const { } Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() { - return impl->cores[CurrentPhysicalCoreIndex()]; + return *impl->cores[CurrentPhysicalCoreIndex()]; } const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const { - return impl->cores[CurrentPhysicalCoreIndex()]; + return *impl->cores[CurrentPhysicalCoreIndex()]; } Kernel::KScheduler* KernelCore::CurrentScheduler() { @@ -899,15 +906,6 @@ Kernel::KScheduler* KernelCore::CurrentScheduler() { return impl->schedulers[core_id].get(); } -std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts() { - return impl->interrupts; -} - -const std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts() - const { - return impl->interrupts; -} - Kernel::TimeManager& KernelCore::TimeManager() { return impl->time_manager; } @@ -925,25 +923,25 @@ const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const { } KAutoObjectWithListContainer& KernelCore::ObjectListContainer() { - return impl->object_list_container; + return *impl->global_object_list_container; } const KAutoObjectWithListContainer& KernelCore::ObjectListContainer() const { - return impl->object_list_container; + return *impl->global_object_list_container; } void KernelCore::InvalidateAllInstructionCaches() { for (auto& physical_core : impl->cores) { - physical_core.ArmInterface().ClearInstructionCache(); + physical_core->ArmInterface().ClearInstructionCache(); } } void KernelCore::InvalidateCpuInstructionCacheRange(VAddr addr, std::size_t size) { for (auto& physical_core : impl->cores) { - if (!physical_core.IsInitialized()) { + if (!physical_core->IsInitialized()) { continue; } - physical_core.ArmInterface().InvalidateCacheRange(addr, size); + physical_core->ArmInterface().InvalidateCacheRange(addr, size); } } @@ -959,33 +957,31 @@ KClientPort* KernelCore::CreateNamedServicePort(std::string name) { return impl->CreateNamedServicePort(std::move(name)); } -void KernelCore::RegisterServerSession(KServerSession* server_session) { - std::lock_guard lk(impl->server_sessions_lock); - impl->server_sessions.insert(server_session); +void KernelCore::RegisterServerObject(KAutoObject* server_object) { + impl->RegisterServerObject(server_object); } -void KernelCore::UnregisterServerSession(KServerSession* server_session) { - std::lock_guard lk(impl->server_sessions_lock); - impl->server_sessions.erase(server_session); +void KernelCore::UnregisterServerObject(KAutoObject* server_object) { + impl->UnregisterServerObject(server_object); } void KernelCore::RegisterKernelObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_objects_lock); + std::scoped_lock lk{impl->registered_objects_lock}; impl->registered_objects.insert(object); } void KernelCore::UnregisterKernelObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_objects_lock); + std::scoped_lock lk{impl->registered_objects_lock}; impl->registered_objects.erase(object); } void KernelCore::RegisterInUseObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_in_use_objects_lock); + std::scoped_lock lk{impl->registered_in_use_objects_lock}; impl->registered_in_use_objects.insert(object); } void KernelCore::UnregisterInUseObject(KAutoObject* object) { - std::lock_guard lk(impl->registered_in_use_objects_lock); + std::scoped_lock lk{impl->registered_in_use_objects_lock}; impl->registered_in_use_objects.erase(object); } @@ -1033,6 +1029,10 @@ KThread* KernelCore::GetCurrentEmuThread() const { return impl->GetCurrentEmuThread(); } +void KernelCore::SetCurrentEmuThread(KThread* thread) { + impl->SetCurrentEmuThread(thread); +} + KMemoryManager& KernelCore::MemoryManager() { return *impl->memory_manager; } @@ -1041,14 +1041,6 @@ const KMemoryManager& KernelCore::MemoryManager() const { return *impl->memory_manager; } -KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() { - return *impl->user_slab_heap_pages; -} - -const KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() const { - return *impl->user_slab_heap_pages; -} - Kernel::KSharedMemory& KernelCore::GetHidSharedMem() { return *impl->hid_shared_mem; } @@ -1081,22 +1073,38 @@ const Kernel::KSharedMemory& KernelCore::GetTimeSharedMem() const { return *impl->time_shared_mem; } -void KernelCore::Suspend(bool in_suspention) { - const bool should_suspend = exception_exited || in_suspention; - { - KScopedSchedulerLock lock(*this); - const auto state = should_suspend ? ThreadState::Runnable : ThreadState::Waiting; - for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { - impl->suspend_threads[core_id]->SetState(state); - impl->suspend_threads[core_id]->SetWaitReasonForDebugging( - ThreadWaitReasonForDebugging::Suspended); - if (!should_suspend) { - impl->suspend_threads[core_id]->DisableDispatch(); +Kernel::KSharedMemory& KernelCore::GetHidBusSharedMem() { + return *impl->hidbus_shared_mem; +} + +const Kernel::KSharedMemory& KernelCore::GetHidBusSharedMem() const { + return *impl->hidbus_shared_mem; +} + +void KernelCore::Suspend(bool suspended) { + const bool should_suspend{exception_exited || suspended}; + const auto activity = should_suspend ? ProcessActivity::Paused : ProcessActivity::Runnable; + + for (auto* process : GetProcessList()) { + process->SetActivity(activity); + + if (should_suspend) { + // Wait for execution to stop + for (auto* thread : process->GetThreadList()) { + thread->WaitUntilSuspended(); } } } } +void KernelCore::ShutdownCores() { + KScopedSchedulerLock lk{*this}; + + for (auto* thread : impl->shutdown_threads) { + void(thread->Run()); + } +} + bool KernelCore::IsMulticore() const { return impl->is_multicore; } @@ -1122,6 +1130,10 @@ std::weak_ptr<Kernel::ServiceThread> KernelCore::CreateServiceThread(const std:: return impl->CreateServiceThread(*this, name); } +std::weak_ptr<Kernel::ServiceThread> KernelCore::GetDefaultServiceThread() const { + return impl->default_service_thread; +} + void KernelCore::ReleaseServiceThread(std::weak_ptr<Kernel::ServiceThread> service_thread) { impl->ReleaseServiceThread(service_thread); } @@ -1142,6 +1154,10 @@ const KWorkerTaskManager& KernelCore::WorkerTaskManager() const { return impl->worker_task_manager; } +const KMemoryLayout& KernelCore::MemoryLayout() const { + return *impl->memory_layout; +} + bool KernelCore::IsPhantomModeForSingleCore() const { return impl->IsPhantomModeForSingleCore(); } |