diff options
Diffstat (limited to 'src')
160 files changed, 4505 insertions, 2423 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3575a3cb3..0ac3d254e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,13 +58,11 @@ if (MSVC) # Warnings /W3 - /we4018 # 'expression': signed/unsigned mismatch + /WX + /we4062 # Enumerator 'identifier' in a switch of enum 'enumeration' is not handled - /we4101 # 'identifier': unreferenced local variable /we4189 # 'identifier': local variable is initialized but not referenced /we4265 # 'class': class has virtual functions, but destructor is not virtual - /we4267 # 'var': conversion from 'size_t' to 'type', possible loss of data - /we4305 # 'context': truncation from 'type1' to 'type2' /we4388 # 'expression': signed/unsigned mismatch /we4389 # 'operator': signed/unsigned mismatch /we4456 # Declaration of 'identifier' hides previous local declaration @@ -75,10 +73,13 @@ if (MSVC) /we4547 # 'operator': operator before comma has no effect; expected operator with side-effect /we4549 # 'operator1': operator before comma has no effect; did you intend 'operator2'? /we4555 # Expression has no effect; expected expression with side-effect - /we4715 # 'function': not all control paths return a value - /we4834 # Discarding return value of function with 'nodiscard' attribute + /we4826 # Conversion from 'type1' to 'type2' is sign-extended. This may cause unexpected runtime behavior. /we5038 # data member 'member1' will be initialized after data member 'member2' + /we5233 # explicit lambda capture 'identifier' is not used /we5245 # 'function': unreferenced function with internal linkage has been removed + + /wd4100 # 'identifier': unreferenced formal parameter + /wd4324 # 'struct_name': structure was padded due to __declspec(align()) ) if (USE_CCACHE) @@ -99,24 +100,18 @@ if (MSVC) set(CMAKE_EXE_LINKER_FLAGS_RELEASE "/DEBUG /MANIFEST:NO /INCREMENTAL:NO /OPT:REF,ICF" CACHE STRING "" FORCE) else() add_compile_options( - -Wall - -Werror=array-bounds - -Werror=implicit-fallthrough + -Werror=all + -Werror=extra -Werror=missing-declarations - -Werror=missing-field-initializers - -Werror=reorder -Werror=shadow - -Werror=sign-compare - -Werror=switch - -Werror=uninitialized - -Werror=unused-function - -Werror=unused-result - -Werror=unused-variable - -Wextra - -Wmissing-declarations + -Werror=unused + -Wno-attributes -Wno-invalid-offsetof -Wno-unused-parameter + + $<$<CXX_COMPILER_ID:Clang>:-Wno-braced-scalar-init> + $<$<CXX_COMPILER_ID:Clang>:-Wno-unused-private-field> ) if (ARCHITECTURE_x86_64) diff --git a/src/audio_core/CMakeLists.txt b/src/audio_core/CMakeLists.txt index 144f1bab2..0a1f3bf18 100644 --- a/src/audio_core/CMakeLists.txt +++ b/src/audio_core/CMakeLists.txt @@ -206,20 +206,11 @@ if (MSVC) /we4244 # 'conversion': conversion from 'type1' to 'type2', possible loss of data /we4245 # 'conversion': conversion from 'type1' to 'type2', signed/unsigned mismatch /we4254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data - /we4456 # Declaration of 'identifier' hides previous local declaration - /we4457 # Declaration of 'identifier' hides function parameter - /we4458 # Declaration of 'identifier' hides class member - /we4459 # Declaration of 'identifier' hides global declaration + /we4800 # Implicit conversion from 'type' to bool. Possible information loss ) else() target_compile_options(audio_core PRIVATE -Werror=conversion - -Werror=ignored-qualifiers - -Werror=shadow - -Werror=unused-variable - - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable> -Wno-sign-conversion ) diff --git a/src/audio_core/in/audio_in_system.cpp b/src/audio_core/in/audio_in_system.cpp index 6b7e6715c..4324cafd8 100644 --- a/src/audio_core/in/audio_in_system.cpp +++ b/src/audio_core/in/audio_in_system.cpp @@ -56,7 +56,7 @@ Result System::IsConfigValid(const std::string_view device_name, return ResultSuccess; } -Result System::Initialize(std::string& device_name, const AudioInParameter& in_params, +Result System::Initialize(std::string device_name, const AudioInParameter& in_params, const u32 handle_, const u64 applet_resource_user_id_) { auto result{IsConfigValid(device_name, in_params)}; if (result.IsError()) { diff --git a/src/audio_core/in/audio_in_system.h b/src/audio_core/in/audio_in_system.h index b9dc0e60f..1c5154638 100644 --- a/src/audio_core/in/audio_in_system.h +++ b/src/audio_core/in/audio_in_system.h @@ -97,7 +97,7 @@ public: * @param applet_resource_user_id - Unused. * @return Result code. */ - Result Initialize(std::string& device_name, const AudioInParameter& in_params, u32 handle, + Result Initialize(std::string device_name, const AudioInParameter& in_params, u32 handle, u64 applet_resource_user_id); /** diff --git a/src/audio_core/out/audio_out_system.cpp b/src/audio_core/out/audio_out_system.cpp index 48a801923..a66208ed9 100644 --- a/src/audio_core/out/audio_out_system.cpp +++ b/src/audio_core/out/audio_out_system.cpp @@ -49,8 +49,8 @@ Result System::IsConfigValid(std::string_view device_name, return Service::Audio::ERR_INVALID_CHANNEL_COUNT; } -Result System::Initialize(std::string& device_name, const AudioOutParameter& in_params, u32 handle_, - u64& applet_resource_user_id_) { +Result System::Initialize(std::string device_name, const AudioOutParameter& in_params, u32 handle_, + u64 applet_resource_user_id_) { auto result = IsConfigValid(device_name, in_params); if (result.IsError()) { return result; diff --git a/src/audio_core/out/audio_out_system.h b/src/audio_core/out/audio_out_system.h index 0817b2f37..b95cb91be 100644 --- a/src/audio_core/out/audio_out_system.h +++ b/src/audio_core/out/audio_out_system.h @@ -88,8 +88,8 @@ public: * @param applet_resource_user_id - Unused. * @return Result code. */ - Result Initialize(std::string& device_name, const AudioOutParameter& in_params, u32 handle, - u64& applet_resource_user_id); + Result Initialize(std::string device_name, const AudioOutParameter& in_params, u32 handle, + u64 applet_resource_user_id); /** * Start this system. diff --git a/src/audio_core/renderer/behavior/info_updater.cpp b/src/audio_core/renderer/behavior/info_updater.cpp index c0a307b89..574cf0982 100644 --- a/src/audio_core/renderer/behavior/info_updater.cpp +++ b/src/audio_core/renderer/behavior/info_updater.cpp @@ -91,7 +91,7 @@ Result InfoUpdater::UpdateVoices(VoiceContext& voice_context, voice_info.Initialize(); for (u32 channel = 0; channel < in_param.channel_count; channel++) { - std::memset(voice_states[channel], 0, sizeof(VoiceState)); + *voice_states[channel] = {}; } } diff --git a/src/audio_core/renderer/command/effect/biquad_filter.cpp b/src/audio_core/renderer/command/effect/biquad_filter.cpp index 1baae74fd..edb30ce72 100644 --- a/src/audio_core/renderer/command/effect/biquad_filter.cpp +++ b/src/audio_core/renderer/command/effect/biquad_filter.cpp @@ -94,7 +94,7 @@ void BiquadFilterCommand::Dump([[maybe_unused]] const ADSP::CommandListProcessor void BiquadFilterCommand::Process(const ADSP::CommandListProcessor& processor) { auto state_{reinterpret_cast<VoiceState::BiquadFilterState*>(state)}; if (needs_init) { - std::memset(state_, 0, sizeof(VoiceState::BiquadFilterState)); + *state_ = {}; } auto input_buffer{ diff --git a/src/audio_core/renderer/command/effect/multi_tap_biquad_filter.cpp b/src/audio_core/renderer/command/effect/multi_tap_biquad_filter.cpp index b3c3ba4ba..48a7cba8a 100644 --- a/src/audio_core/renderer/command/effect/multi_tap_biquad_filter.cpp +++ b/src/audio_core/renderer/command/effect/multi_tap_biquad_filter.cpp @@ -30,7 +30,7 @@ void MultiTapBiquadFilterCommand::Process(const ADSP::CommandListProcessor& proc for (u32 i = 0; i < filter_tap_count; i++) { auto state{reinterpret_cast<VoiceState::BiquadFilterState*>(states[i])}; if (needs_init[i]) { - std::memset(state, 0, sizeof(VoiceState::BiquadFilterState)); + *state = {}; } ApplyBiquadFilterFloat(output_buffer, input_buffer, biquads[i].b, biquads[i].a, *state, diff --git a/src/audio_core/renderer/system.cpp b/src/audio_core/renderer/system.cpp index bde794cd1..4fac30c7c 100644 --- a/src/audio_core/renderer/system.cpp +++ b/src/audio_core/renderer/system.cpp @@ -98,9 +98,8 @@ System::System(Core::System& core_, Kernel::KEvent* adsp_rendered_event_) : core{core_}, adsp{core.AudioCore().GetADSP()}, adsp_rendered_event{adsp_rendered_event_} {} Result System::Initialize(const AudioRendererParameterInternal& params, - Kernel::KTransferMemory* transfer_memory, const u64 transfer_memory_size, - const u32 process_handle_, const u64 applet_resource_user_id_, - const s32 session_id_) { + Kernel::KTransferMemory* transfer_memory, u64 transfer_memory_size, + u32 process_handle_, u64 applet_resource_user_id_, s32 session_id_) { if (!CheckValidRevision(params.revision)) { return Service::Audio::ERR_INVALID_REVISION; } @@ -354,6 +353,8 @@ Result System::Initialize(const AudioRendererParameterInternal& params, render_time_limit_percent = 100; drop_voice = params.voice_drop_enabled && params.execution_mode == ExecutionMode::Auto; + drop_voice_param = 1.0f; + num_voices_dropped = 0; allocator.Align(0x40); command_workbuffer_size = allocator.GetRemainingSize(); @@ -547,7 +548,7 @@ u32 System::GetRenderingTimeLimit() const { return render_time_limit_percent; } -void System::SetRenderingTimeLimit(const u32 limit) { +void System::SetRenderingTimeLimit(u32 limit) { render_time_limit_percent = limit; } @@ -635,7 +636,7 @@ void System::SendCommandToDsp() { } u64 System::GenerateCommand(std::span<u8> in_command_buffer, - [[maybe_unused]] const u64 command_buffer_size_) { + [[maybe_unused]] u64 command_buffer_size_) { PoolMapper::ClearUseState(memory_pool_workbuffer, memory_pool_count); const auto start_time{core.CoreTiming().GetClockTicks()}; @@ -693,7 +694,8 @@ u64 System::GenerateCommand(std::span<u8> in_command_buffer, voice_context.SortInfo(); - const auto start_estimated_time{command_buffer.estimated_process_time}; + const auto start_estimated_time{drop_voice_param * + static_cast<f32>(command_buffer.estimated_process_time)}; command_generator.GenerateVoiceCommands(); command_generator.GenerateSubMixCommands(); @@ -712,11 +714,16 @@ u64 System::GenerateCommand(std::span<u8> in_command_buffer, render_context.behavior->IsAudioRendererProcessingTimeLimit70PercentSupported(); time_limit_percent = 70.0f; } + + const auto end_estimated_time{drop_voice_param * + static_cast<f32>(command_buffer.estimated_process_time)}; + const auto estimated_time{start_estimated_time - end_estimated_time}; + const auto time_limit{static_cast<u32>( - static_cast<f32>(start_estimated_time - command_buffer.estimated_process_time) + - (((time_limit_percent / 100.0f) * 2'880'000.0) * - (static_cast<f32>(render_time_limit_percent) / 100.0f)))}; - num_voices_dropped = DropVoices(command_buffer, start_estimated_time, time_limit); + estimated_time + (((time_limit_percent / 100.0f) * 2'880'000.0) * + (static_cast<f32>(render_time_limit_percent) / 100.0f)))}; + num_voices_dropped = + DropVoices(command_buffer, static_cast<u32>(start_estimated_time), time_limit); } command_list_header->buffer_size = command_buffer.size; @@ -737,24 +744,33 @@ u64 System::GenerateCommand(std::span<u8> in_command_buffer, return command_buffer.size; } -u32 System::DropVoices(CommandBuffer& command_buffer, const u32 estimated_process_time, - const u32 time_limit) { +f32 System::GetVoiceDropParameter() const { + return drop_voice_param; +} + +void System::SetVoiceDropParameter(f32 voice_drop_) { + drop_voice_param = voice_drop_; +} + +u32 System::DropVoices(CommandBuffer& command_buffer, u32 estimated_process_time, u32 time_limit) { u32 i{0}; auto command_list{command_buffer.command_list.data() + sizeof(CommandListHeader)}; - ICommand* cmd{}; + ICommand* cmd{nullptr}; - for (; i < command_buffer.count; i++) { + // Find a first valid voice to drop + while (i < command_buffer.count) { cmd = reinterpret_cast<ICommand*>(command_list); - if (cmd->type != CommandId::Performance && - cmd->type != CommandId::DataSourcePcmInt16Version1 && - cmd->type != CommandId::DataSourcePcmInt16Version2 && - cmd->type != CommandId::DataSourcePcmFloatVersion1 && - cmd->type != CommandId::DataSourcePcmFloatVersion2 && - cmd->type != CommandId::DataSourceAdpcmVersion1 && - cmd->type != CommandId::DataSourceAdpcmVersion2) { + if (cmd->type == CommandId::Performance || + cmd->type == CommandId::DataSourcePcmInt16Version1 || + cmd->type == CommandId::DataSourcePcmInt16Version2 || + cmd->type == CommandId::DataSourcePcmFloatVersion1 || + cmd->type == CommandId::DataSourcePcmFloatVersion2 || + cmd->type == CommandId::DataSourceAdpcmVersion1 || + cmd->type == CommandId::DataSourceAdpcmVersion2) { break; } command_list += cmd->size; + i++; } if (cmd == nullptr || command_buffer.count == 0 || i >= command_buffer.count) { @@ -767,6 +783,7 @@ u32 System::DropVoices(CommandBuffer& command_buffer, const u32 estimated_proces const auto node_id_type{cmd->node_id >> 28}; const auto node_id_base{cmd->node_id & 0xFFF}; + // If the new estimated process time falls below the limit, we're done dropping. if (estimated_process_time <= time_limit) { break; } @@ -775,6 +792,7 @@ u32 System::DropVoices(CommandBuffer& command_buffer, const u32 estimated_proces break; } + // Don't drop voices marked with the highest priority. auto& voice_info{voice_context.GetInfo(node_id_base)}; if (voice_info.priority == HighestVoicePriority) { break; @@ -783,18 +801,23 @@ u32 System::DropVoices(CommandBuffer& command_buffer, const u32 estimated_proces voices_dropped++; voice_info.voice_dropped = true; - if (i < command_buffer.count) { - while (cmd->node_id == node_id) { - if (cmd->type == CommandId::DepopPrepare) { - cmd->enabled = true; - } else if (cmd->type == CommandId::Performance || !cmd->enabled) { - cmd->enabled = false; - } - i++; - command_list += cmd->size; - cmd = reinterpret_cast<ICommand*>(command_list); + // First iteration should drop the voice, and then iterate through all of the commands tied + // to the voice. We don't need reverb on a voice which we've just removed, for example. + // Depops can't be removed otherwise we'll introduce audio popping, and we don't + // remove perf commands. Lower the estimated time for each command dropped. + while (i < command_buffer.count && cmd->node_id == node_id) { + if (cmd->type == CommandId::DepopPrepare) { + cmd->enabled = true; + } else if (cmd->enabled && cmd->type != CommandId::Performance) { + cmd->enabled = false; + estimated_process_time -= static_cast<u32>( + drop_voice_param * static_cast<f32>(cmd->estimated_process_time)); } + command_list += cmd->size; + cmd = reinterpret_cast<ICommand*>(command_list); + i++; } + i++; } return voices_dropped; } diff --git a/src/audio_core/renderer/system.h b/src/audio_core/renderer/system.h index bcbe65b07..429196e41 100644 --- a/src/audio_core/renderer/system.h +++ b/src/audio_core/renderer/system.h @@ -196,6 +196,20 @@ public: */ u32 DropVoices(CommandBuffer& command_buffer, u32 estimated_process_time, u32 time_limit); + /** + * Get the current voice drop parameter. + * + * @return The current voice drop. + */ + f32 GetVoiceDropParameter() const; + + /** + * Set the voice drop parameter. + * + * @param The new voice drop. + */ + void SetVoiceDropParameter(f32 voice_drop); + private: /// Core system Core::System& core; @@ -301,6 +315,8 @@ private: u32 num_voices_dropped{}; /// Tick that rendering started u64 render_start_tick{}; + /// Parameter to control the threshold for dropping voices if the audio graph gets too large + f32 drop_voice_param{1.0f}; }; } // namespace AudioRenderer diff --git a/src/audio_core/renderer/voice/voice_context.cpp b/src/audio_core/renderer/voice/voice_context.cpp index eafb51b01..a501a677d 100644 --- a/src/audio_core/renderer/voice/voice_context.cpp +++ b/src/audio_core/renderer/voice/voice_context.cpp @@ -74,8 +74,8 @@ void VoiceContext::SortInfo() { } std::ranges::sort(sorted_voice_info, [](const VoiceInfo* a, const VoiceInfo* b) { - return a->priority != b->priority ? a->priority < b->priority - : a->sort_order < b->sort_order; + return a->priority != b->priority ? a->priority > b->priority + : a->sort_order > b->sort_order; }); } diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index a02696873..c0555f840 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -156,12 +156,13 @@ if (MSVC) ) target_compile_options(common PRIVATE /W4 - /WX + + /we4242 # 'identifier': conversion from 'type1' to 'type2', possible loss of data + /we4254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data + /we4800 # Implicit conversion from 'type' to bool. Possible information loss ) else() target_compile_options(common PRIVATE - -Werror - $<$<CXX_COMPILER_ID:Clang>:-fsized-deallocation> ) endif() @@ -169,7 +170,11 @@ endif() create_target_directory_groups(common) target_link_libraries(common PUBLIC ${Boost_LIBRARIES} fmt::fmt microprofile Threads::Threads) -target_link_libraries(common PRIVATE lz4::lz4) +if (TARGET lz4::lz4) + target_link_libraries(common PRIVATE lz4::lz4) +else() + target_link_libraries(common PRIVATE LZ4::lz4_shared) +endif() if (TARGET zstd::zstd) target_link_libraries(common PRIVATE zstd::zstd) else() diff --git a/src/common/bit_field.h b/src/common/bit_field.h index 7e1df62b1..e4e58ea45 100644 --- a/src/common/bit_field.h +++ b/src/common/bit_field.h @@ -141,10 +141,6 @@ public: constexpr BitField(BitField&&) noexcept = default; constexpr BitField& operator=(BitField&&) noexcept = default; - [[nodiscard]] constexpr operator T() const { - return Value(); - } - constexpr void Assign(const T& value) { #ifdef _MSC_VER storage = static_cast<StorageType>((storage & ~mask) | FormatValue(value)); @@ -162,6 +158,17 @@ public: return ExtractValue(storage); } + template <typename ConvertedToType> + [[nodiscard]] constexpr ConvertedToType As() const { + static_assert(!std::is_same_v<T, ConvertedToType>, + "Unnecessary cast. Use Value() instead."); + return static_cast<ConvertedToType>(Value()); + } + + [[nodiscard]] constexpr operator T() const { + return Value(); + } + [[nodiscard]] constexpr explicit operator bool() const { return Value() != 0; } diff --git a/src/common/bounded_threadsafe_queue.h b/src/common/bounded_threadsafe_queue.h index 7e465549b..21217801e 100644 --- a/src/common/bounded_threadsafe_queue.h +++ b/src/common/bounded_threadsafe_queue.h @@ -21,11 +21,6 @@ constexpr size_t hardware_interference_size = std::hardware_destructive_interfer constexpr size_t hardware_interference_size = 64; #endif -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4324) -#endif - template <typename T, size_t capacity = 0x400> class MPSCQueue { public: @@ -160,8 +155,4 @@ private: static_assert(std::is_nothrow_destructible_v<T>, "T must be nothrow destructible"); }; -#ifdef _MSC_VER -#pragma warning(pop) -#endif - } // namespace Common diff --git a/src/common/concepts.h b/src/common/concepts.h index a97555f6a..a9acff3e7 100644 --- a/src/common/concepts.h +++ b/src/common/concepts.h @@ -3,24 +3,14 @@ #pragma once +#include <iterator> #include <type_traits> namespace Common { -// Check if type is like an STL container +// Check if type satisfies the ContiguousContainer named requirement. template <typename T> -concept IsSTLContainer = requires(T t) { - typename T::value_type; - typename T::iterator; - typename T::const_iterator; - // TODO(ogniK): Replace below is std::same_as<void> when MSVC supports it. - t.begin(); - t.end(); - t.cbegin(); - t.cend(); - t.data(); - t.size(); -}; +concept IsContiguousContainer = std::contiguous_iterator<typename T::iterator>; // TODO: Replace with std::derived_from when the <concepts> header // is available on all supported platforms. @@ -34,4 +24,12 @@ concept DerivedFrom = requires { template <typename From, typename To> concept ConvertibleTo = std::is_convertible_v<From, To>; +// No equivalents in the stdlib + +template <typename T> +concept IsArithmetic = std::is_arithmetic_v<T>; + +template <typename T> +concept IsIntegral = std::is_integral_v<T>; + } // namespace Common diff --git a/src/common/fixed_point.h b/src/common/fixed_point.h index 4a0f72cc9..f899b0d54 100644 --- a/src/common/fixed_point.h +++ b/src/common/fixed_point.h @@ -4,14 +4,7 @@ // From: https://github.com/eteran/cpp-utilities/blob/master/fixed/include/cpp-utilities/fixed.h // See also: http://stackoverflow.com/questions/79677/whats-the-best-way-to-do-fixed-point-math -#ifndef FIXED_H_ -#define FIXED_H_ - -#if __cplusplus >= 201402L -#define CONSTEXPR14 constexpr -#else -#define CONSTEXPR14 -#endif +#pragma once #include <cstddef> // for size_t #include <cstdint> @@ -19,6 +12,8 @@ #include <ostream> #include <type_traits> +#include <common/concepts.h> + namespace Common { template <size_t I, size_t F> @@ -57,8 +52,8 @@ struct type_from_size<64> { static constexpr size_t size = 64; using value_type = int64_t; - using unsigned_type = std::make_unsigned<value_type>::type; - using signed_type = std::make_signed<value_type>::type; + using unsigned_type = std::make_unsigned_t<value_type>; + using signed_type = std::make_signed_t<value_type>; using next_size = type_from_size<128>; }; @@ -68,8 +63,8 @@ struct type_from_size<32> { static constexpr size_t size = 32; using value_type = int32_t; - using unsigned_type = std::make_unsigned<value_type>::type; - using signed_type = std::make_signed<value_type>::type; + using unsigned_type = std::make_unsigned_t<value_type>; + using signed_type = std::make_signed_t<value_type>; using next_size = type_from_size<64>; }; @@ -79,8 +74,8 @@ struct type_from_size<16> { static constexpr size_t size = 16; using value_type = int16_t; - using unsigned_type = std::make_unsigned<value_type>::type; - using signed_type = std::make_signed<value_type>::type; + using unsigned_type = std::make_unsigned_t<value_type>; + using signed_type = std::make_signed_t<value_type>; using next_size = type_from_size<32>; }; @@ -90,8 +85,8 @@ struct type_from_size<8> { static constexpr size_t size = 8; using value_type = int8_t; - using unsigned_type = std::make_unsigned<value_type>::type; - using signed_type = std::make_signed<value_type>::type; + using unsigned_type = std::make_unsigned_t<value_type>; + using signed_type = std::make_signed_t<value_type>; using next_size = type_from_size<16>; }; @@ -106,9 +101,9 @@ constexpr B next_to_base(N rhs) { struct divide_by_zero : std::exception {}; template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> divide( +constexpr FixedPoint<I, F> divide( FixedPoint<I, F> numerator, FixedPoint<I, F> denominator, FixedPoint<I, F>& remainder, - typename std::enable_if<type_from_size<I + F>::next_size::is_specialized>::type* = nullptr) { + std::enable_if_t<type_from_size<I + F>::next_size::is_specialized>* = nullptr) { using next_type = typename FixedPoint<I, F>::next_type; using base_type = typename FixedPoint<I, F>::base_type; @@ -126,9 +121,9 @@ CONSTEXPR14 FixedPoint<I, F> divide( } template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> divide( +constexpr FixedPoint<I, F> divide( FixedPoint<I, F> numerator, FixedPoint<I, F> denominator, FixedPoint<I, F>& remainder, - typename std::enable_if<!type_from_size<I + F>::next_size::is_specialized>::type* = nullptr) { + std::enable_if_t<!type_from_size<I + F>::next_size::is_specialized>* = nullptr) { using unsigned_type = typename FixedPoint<I, F>::unsigned_type; @@ -196,9 +191,9 @@ CONSTEXPR14 FixedPoint<I, F> divide( // this is the usual implementation of multiplication template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> multiply( +constexpr FixedPoint<I, F> multiply( FixedPoint<I, F> lhs, FixedPoint<I, F> rhs, - typename std::enable_if<type_from_size<I + F>::next_size::is_specialized>::type* = nullptr) { + std::enable_if_t<type_from_size<I + F>::next_size::is_specialized>* = nullptr) { using next_type = typename FixedPoint<I, F>::next_type; using base_type = typename FixedPoint<I, F>::base_type; @@ -215,9 +210,9 @@ CONSTEXPR14 FixedPoint<I, F> multiply( // it is slightly slower, but is more robust since it doesn't // require and upgraded type template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> multiply( +constexpr FixedPoint<I, F> multiply( FixedPoint<I, F> lhs, FixedPoint<I, F> rhs, - typename std::enable_if<!type_from_size<I + F>::next_size::is_specialized>::type* = nullptr) { + std::enable_if_t<!type_from_size<I + F>::next_size::is_specialized>* = nullptr) { using base_type = typename FixedPoint<I, F>::base_type; @@ -272,19 +267,20 @@ public: static constexpr base_type one = base_type(1) << fractional_bits; public: // constructors - FixedPoint() = default; - FixedPoint(const FixedPoint&) = default; - FixedPoint(FixedPoint&&) = default; - FixedPoint& operator=(const FixedPoint&) = default; + constexpr FixedPoint() = default; + + constexpr FixedPoint(const FixedPoint&) = default; + constexpr FixedPoint& operator=(const FixedPoint&) = default; + + constexpr FixedPoint(FixedPoint&&) noexcept = default; + constexpr FixedPoint& operator=(FixedPoint&&) noexcept = default; - template <class Number> - constexpr FixedPoint( - Number n, typename std::enable_if<std::is_arithmetic<Number>::value>::type* = nullptr) - : data_(static_cast<base_type>(n * one)) {} + template <IsArithmetic Number> + constexpr FixedPoint(Number n) : data_(static_cast<base_type>(n * one)) {} public: // conversion template <size_t I2, size_t F2> - CONSTEXPR14 explicit FixedPoint(FixedPoint<I2, F2> other) { + constexpr explicit FixedPoint(FixedPoint<I2, F2> other) { static_assert(I2 <= I && F2 <= F, "Scaling conversion can only upgrade types"); using T = FixedPoint<I2, F2>; @@ -308,36 +304,14 @@ public: } public: // comparison operators - constexpr bool operator==(FixedPoint rhs) const { - return data_ == rhs.data_; - } - - constexpr bool operator!=(FixedPoint rhs) const { - return data_ != rhs.data_; - } - - constexpr bool operator<(FixedPoint rhs) const { - return data_ < rhs.data_; - } - - constexpr bool operator>(FixedPoint rhs) const { - return data_ > rhs.data_; - } - - constexpr bool operator<=(FixedPoint rhs) const { - return data_ <= rhs.data_; - } - - constexpr bool operator>=(FixedPoint rhs) const { - return data_ >= rhs.data_; - } + friend constexpr auto operator<=>(FixedPoint lhs, FixedPoint rhs) = default; public: // unary operators - constexpr bool operator!() const { + [[nodiscard]] constexpr bool operator!() const { return !data_; } - constexpr FixedPoint operator~() const { + [[nodiscard]] constexpr FixedPoint operator~() const { // NOTE(eteran): this will often appear to "just negate" the value // that is not an error, it is because -x == (~x+1) // and that "+1" is adding an infinitesimally small fraction to the @@ -345,89 +319,87 @@ public: // unary operators return FixedPoint::from_base(~data_); } - constexpr FixedPoint operator-() const { + [[nodiscard]] constexpr FixedPoint operator-() const { return FixedPoint::from_base(-data_); } - constexpr FixedPoint operator+() const { + [[nodiscard]] constexpr FixedPoint operator+() const { return FixedPoint::from_base(+data_); } - CONSTEXPR14 FixedPoint& operator++() { + constexpr FixedPoint& operator++() { data_ += one; return *this; } - CONSTEXPR14 FixedPoint& operator--() { + constexpr FixedPoint& operator--() { data_ -= one; return *this; } - CONSTEXPR14 FixedPoint operator++(int) { + constexpr FixedPoint operator++(int) { FixedPoint tmp(*this); data_ += one; return tmp; } - CONSTEXPR14 FixedPoint operator--(int) { + constexpr FixedPoint operator--(int) { FixedPoint tmp(*this); data_ -= one; return tmp; } public: // basic math operators - CONSTEXPR14 FixedPoint& operator+=(FixedPoint n) { + constexpr FixedPoint& operator+=(FixedPoint n) { data_ += n.data_; return *this; } - CONSTEXPR14 FixedPoint& operator-=(FixedPoint n) { + constexpr FixedPoint& operator-=(FixedPoint n) { data_ -= n.data_; return *this; } - CONSTEXPR14 FixedPoint& operator*=(FixedPoint n) { + constexpr FixedPoint& operator*=(FixedPoint n) { return assign(detail::multiply(*this, n)); } - CONSTEXPR14 FixedPoint& operator/=(FixedPoint n) { + constexpr FixedPoint& operator/=(FixedPoint n) { FixedPoint temp; return assign(detail::divide(*this, n, temp)); } private: - CONSTEXPR14 FixedPoint& assign(FixedPoint rhs) { + constexpr FixedPoint& assign(FixedPoint rhs) { data_ = rhs.data_; return *this; } public: // binary math operators, effects underlying bit pattern since these // don't really typically make sense for non-integer values - CONSTEXPR14 FixedPoint& operator&=(FixedPoint n) { + constexpr FixedPoint& operator&=(FixedPoint n) { data_ &= n.data_; return *this; } - CONSTEXPR14 FixedPoint& operator|=(FixedPoint n) { + constexpr FixedPoint& operator|=(FixedPoint n) { data_ |= n.data_; return *this; } - CONSTEXPR14 FixedPoint& operator^=(FixedPoint n) { + constexpr FixedPoint& operator^=(FixedPoint n) { data_ ^= n.data_; return *this; } - template <class Integer, - class = typename std::enable_if<std::is_integral<Integer>::value>::type> - CONSTEXPR14 FixedPoint& operator>>=(Integer n) { + template <IsIntegral Integer> + constexpr FixedPoint& operator>>=(Integer n) { data_ >>= n; return *this; } - template <class Integer, - class = typename std::enable_if<std::is_integral<Integer>::value>::type> - CONSTEXPR14 FixedPoint& operator<<=(Integer n) { + template <IsIntegral Integer> + constexpr FixedPoint& operator<<=(Integer n) { data_ <<= n; return *this; } @@ -437,42 +409,42 @@ public: // conversion to basic types data_ += (data_ & fractional_mask) >> 1; } - constexpr int to_int() { + [[nodiscard]] constexpr int to_int() { round_up(); return static_cast<int>((data_ & integer_mask) >> fractional_bits); } - constexpr unsigned int to_uint() const { + [[nodiscard]] constexpr unsigned int to_uint() { round_up(); return static_cast<unsigned int>((data_ & integer_mask) >> fractional_bits); } - constexpr int64_t to_long() { + [[nodiscard]] constexpr int64_t to_long() { round_up(); return static_cast<int64_t>((data_ & integer_mask) >> fractional_bits); } - constexpr int to_int_floor() const { + [[nodiscard]] constexpr int to_int_floor() const { return static_cast<int>((data_ & integer_mask) >> fractional_bits); } - constexpr int64_t to_long_floor() { + [[nodiscard]] constexpr int64_t to_long_floor() const { return static_cast<int64_t>((data_ & integer_mask) >> fractional_bits); } - constexpr unsigned int to_uint_floor() const { + [[nodiscard]] constexpr unsigned int to_uint_floor() const { return static_cast<unsigned int>((data_ & integer_mask) >> fractional_bits); } - constexpr float to_float() const { + [[nodiscard]] constexpr float to_float() const { return static_cast<float>(data_) / FixedPoint::one; } - constexpr double to_double() const { + [[nodiscard]] constexpr double to_double() const { return static_cast<double>(data_) / FixedPoint::one; } - constexpr base_type to_raw() const { + [[nodiscard]] constexpr base_type to_raw() const { return data_; } @@ -480,27 +452,27 @@ public: // conversion to basic types data_ &= fractional_mask; } - constexpr base_type get_frac() const { + [[nodiscard]] constexpr base_type get_frac() const { return data_ & fractional_mask; } public: - CONSTEXPR14 void swap(FixedPoint& rhs) { + constexpr void swap(FixedPoint& rhs) noexcept { using std::swap; swap(data_, rhs.data_); } public: - base_type data_; + base_type data_{}; }; // if we have the same fractional portion, but differing integer portions, we trivially upgrade the // smaller type template <size_t I1, size_t I2, size_t F> -CONSTEXPR14 typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type -operator+(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { +constexpr std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>> operator+( + FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { - using T = typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type; + using T = std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>; const T l = T::from_base(lhs.to_raw()); const T r = T::from_base(rhs.to_raw()); @@ -508,10 +480,10 @@ operator+(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { } template <size_t I1, size_t I2, size_t F> -CONSTEXPR14 typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type -operator-(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { +constexpr std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>> operator-( + FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { - using T = typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type; + using T = std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>; const T l = T::from_base(lhs.to_raw()); const T r = T::from_base(rhs.to_raw()); @@ -519,10 +491,10 @@ operator-(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { } template <size_t I1, size_t I2, size_t F> -CONSTEXPR14 typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type -operator*(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { +constexpr std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>> operator*( + FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { - using T = typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type; + using T = std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>; const T l = T::from_base(lhs.to_raw()); const T r = T::from_base(rhs.to_raw()); @@ -530,10 +502,10 @@ operator*(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { } template <size_t I1, size_t I2, size_t F> -CONSTEXPR14 typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type -operator/(FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { +constexpr std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>> operator/( + FixedPoint<I1, F> lhs, FixedPoint<I2, F> rhs) { - using T = typename std::conditional<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>::type; + using T = std::conditional_t<I1 >= I2, FixedPoint<I1, F>, FixedPoint<I2, F>>; const T l = T::from_base(lhs.to_raw()); const T r = T::from_base(rhs.to_raw()); @@ -548,159 +520,133 @@ std::ostream& operator<<(std::ostream& os, FixedPoint<I, F> f) { // basic math operators template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> operator+(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { +constexpr FixedPoint<I, F> operator+(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { lhs += rhs; return lhs; } template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> operator-(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { +constexpr FixedPoint<I, F> operator-(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { lhs -= rhs; return lhs; } template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> operator*(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { +constexpr FixedPoint<I, F> operator*(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { lhs *= rhs; return lhs; } template <size_t I, size_t F> -CONSTEXPR14 FixedPoint<I, F> operator/(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { +constexpr FixedPoint<I, F> operator/(FixedPoint<I, F> lhs, FixedPoint<I, F> rhs) { lhs /= rhs; return lhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator+(FixedPoint<I, F> lhs, Number rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator+(FixedPoint<I, F> lhs, Number rhs) { lhs += FixedPoint<I, F>(rhs); return lhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator-(FixedPoint<I, F> lhs, Number rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator-(FixedPoint<I, F> lhs, Number rhs) { lhs -= FixedPoint<I, F>(rhs); return lhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator*(FixedPoint<I, F> lhs, Number rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator*(FixedPoint<I, F> lhs, Number rhs) { lhs *= FixedPoint<I, F>(rhs); return lhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator/(FixedPoint<I, F> lhs, Number rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator/(FixedPoint<I, F> lhs, Number rhs) { lhs /= FixedPoint<I, F>(rhs); return lhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator+(Number lhs, FixedPoint<I, F> rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator+(Number lhs, FixedPoint<I, F> rhs) { FixedPoint<I, F> tmp(lhs); tmp += rhs; return tmp; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator-(Number lhs, FixedPoint<I, F> rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator-(Number lhs, FixedPoint<I, F> rhs) { FixedPoint<I, F> tmp(lhs); tmp -= rhs; return tmp; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator*(Number lhs, FixedPoint<I, F> rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator*(Number lhs, FixedPoint<I, F> rhs) { FixedPoint<I, F> tmp(lhs); tmp *= rhs; return tmp; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator/(Number lhs, FixedPoint<I, F> rhs) { +template <size_t I, size_t F, IsArithmetic Number> +constexpr FixedPoint<I, F> operator/(Number lhs, FixedPoint<I, F> rhs) { FixedPoint<I, F> tmp(lhs); tmp /= rhs; return tmp; } // shift operators -template <size_t I, size_t F, class Integer, - class = typename std::enable_if<std::is_integral<Integer>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator<<(FixedPoint<I, F> lhs, Integer rhs) { +template <size_t I, size_t F, IsIntegral Integer> +constexpr FixedPoint<I, F> operator<<(FixedPoint<I, F> lhs, Integer rhs) { lhs <<= rhs; return lhs; } -template <size_t I, size_t F, class Integer, - class = typename std::enable_if<std::is_integral<Integer>::value>::type> -CONSTEXPR14 FixedPoint<I, F> operator>>(FixedPoint<I, F> lhs, Integer rhs) { +template <size_t I, size_t F, IsIntegral Integer> +constexpr FixedPoint<I, F> operator>>(FixedPoint<I, F> lhs, Integer rhs) { lhs >>= rhs; return lhs; } // comparison operators -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator>(FixedPoint<I, F> lhs, Number rhs) { return lhs > FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator<(FixedPoint<I, F> lhs, Number rhs) { return lhs < FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator>=(FixedPoint<I, F> lhs, Number rhs) { return lhs >= FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator<=(FixedPoint<I, F> lhs, Number rhs) { return lhs <= FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator==(FixedPoint<I, F> lhs, Number rhs) { return lhs == FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator!=(FixedPoint<I, F> lhs, Number rhs) { return lhs != FixedPoint<I, F>(rhs); } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator>(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) > rhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator<(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) < rhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator>=(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) >= rhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator<=(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) <= rhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator==(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) == rhs; } -template <size_t I, size_t F, class Number, - class = typename std::enable_if<std::is_arithmetic<Number>::value>::type> +template <size_t I, size_t F, IsArithmetic Number> constexpr bool operator!=(Number lhs, FixedPoint<I, F> rhs) { return FixedPoint<I, F>(lhs) != rhs; } } // namespace Common - -#undef CONSTEXPR14 - -#endif diff --git a/src/common/fs/file.h b/src/common/fs/file.h index 69b53384c..167c4d826 100644 --- a/src/common/fs/file.h +++ b/src/common/fs/file.h @@ -209,8 +209,8 @@ public: /** * Helper function which deduces the value type of a contiguous STL container used in ReadSpan. - * If T is not a contiguous STL container as defined by the concept IsSTLContainer, this calls - * ReadObject and T must be a trivially copyable object. + * If T is not a contiguous container as defined by the concept IsContiguousContainer, this + * calls ReadObject and T must be a trivially copyable object. * * See ReadSpan for more details if T is a contiguous container. * See ReadObject for more details if T is a trivially copyable object. @@ -223,7 +223,7 @@ public: */ template <typename T> [[nodiscard]] size_t Read(T& data) const { - if constexpr (IsSTLContainer<T>) { + if constexpr (IsContiguousContainer<T>) { using ContiguousType = typename T::value_type; static_assert(std::is_trivially_copyable_v<ContiguousType>, "Data type must be trivially copyable."); @@ -235,8 +235,8 @@ public: /** * Helper function which deduces the value type of a contiguous STL container used in WriteSpan. - * If T is not a contiguous STL container as defined by the concept IsSTLContainer, this calls - * WriteObject and T must be a trivially copyable object. + * If T is not a contiguous STL container as defined by the concept IsContiguousContainer, this + * calls WriteObject and T must be a trivially copyable object. * * See WriteSpan for more details if T is a contiguous container. * See WriteObject for more details if T is a trivially copyable object. @@ -249,7 +249,7 @@ public: */ template <typename T> [[nodiscard]] size_t Write(const T& data) const { - if constexpr (IsSTLContainer<T>) { + if constexpr (IsContiguousContainer<T>) { using ContiguousType = typename T::value_type; static_assert(std::is_trivially_copyable_v<ContiguousType>, "Data type must be trivially copyable."); diff --git a/src/common/input.h b/src/common/input.h index b533f3844..cb30b7254 100644 --- a/src/common/input.h +++ b/src/common/input.h @@ -100,7 +100,6 @@ enum class CameraError { enum class VibrationAmplificationType { Linear, Exponential, - Test, }; // Analog properties for calibration @@ -325,6 +324,10 @@ public: return VibrationError::NotSupported; } + virtual bool IsVibrationEnabled() { + return false; + } + virtual PollingError SetPollingMode([[maybe_unused]] PollingMode polling_mode) { return PollingError::NotSupported; } diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index abeb5859b..113e663b5 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -190,6 +190,9 @@ add_library(core STATIC hle/kernel/k_code_memory.h hle/kernel/k_condition_variable.cpp hle/kernel/k_condition_variable.h + hle/kernel/k_dynamic_page_manager.h + hle/kernel/k_dynamic_resource_manager.h + hle/kernel/k_dynamic_slab_heap.h hle/kernel/k_event.cpp hle/kernel/k_event.h hle/kernel/k_handle_table.cpp @@ -240,6 +243,8 @@ add_library(core STATIC hle/kernel/k_server_session.h hle/kernel/k_session.cpp hle/kernel/k_session.h + hle/kernel/k_session_request.cpp + hle/kernel/k_session_request.h hle/kernel/k_shared_memory.cpp hle/kernel/k_shared_memory.h hle/kernel/k_shared_memory_info.h @@ -769,19 +774,15 @@ if (MSVC) /we4244 # 'conversion': conversion from 'type1' to 'type2', possible loss of data /we4245 # 'conversion': conversion from 'type1' to 'type2', signed/unsigned mismatch /we4254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data + /we4800 # Implicit conversion from 'type' to bool. Possible information loss ) else() target_compile_options(core PRIVATE -Werror=conversion - -Werror=ignored-qualifiers - $<$<CXX_COMPILER_ID:GNU>:-Werror=class-memaccess> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable> + -Wno-sign-conversion $<$<CXX_COMPILER_ID:Clang>:-fsized-deallocation> - - -Wno-sign-conversion ) endif() diff --git a/src/core/arm/arm_interface.cpp b/src/core/arm/arm_interface.cpp index 953d96439..29ba562dc 100644 --- a/src/core/arm/arm_interface.cpp +++ b/src/core/arm/arm_interface.cpp @@ -134,6 +134,14 @@ void ARM_Interface::Run() { } system.ExitDynarmicProfile(); + // If the thread is scheduled for termination, exit the thread. + if (current_thread->HasDpc()) { + if (current_thread->IsTerminationRequested()) { + current_thread->Exit(); + UNREACHABLE(); + } + } + // Notify the debugger and go to sleep if a breakpoint was hit, // or if the thread is unable to continue for any reason. if (Has(hr, breakpoint) || Has(hr, no_execute)) { diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index d1e70f19d..287ba102e 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -450,7 +450,7 @@ std::vector<ARM_Interface::BacktraceEntry> ARM_Dynarmic_32::GetBacktrace(Core::S // Frame records are two words long: // fp+0 : pointer to previous frame record // fp+4 : value of lr for frame - while (true) { + for (size_t i = 0; i < 256; i++) { out.push_back({"", 0, lr, 0, ""}); if (!fp || (fp % 4 != 0) || !memory.IsValidVirtualAddressRange(fp, 8)) { break; diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 22b5d5656..afb7fb3a0 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -517,7 +517,7 @@ std::vector<ARM_Interface::BacktraceEntry> ARM_Dynarmic_64::GetBacktrace(Core::S // Frame records are two words long: // fp+0 : pointer to previous frame record // fp+8 : value of lr for frame - while (true) { + for (size_t i = 0; i < 256; i++) { out.push_back({"", 0, lr, 0, ""}); if (!fp || (fp % 4 != 0) || !memory.IsValidVirtualAddressRange(fp, 16)) { break; diff --git a/src/core/core.cpp b/src/core/core.cpp index 1deeee154..40a610435 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -133,6 +133,50 @@ struct System::Impl { : kernel{system}, fs_controller{system}, memory{system}, hid_core{}, room_network{}, cpu_manager{system}, reporter{system}, applet_manager{system}, time_manager{system} {} + void Initialize(System& system) { + device_memory = std::make_unique<Core::DeviceMemory>(); + + is_multicore = Settings::values.use_multi_core.GetValue(); + + core_timing.SetMulticore(is_multicore); + core_timing.Initialize([&system]() { system.RegisterHostThread(); }); + + const auto posix_time = std::chrono::system_clock::now().time_since_epoch(); + const auto current_time = + std::chrono::duration_cast<std::chrono::seconds>(posix_time).count(); + Settings::values.custom_rtc_differential = + Settings::values.custom_rtc.value_or(current_time) - current_time; + + // Create a default fs if one doesn't already exist. + if (virtual_filesystem == nullptr) { + virtual_filesystem = std::make_shared<FileSys::RealVfsFilesystem>(); + } + if (content_provider == nullptr) { + content_provider = std::make_unique<FileSys::ContentProviderUnion>(); + } + + // Create default implementations of applets if one is not provided. + applet_manager.SetDefaultAppletsIfMissing(); + + is_async_gpu = Settings::values.use_asynchronous_gpu_emulation.GetValue(); + + kernel.SetMulticore(is_multicore); + cpu_manager.SetMulticore(is_multicore); + cpu_manager.SetAsyncGpu(is_async_gpu); + } + + void ReinitializeIfNecessary(System& system) { + if (is_multicore == Settings::values.use_multi_core.GetValue()) { + return; + } + + LOG_DEBUG(Kernel, "Re-initializing"); + + is_multicore = Settings::values.use_multi_core.GetValue(); + + Initialize(system); + } + SystemResultStatus Run() { std::unique_lock<std::mutex> lk(suspend_guard); status = SystemResultStatus::Success; @@ -178,37 +222,14 @@ struct System::Impl { debugger = std::make_unique<Debugger>(system, port); } - SystemResultStatus Init(System& system, Frontend::EmuWindow& emu_window) { + SystemResultStatus SetupForMainProcess(System& system, Frontend::EmuWindow& emu_window) { LOG_DEBUG(Core, "initialized OK"); - device_memory = std::make_unique<Core::DeviceMemory>(); - - is_multicore = Settings::values.use_multi_core.GetValue(); - is_async_gpu = Settings::values.use_asynchronous_gpu_emulation.GetValue(); - - kernel.SetMulticore(is_multicore); - cpu_manager.SetMulticore(is_multicore); - cpu_manager.SetAsyncGpu(is_async_gpu); - core_timing.SetMulticore(is_multicore); + // Setting changes may require a full system reinitialization (e.g., disabling multicore). + ReinitializeIfNecessary(system); kernel.Initialize(); cpu_manager.Initialize(); - core_timing.Initialize([&system]() { system.RegisterHostThread(); }); - - const auto posix_time = std::chrono::system_clock::now().time_since_epoch(); - const auto current_time = - std::chrono::duration_cast<std::chrono::seconds>(posix_time).count(); - Settings::values.custom_rtc_differential = - Settings::values.custom_rtc.value_or(current_time) - current_time; - - // Create a default fs if one doesn't already exist. - if (virtual_filesystem == nullptr) - virtual_filesystem = std::make_shared<FileSys::RealVfsFilesystem>(); - if (content_provider == nullptr) - content_provider = std::make_unique<FileSys::ContentProviderUnion>(); - - /// Create default implementations of applets if one is not provided. - applet_manager.SetDefaultAppletsIfMissing(); /// Reset all glue registrations arp_manager.ResetAll(); @@ -253,11 +274,11 @@ struct System::Impl { return SystemResultStatus::ErrorGetLoader; } - SystemResultStatus init_result{Init(system, emu_window)}; + SystemResultStatus init_result{SetupForMainProcess(system, emu_window)}; if (init_result != SystemResultStatus::Success) { LOG_CRITICAL(Core, "Failed to initialize system (Error {})!", static_cast<int>(init_result)); - Shutdown(); + ShutdownMainProcess(); return init_result; } @@ -276,7 +297,7 @@ struct System::Impl { const auto [load_result, load_parameters] = app_loader->Load(*main_process, system); if (load_result != Loader::ResultStatus::Success) { LOG_CRITICAL(Core, "Failed to load ROM (Error {})!", load_result); - Shutdown(); + ShutdownMainProcess(); return static_cast<SystemResultStatus>( static_cast<u32>(SystemResultStatus::ErrorLoader) + static_cast<u32>(load_result)); @@ -335,7 +356,7 @@ struct System::Impl { return status; } - void Shutdown() { + void ShutdownMainProcess() { SetShuttingDown(true); // Log last frame performance stats if game was loded @@ -363,13 +384,14 @@ struct System::Impl { kernel.ShutdownCores(); cpu_manager.Shutdown(); debugger.reset(); + services->KillNVNFlinger(); kernel.CloseServices(); services.reset(); service_manager.reset(); cheat_engine.reset(); telemetry_session.reset(); time_manager.Shutdown(); - core_timing.Shutdown(); + core_timing.ClearPendingEvents(); app_loader.reset(); audio_core.reset(); gpu_core.reset(); @@ -377,7 +399,6 @@ struct System::Impl { perf_stats.reset(); kernel.Shutdown(); memory.Reset(); - applet_manager.ClearAll(); if (auto room_member = room_network.GetRoomMember().lock()) { Network::GameInfo game_info{}; @@ -520,6 +541,10 @@ const CpuManager& System::GetCpuManager() const { return impl->cpu_manager; } +void System::Initialize() { + impl->Initialize(*this); +} + SystemResultStatus System::Run() { return impl->Run(); } @@ -540,8 +565,8 @@ void System::InvalidateCpuInstructionCacheRange(VAddr addr, std::size_t size) { impl->kernel.InvalidateCpuInstructionCacheRange(addr, size); } -void System::Shutdown() { - impl->Shutdown(); +void System::ShutdownMainProcess() { + impl->ShutdownMainProcess(); } bool System::IsShuttingDown() const { diff --git a/src/core/core.h b/src/core/core.h index 7843cc8ad..4ebedffd9 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -143,6 +143,12 @@ public: System& operator=(System&&) = delete; /** + * Initializes the system + * This function will initialize core functionaility used for system emulation + */ + void Initialize(); + + /** * Run the OS and Application * This function will start emulation and run the relevant devices */ @@ -166,8 +172,8 @@ public: void InvalidateCpuInstructionCacheRange(VAddr addr, std::size_t size); - /// Shutdown the emulated system. - void Shutdown(); + /// Shutdown the main emulated process. + void ShutdownMainProcess(); /// Check if the core is shutting down. [[nodiscard]] bool IsShuttingDown() const; diff --git a/src/core/core_timing.cpp b/src/core/core_timing.cpp index 2678ce532..0e7b5f943 100644 --- a/src/core/core_timing.cpp +++ b/src/core/core_timing.cpp @@ -40,7 +40,9 @@ struct CoreTiming::Event { CoreTiming::CoreTiming() : clock{Common::CreateBestMatchingClock(Hardware::BASE_CLOCK_RATE, Hardware::CNTFREQ)} {} -CoreTiming::~CoreTiming() = default; +CoreTiming::~CoreTiming() { + Reset(); +} void CoreTiming::ThreadEntry(CoreTiming& instance) { constexpr char name[] = "HostTiming"; @@ -53,6 +55,7 @@ void CoreTiming::ThreadEntry(CoreTiming& instance) { } void CoreTiming::Initialize(std::function<void()>&& on_thread_init_) { + Reset(); on_thread_init = std::move(on_thread_init_); event_fifo_id = 0; shutting_down = false; @@ -65,17 +68,8 @@ void CoreTiming::Initialize(std::function<void()>&& on_thread_init_) { } } -void CoreTiming::Shutdown() { - paused = true; - shutting_down = true; - pause_event.Set(); - event.Set(); - if (timer_thread) { - timer_thread->join(); - } - ClearPendingEvents(); - timer_thread.reset(); - has_started = false; +void CoreTiming::ClearPendingEvents() { + event_queue.clear(); } void CoreTiming::Pause(bool is_paused) { @@ -196,10 +190,6 @@ u64 CoreTiming::GetClockTicks() const { return CpuCyclesToClockCycles(ticks); } -void CoreTiming::ClearPendingEvents() { - event_queue.clear(); -} - void CoreTiming::RemoveEvent(const std::shared_ptr<EventType>& event_type) { std::scoped_lock lock{basic_lock}; @@ -307,6 +297,18 @@ void CoreTiming::ThreadLoop() { } } +void CoreTiming::Reset() { + paused = true; + shutting_down = true; + pause_event.Set(); + event.Set(); + if (timer_thread) { + timer_thread->join(); + } + timer_thread.reset(); + has_started = false; +} + std::chrono::nanoseconds CoreTiming::GetGlobalTimeNs() const { if (is_multicore) { return clock->GetTimeNS(); diff --git a/src/core/core_timing.h b/src/core/core_timing.h index 3259397b2..b5925193c 100644 --- a/src/core/core_timing.h +++ b/src/core/core_timing.h @@ -61,19 +61,14 @@ public: /// required to end slice - 1 and start slice 0 before the first cycle of code is executed. void Initialize(std::function<void()>&& on_thread_init_); - /// Tears down all timing related functionality. - void Shutdown(); + /// Clear all pending events. This should ONLY be done on exit. + void ClearPendingEvents(); /// Sets if emulation is multicore or single core, must be set before Initialize void SetMulticore(bool is_multicore_) { is_multicore = is_multicore_; } - /// Check if it's using host timing. - bool IsHostTiming() const { - return is_multicore; - } - /// Pauses/Unpauses the execution of the timer thread. void Pause(bool is_paused); @@ -136,12 +131,11 @@ public: private: struct Event; - /// Clear all pending events. This should ONLY be done on exit. - void ClearPendingEvents(); - static void ThreadEntry(CoreTiming& instance); void ThreadLoop(); + void Reset(); + std::unique_ptr<Common::WallClock> clock; s64 global_timer = 0; diff --git a/src/core/device_memory.h b/src/core/device_memory.h index df61b0c0b..90510733c 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -31,12 +31,14 @@ public: DramMemoryMap::Base; } - u8* GetPointer(PAddr addr) { - return buffer.BackingBasePointer() + (addr - DramMemoryMap::Base); + template <typename T> + T* GetPointer(PAddr addr) { + return reinterpret_cast<T*>(buffer.BackingBasePointer() + (addr - DramMemoryMap::Base)); } - const u8* GetPointer(PAddr addr) const { - return buffer.BackingBasePointer() + (addr - DramMemoryMap::Base); + template <typename T> + const T* GetPointer(PAddr addr) const { + return reinterpret_cast<T*>(buffer.BackingBasePointer() + (addr - DramMemoryMap::Base)); } Common::HostMemory buffer; diff --git a/src/core/file_sys/card_image.cpp b/src/core/file_sys/card_image.cpp index f23d9373b..5d02865f4 100644 --- a/src/core/file_sys/card_image.cpp +++ b/src/core/file_sys/card_image.cpp @@ -232,8 +232,8 @@ const std::vector<std::shared_ptr<NCA>>& XCI::GetNCAs() const { std::shared_ptr<NCA> XCI::GetNCAByType(NCAContentType type) const { const auto program_id = secure_partition->GetProgramTitleID(); - const auto iter = std::find_if( - ncas.begin(), ncas.end(), [this, type, program_id](const std::shared_ptr<NCA>& nca) { + const auto iter = + std::find_if(ncas.begin(), ncas.end(), [type, program_id](const std::shared_ptr<NCA>& nca) { return nca->GetType() == type && nca->GetTitleId() == program_id; }); return iter == ncas.end() ? nullptr : *iter; diff --git a/src/core/file_sys/control_metadata.cpp b/src/core/file_sys/control_metadata.cpp index be25da2f6..50f44f598 100644 --- a/src/core/file_sys/control_metadata.cpp +++ b/src/core/file_sys/control_metadata.cpp @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include "common/settings.h" #include "common/string_util.h" #include "common/swap.h" #include "core/file_sys/control_metadata.h" @@ -37,6 +38,27 @@ std::string LanguageEntry::GetDeveloperName() const { developer_name.size()); } +constexpr std::array<Language, 18> language_to_codes = {{ + Language::Japanese, + Language::AmericanEnglish, + Language::French, + Language::German, + Language::Italian, + Language::Spanish, + Language::Chinese, + Language::Korean, + Language::Dutch, + Language::Portuguese, + Language::Russian, + Language::Taiwanese, + Language::BritishEnglish, + Language::CanadianFrench, + Language::LatinAmericanSpanish, + Language::Chinese, + Language::Taiwanese, + Language::BrazilianPortuguese, +}}; + NACP::NACP() = default; NACP::NACP(VirtualFile file) { @@ -45,9 +67,13 @@ NACP::NACP(VirtualFile file) { NACP::~NACP() = default; -const LanguageEntry& NACP::GetLanguageEntry(Language language) const { - if (language != Language::Default) { - return raw.language_entries.at(static_cast<u8>(language)); +const LanguageEntry& NACP::GetLanguageEntry() const { + Language language = language_to_codes[Settings::values.language_index.GetValue()]; + + { + const auto& language_entry = raw.language_entries.at(static_cast<u8>(language)); + if (!language_entry.GetApplicationName().empty()) + return language_entry; } for (const auto& language_entry : raw.language_entries) { @@ -55,16 +81,15 @@ const LanguageEntry& NACP::GetLanguageEntry(Language language) const { return language_entry; } - // Fallback to English - return GetLanguageEntry(Language::AmericanEnglish); + return raw.language_entries.at(static_cast<u8>(Language::AmericanEnglish)); } -std::string NACP::GetApplicationName(Language language) const { - return GetLanguageEntry(language).GetApplicationName(); +std::string NACP::GetApplicationName() const { + return GetLanguageEntry().GetApplicationName(); } -std::string NACP::GetDeveloperName(Language language) const { - return GetLanguageEntry(language).GetDeveloperName(); +std::string NACP::GetDeveloperName() const { + return GetLanguageEntry().GetDeveloperName(); } u64 NACP::GetTitleId() const { diff --git a/src/core/file_sys/control_metadata.h b/src/core/file_sys/control_metadata.h index 75295519c..6a81873b1 100644 --- a/src/core/file_sys/control_metadata.h +++ b/src/core/file_sys/control_metadata.h @@ -101,9 +101,9 @@ public: explicit NACP(VirtualFile file); ~NACP(); - const LanguageEntry& GetLanguageEntry(Language language = Language::Default) const; - std::string GetApplicationName(Language language = Language::Default) const; - std::string GetDeveloperName(Language language = Language::Default) const; + const LanguageEntry& GetLanguageEntry() const; + std::string GetApplicationName() const; + std::string GetDeveloperName() const; u64 GetTitleId() const; u64 GetDLCBaseTitleId() const; std::string GetVersionString() const; diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 08d489eab..f00479bd3 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -127,7 +127,7 @@ void ProgramMetadata::LoadManual(bool is_64_bit, ProgramAddressSpaceType address } bool ProgramMetadata::Is64BitProgram() const { - return npdm_header.has_64_bit_instructions; + return npdm_header.has_64_bit_instructions.As<bool>(); } ProgramAddressSpaceType ProgramMetadata::GetAddressSpaceType() const { diff --git a/src/core/file_sys/savedata_factory.cpp b/src/core/file_sys/savedata_factory.cpp index 8c1b2523c..1567da231 100644 --- a/src/core/file_sys/savedata_factory.cpp +++ b/src/core/file_sys/savedata_factory.cpp @@ -5,6 +5,7 @@ #include "common/assert.h" #include "common/common_types.h" #include "common/logging/log.h" +#include "common/uuid.h" #include "core/core.h" #include "core/file_sys/savedata_factory.h" #include "core/file_sys/vfs.h" @@ -59,6 +60,36 @@ bool ShouldSaveDataBeAutomaticallyCreated(SaveDataSpaceId space, const SaveDataA attr.title_id == 0 && attr.save_id == 0); } +std::string GetFutureSaveDataPath(SaveDataSpaceId space_id, SaveDataType type, u64 title_id, + u128 user_id) { + // Only detect nand user saves. + const auto space_id_path = [space_id]() -> std::string_view { + switch (space_id) { + case SaveDataSpaceId::NandUser: + return "/user/save"; + default: + return ""; + } + }(); + + if (space_id_path.empty()) { + return ""; + } + + Common::UUID uuid; + std::memcpy(uuid.uuid.data(), user_id.data(), sizeof(Common::UUID)); + + // Only detect account/device saves from the future location. + switch (type) { + case SaveDataType::SaveData: + return fmt::format("{}/account/{}/{:016X}/1", space_id_path, uuid.RawString(), title_id); + case SaveDataType::DeviceSaveData: + return fmt::format("{}/device/{:016X}/1", space_id_path, title_id); + default: + return ""; + } +} + } // Anonymous namespace std::string SaveDataAttribute::DebugInfo() const { @@ -82,7 +113,7 @@ ResultVal<VirtualDir> SaveDataFactory::Create(SaveDataSpaceId space, PrintSaveDataAttributeWarnings(meta); const auto save_directory = - GetFullPath(system, space, meta.type, meta.title_id, meta.user_id, meta.save_id); + GetFullPath(system, dir, space, meta.type, meta.title_id, meta.user_id, meta.save_id); auto out = dir->CreateDirectoryRelative(save_directory); @@ -99,7 +130,7 @@ ResultVal<VirtualDir> SaveDataFactory::Open(SaveDataSpaceId space, const SaveDataAttribute& meta) const { const auto save_directory = - GetFullPath(system, space, meta.type, meta.title_id, meta.user_id, meta.save_id); + GetFullPath(system, dir, space, meta.type, meta.title_id, meta.user_id, meta.save_id); auto out = dir->GetDirectoryRelative(save_directory); @@ -134,9 +165,9 @@ std::string SaveDataFactory::GetSaveDataSpaceIdPath(SaveDataSpaceId space) { } } -std::string SaveDataFactory::GetFullPath(Core::System& system, SaveDataSpaceId space, - SaveDataType type, u64 title_id, u128 user_id, - u64 save_id) { +std::string SaveDataFactory::GetFullPath(Core::System& system, VirtualDir dir, + SaveDataSpaceId space, SaveDataType type, u64 title_id, + u128 user_id, u64 save_id) { // According to switchbrew, if a save is of type SaveData and the title id field is 0, it should // be interpreted as the title id of the current process. if (type == SaveDataType::SaveData || type == SaveDataType::DeviceSaveData) { @@ -145,6 +176,17 @@ std::string SaveDataFactory::GetFullPath(Core::System& system, SaveDataSpaceId s } } + // For compat with a future impl. + if (std::string future_path = + GetFutureSaveDataPath(space, type, title_id & ~(0xFFULL), user_id); + !future_path.empty()) { + // Check if this location exists, and prefer it over the old. + if (const auto future_dir = dir->GetDirectoryRelative(future_path); future_dir != nullptr) { + LOG_INFO(Service_FS, "Using save at new location: {}", future_path); + return future_path; + } + } + std::string out = GetSaveDataSpaceIdPath(space); switch (type) { @@ -167,7 +209,8 @@ std::string SaveDataFactory::GetFullPath(Core::System& system, SaveDataSpaceId s SaveDataSize SaveDataFactory::ReadSaveDataSize(SaveDataType type, u64 title_id, u128 user_id) const { - const auto path = GetFullPath(system, SaveDataSpaceId::NandUser, type, title_id, user_id, 0); + const auto path = + GetFullPath(system, dir, SaveDataSpaceId::NandUser, type, title_id, user_id, 0); const auto relative_dir = GetOrCreateDirectoryRelative(dir, path); const auto size_file = relative_dir->GetFile(SAVE_DATA_SIZE_FILENAME); @@ -185,7 +228,8 @@ SaveDataSize SaveDataFactory::ReadSaveDataSize(SaveDataType type, u64 title_id, void SaveDataFactory::WriteSaveDataSize(SaveDataType type, u64 title_id, u128 user_id, SaveDataSize new_value) const { - const auto path = GetFullPath(system, SaveDataSpaceId::NandUser, type, title_id, user_id, 0); + const auto path = + GetFullPath(system, dir, SaveDataSpaceId::NandUser, type, title_id, user_id, 0); const auto relative_dir = GetOrCreateDirectoryRelative(dir, path); const auto size_file = relative_dir->CreateFile(SAVE_DATA_SIZE_FILENAME); diff --git a/src/core/file_sys/savedata_factory.h b/src/core/file_sys/savedata_factory.h index a763b94c8..d3633ef03 100644 --- a/src/core/file_sys/savedata_factory.h +++ b/src/core/file_sys/savedata_factory.h @@ -95,8 +95,8 @@ public: VirtualDir GetSaveDataSpaceDirectory(SaveDataSpaceId space) const; static std::string GetSaveDataSpaceIdPath(SaveDataSpaceId space); - static std::string GetFullPath(Core::System& system, SaveDataSpaceId space, SaveDataType type, - u64 title_id, u128 user_id, u64 save_id); + static std::string GetFullPath(Core::System& system, VirtualDir dir, SaveDataSpaceId space, + SaveDataType type, u64 title_id, u128 user_id, u64 save_id); SaveDataSize ReadSaveDataSize(SaveDataType type, u64 title_id, u128 user_id) const; void WriteSaveDataSize(SaveDataType type, u64 title_id, u128 user_id, diff --git a/src/core/hid/emulated_controller.cpp b/src/core/hid/emulated_controller.cpp index 025f1c78e..ec1364452 100644 --- a/src/core/hid/emulated_controller.cpp +++ b/src/core/hid/emulated_controller.cpp @@ -970,14 +970,7 @@ bool EmulatedController::SetVibration(std::size_t device_index, VibrationValue v Common::Input::VibrationError::None; } -bool EmulatedController::TestVibration(std::size_t device_index) { - if (device_index >= output_devices.size()) { - return false; - } - if (!output_devices[device_index]) { - return false; - } - +bool EmulatedController::IsVibrationEnabled(std::size_t device_index) { const auto player_index = NpadIdTypeToIndex(npad_id_type); const auto& player = Settings::values.players.GetValue()[player_index]; @@ -985,31 +978,15 @@ bool EmulatedController::TestVibration(std::size_t device_index) { return false; } - const Common::Input::VibrationStatus test_vibration = { - .low_amplitude = 0.001f, - .low_frequency = DEFAULT_VIBRATION_VALUE.low_frequency, - .high_amplitude = 0.001f, - .high_frequency = DEFAULT_VIBRATION_VALUE.high_frequency, - .type = Common::Input::VibrationAmplificationType::Test, - }; - - const Common::Input::VibrationStatus zero_vibration = { - .low_amplitude = DEFAULT_VIBRATION_VALUE.low_amplitude, - .low_frequency = DEFAULT_VIBRATION_VALUE.low_frequency, - .high_amplitude = DEFAULT_VIBRATION_VALUE.high_amplitude, - .high_frequency = DEFAULT_VIBRATION_VALUE.high_frequency, - .type = Common::Input::VibrationAmplificationType::Test, - }; - - // Send a slight vibration to test for rumble support - output_devices[device_index]->SetVibration(test_vibration); + if (device_index >= output_devices.size()) { + return false; + } - // Wait for about 15ms to ensure the controller is ready for the stop command - std::this_thread::sleep_for(std::chrono::milliseconds(15)); + if (!output_devices[device_index]) { + return false; + } - // Stop any vibration and return the result - return output_devices[device_index]->SetVibration(zero_vibration) == - Common::Input::VibrationError::None; + return output_devices[device_index]->IsVibrationEnabled(); } bool EmulatedController::SetPollingMode(Common::Input::PollingMode polling_mode) { @@ -1048,6 +1025,7 @@ bool EmulatedController::HasNfc() const { case NpadStyleIndex::JoyconRight: case NpadStyleIndex::JoyconDual: case NpadStyleIndex::ProController: + case NpadStyleIndex::Handheld: break; default: return false; @@ -1158,27 +1136,27 @@ bool EmulatedController::IsControllerSupported(bool use_temporary_value) const { const auto type = is_configuring && use_temporary_value ? tmp_npad_type : npad_type; switch (type) { case NpadStyleIndex::ProController: - return supported_style_tag.fullkey; + return supported_style_tag.fullkey.As<bool>(); case NpadStyleIndex::Handheld: - return supported_style_tag.handheld; + return supported_style_tag.handheld.As<bool>(); case NpadStyleIndex::JoyconDual: - return supported_style_tag.joycon_dual; + return supported_style_tag.joycon_dual.As<bool>(); case NpadStyleIndex::JoyconLeft: - return supported_style_tag.joycon_left; + return supported_style_tag.joycon_left.As<bool>(); case NpadStyleIndex::JoyconRight: - return supported_style_tag.joycon_right; + return supported_style_tag.joycon_right.As<bool>(); case NpadStyleIndex::GameCube: - return supported_style_tag.gamecube; + return supported_style_tag.gamecube.As<bool>(); case NpadStyleIndex::Pokeball: - return supported_style_tag.palma; + return supported_style_tag.palma.As<bool>(); case NpadStyleIndex::NES: - return supported_style_tag.lark; + return supported_style_tag.lark.As<bool>(); case NpadStyleIndex::SNES: - return supported_style_tag.lucia; + return supported_style_tag.lucia.As<bool>(); case NpadStyleIndex::N64: - return supported_style_tag.lagoon; + return supported_style_tag.lagoon.As<bool>(); case NpadStyleIndex::SegaGenesis: - return supported_style_tag.lager; + return supported_style_tag.lager.As<bool>(); default: return false; } @@ -1234,12 +1212,6 @@ bool EmulatedController::IsConnected(bool get_temporary_value) const { return is_connected; } -bool EmulatedController::IsVibrationEnabled() const { - const auto player_index = NpadIdTypeToIndex(npad_id_type); - const auto& player = Settings::values.players.GetValue()[player_index]; - return player.vibration_enabled; -} - NpadIdType EmulatedController::GetNpadIdType() const { std::scoped_lock lock{mutex}; return npad_id_type; diff --git a/src/core/hid/emulated_controller.h b/src/core/hid/emulated_controller.h index 319226bf8..d004ca56a 100644 --- a/src/core/hid/emulated_controller.h +++ b/src/core/hid/emulated_controller.h @@ -206,9 +206,6 @@ public: */ bool IsConnected(bool get_temporary_value = false) const; - /// Returns true if vibration is enabled - bool IsVibrationEnabled() const; - /// Removes all callbacks created from input devices void UnloadInput(); @@ -339,7 +336,7 @@ public: * Sends a small vibration to the output device * @return true if SetVibration was successfull */ - bool TestVibration(std::size_t device_index); + bool IsVibrationEnabled(std::size_t device_index); /** * Sets the desired data to be polled from a controller diff --git a/src/core/hle/ipc_helpers.h b/src/core/hle/ipc_helpers.h index 0cc26a211..18fde8bd6 100644 --- a/src/core/hle/ipc_helpers.h +++ b/src/core/hle/ipc_helpers.h @@ -86,13 +86,13 @@ public: u32 num_domain_objects{}; const bool always_move_handles{ (static_cast<u32>(flags) & static_cast<u32>(Flags::AlwaysMoveHandles)) != 0}; - if (!ctx.Session()->IsDomain() || always_move_handles) { + if (!ctx.Session()->GetSessionRequestManager()->IsDomain() || always_move_handles) { num_handles_to_move = num_objects_to_move; } else { num_domain_objects = num_objects_to_move; } - if (ctx.Session()->IsDomain()) { + if (ctx.Session()->GetSessionRequestManager()->IsDomain()) { raw_data_size += static_cast<u32>(sizeof(DomainMessageHeader) / sizeof(u32) + num_domain_objects); ctx.write_size += num_domain_objects; @@ -125,7 +125,8 @@ public: if (!ctx.IsTipc()) { AlignWithPadding(); - if (ctx.Session()->IsDomain() && ctx.HasDomainMessageHeader()) { + if (ctx.Session()->GetSessionRequestManager()->IsDomain() && + ctx.HasDomainMessageHeader()) { IPC::DomainMessageHeader domain_header{}; domain_header.num_objects = num_domain_objects; PushRaw(domain_header); @@ -145,7 +146,7 @@ public: template <class T> void PushIpcInterface(std::shared_ptr<T> iface) { - if (context->Session()->IsDomain()) { + if (context->Session()->GetSessionRequestManager()->IsDomain()) { context->AddDomainObject(std::move(iface)); } else { kernel.CurrentProcess()->GetResourceLimit()->Reserve( @@ -386,7 +387,7 @@ public: template <class T> std::weak_ptr<T> PopIpcInterface() { - ASSERT(context->Session()->IsDomain()); + ASSERT(context->Session()->GetSessionRequestManager()->IsDomain()); ASSERT(context->GetDomainMessageHeader().input_object_count > 0); return context->GetDomainHandler<T>(Pop<u32>() - 1); } @@ -405,7 +406,7 @@ inline s32 RequestParser::Pop() { } // Ignore the -Wclass-memaccess warning on memcpy for non-trivially default constructible objects. -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #endif @@ -416,7 +417,7 @@ void RequestParser::PopRaw(T& value) { std::memcpy(&value, cmdbuf + index, sizeof(T)); index += (sizeof(T) + 3) / 4; // round up to word length } -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) #pragma GCC diagnostic pop #endif diff --git a/src/core/hle/kernel/global_scheduler_context.cpp b/src/core/hle/kernel/global_scheduler_context.cpp index 65576b8c4..fd911a3a5 100644 --- a/src/core/hle/kernel/global_scheduler_context.cpp +++ b/src/core/hle/kernel/global_scheduler_context.cpp @@ -49,4 +49,26 @@ bool GlobalSchedulerContext::IsLocked() const { return scheduler_lock.IsLockedByCurrentThread(); } +void GlobalSchedulerContext::RegisterDummyThreadForWakeup(KThread* thread) { + ASSERT(IsLocked()); + + woken_dummy_threads.insert(thread); +} + +void GlobalSchedulerContext::UnregisterDummyThreadForWakeup(KThread* thread) { + ASSERT(IsLocked()); + + woken_dummy_threads.erase(thread); +} + +void GlobalSchedulerContext::WakeupWaitingDummyThreads() { + ASSERT(IsLocked()); + + for (auto* thread : woken_dummy_threads) { + thread->DummyThreadEndWait(); + } + + woken_dummy_threads.clear(); +} + } // namespace Kernel diff --git a/src/core/hle/kernel/global_scheduler_context.h b/src/core/hle/kernel/global_scheduler_context.h index 67bb9852d..220ed6192 100644 --- a/src/core/hle/kernel/global_scheduler_context.h +++ b/src/core/hle/kernel/global_scheduler_context.h @@ -4,6 +4,7 @@ #pragma once #include <atomic> +#include <set> #include <vector> #include "common/common_types.h" @@ -58,6 +59,10 @@ public: /// Returns true if the global scheduler lock is acquired bool IsLocked() const; + void UnregisterDummyThreadForWakeup(KThread* thread); + void RegisterDummyThreadForWakeup(KThread* thread); + void WakeupWaitingDummyThreads(); + [[nodiscard]] LockType& SchedulerLock() { return scheduler_lock; } @@ -76,6 +81,9 @@ private: KSchedulerPriorityQueue priority_queue; LockType scheduler_lock; + /// Lists dummy threads pending wakeup on lock release + std::set<KThread*> woken_dummy_threads; + /// Lists all thread ids that aren't deleted/etc. std::vector<KThread*> thread_list; std::mutex global_list_guard; diff --git a/src/core/hle/kernel/hle_ipc.cpp b/src/core/hle/kernel/hle_ipc.cpp index 5b3feec66..e4f43a053 100644 --- a/src/core/hle/kernel/hle_ipc.cpp +++ b/src/core/hle/kernel/hle_ipc.cpp @@ -19,6 +19,7 @@ #include "core/hle/kernel/k_server_session.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/service_thread.h" #include "core/memory.h" namespace Kernel { @@ -56,16 +57,103 @@ bool SessionRequestManager::HasSessionRequestHandler(const HLERequestContext& co } } +Result SessionRequestManager::CompleteSyncRequest(KServerSession* server_session, + HLERequestContext& context) { + Result result = ResultSuccess; + + // If the session has been converted to a domain, handle the domain request + if (this->HasSessionRequestHandler(context)) { + if (IsDomain() && context.HasDomainMessageHeader()) { + result = HandleDomainSyncRequest(server_session, context); + // If there is no domain header, the regular session handler is used + } else if (this->HasSessionHandler()) { + // If this manager has an associated HLE handler, forward the request to it. + result = this->SessionHandler().HandleSyncRequest(*server_session, context); + } + } else { + ASSERT_MSG(false, "Session handler is invalid, stubbing response!"); + IPC::ResponseBuilder rb(context, 2); + rb.Push(ResultSuccess); + } + + if (convert_to_domain) { + ASSERT_MSG(!IsDomain(), "ServerSession is already a domain instance."); + this->ConvertToDomain(); + convert_to_domain = false; + } + + return result; +} + +Result SessionRequestManager::HandleDomainSyncRequest(KServerSession* server_session, + HLERequestContext& context) { + if (!context.HasDomainMessageHeader()) { + return ResultSuccess; + } + + // Set domain handlers in HLE context, used for domain objects (IPC interfaces) as inputs + context.SetSessionRequestManager(server_session->GetSessionRequestManager()); + + // If there is a DomainMessageHeader, then this is CommandType "Request" + const auto& domain_message_header = context.GetDomainMessageHeader(); + const u32 object_id{domain_message_header.object_id}; + switch (domain_message_header.command) { + case IPC::DomainMessageHeader::CommandType::SendMessage: + if (object_id > this->DomainHandlerCount()) { + LOG_CRITICAL(IPC, + "object_id {} is too big! This probably means a recent service call " + "needed to return a new interface!", + object_id); + ASSERT(false); + return ResultSuccess; // Ignore error if asserts are off + } + if (auto strong_ptr = this->DomainHandler(object_id - 1).lock()) { + return strong_ptr->HandleSyncRequest(*server_session, context); + } else { + ASSERT(false); + return ResultSuccess; + } + + case IPC::DomainMessageHeader::CommandType::CloseVirtualHandle: { + LOG_DEBUG(IPC, "CloseVirtualHandle, object_id=0x{:08X}", object_id); + + this->CloseDomainHandler(object_id - 1); + + IPC::ResponseBuilder rb{context, 2}; + rb.Push(ResultSuccess); + return ResultSuccess; + } + } + + LOG_CRITICAL(IPC, "Unknown domain command={}", domain_message_header.command.Value()); + ASSERT(false); + return ResultSuccess; +} + +Result SessionRequestManager::QueueSyncRequest(KSession* parent, + std::shared_ptr<HLERequestContext>&& context) { + // Ensure we have a session request handler + if (this->HasSessionRequestHandler(*context)) { + if (auto strong_ptr = this->GetServiceThread().lock()) { + strong_ptr->QueueSyncRequest(*parent, std::move(context)); + } else { + ASSERT_MSG(false, "strong_ptr is nullptr!"); + } + } else { + ASSERT_MSG(false, "handler is invalid!"); + } + + return ResultSuccess; +} + void SessionRequestHandler::ClientConnected(KServerSession* session) { - session->ClientConnected(shared_from_this()); + session->GetSessionRequestManager()->SetSessionHandler(shared_from_this()); // Ensure our server session is tracked globally. kernel.RegisterServerObject(session); } -void SessionRequestHandler::ClientDisconnected(KServerSession* session) { - session->ClientDisconnected(); -} +void SessionRequestHandler::ClientDisconnected(KServerSession* session) {} HLERequestContext::HLERequestContext(KernelCore& kernel_, Core::Memory::Memory& memory_, KServerSession* server_session_, KThread* thread_) @@ -126,7 +214,7 @@ void HLERequestContext::ParseCommandBuffer(const KHandleTable& handle_table, u32 // Padding to align to 16 bytes rp.AlignWithPadding(); - if (Session()->IsDomain() && + if (Session()->GetSessionRequestManager()->IsDomain() && ((command_header->type == IPC::CommandType::Request || command_header->type == IPC::CommandType::RequestWithContext) || !incoming)) { @@ -135,7 +223,7 @@ void HLERequestContext::ParseCommandBuffer(const KHandleTable& handle_table, u32 if (incoming || domain_message_header) { domain_message_header = rp.PopRaw<IPC::DomainMessageHeader>(); } else { - if (Session()->IsDomain()) { + if (Session()->GetSessionRequestManager()->IsDomain()) { LOG_WARNING(IPC, "Domain request has no DomainMessageHeader!"); } } @@ -228,12 +316,12 @@ Result HLERequestContext::WriteToOutgoingCommandBuffer(KThread& requesting_threa // Write the domain objects to the command buffer, these go after the raw untranslated data. // TODO(Subv): This completely ignores C buffers. - if (Session()->IsDomain()) { + if (server_session->GetSessionRequestManager()->IsDomain()) { current_offset = domain_offset - static_cast<u32>(outgoing_domain_objects.size()); - for (const auto& object : outgoing_domain_objects) { - server_session->AppendDomainHandler(object); - cmd_buf[current_offset++] = - static_cast<u32_le>(server_session->NumDomainRequestHandlers()); + for (auto& object : outgoing_domain_objects) { + server_session->GetSessionRequestManager()->AppendDomainHandler(std::move(object)); + cmd_buf[current_offset++] = static_cast<u32_le>( + server_session->GetSessionRequestManager()->DomainHandlerCount()); } } diff --git a/src/core/hle/kernel/hle_ipc.h b/src/core/hle/kernel/hle_ipc.h index e258e2cdf..1083638a9 100644 --- a/src/core/hle/kernel/hle_ipc.h +++ b/src/core/hle/kernel/hle_ipc.h @@ -121,6 +121,10 @@ public: is_domain = true; } + void ConvertToDomainOnRequestEnd() { + convert_to_domain = true; + } + std::size_t DomainHandlerCount() const { return domain_handlers.size(); } @@ -164,7 +168,12 @@ public: bool HasSessionRequestHandler(const HLERequestContext& context) const; + Result HandleDomainSyncRequest(KServerSession* server_session, HLERequestContext& context); + Result CompleteSyncRequest(KServerSession* server_session, HLERequestContext& context); + Result QueueSyncRequest(KSession* parent, std::shared_ptr<HLERequestContext>&& context); + private: + bool convert_to_domain{}; bool is_domain{}; SessionRequestHandlerPtr session_handler; std::vector<SessionRequestHandlerPtr> domain_handlers; @@ -295,7 +304,7 @@ public: */ template <typename T, typename = std::enable_if_t<!std::is_pointer_v<T>>> std::size_t WriteBuffer(const T& data, std::size_t buffer_index = 0) const { - if constexpr (Common::IsSTLContainer<T>) { + if constexpr (Common::IsContiguousContainer<T>) { using ContiguousType = typename T::value_type; static_assert(std::is_trivially_copyable_v<ContiguousType>, "Container to WriteBuffer must contain trivially copyable objects"); diff --git a/src/core/hle/kernel/init/init_slab_setup.cpp b/src/core/hle/kernel/init/init_slab_setup.cpp index 9b6b284d0..477e4e407 100644 --- a/src/core/hle/kernel/init/init_slab_setup.cpp +++ b/src/core/hle/kernel/init/init_slab_setup.cpp @@ -18,6 +18,7 @@ #include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_session.h" +#include "core/hle/kernel/k_session_request.h" #include "core/hle/kernel/k_shared_memory.h" #include "core/hle/kernel/k_shared_memory_info.h" #include "core/hle/kernel/k_system_control.h" @@ -34,6 +35,7 @@ namespace Kernel::Init { HANDLER(KThread, (SLAB_COUNT(KThread)), ##__VA_ARGS__) \ HANDLER(KEvent, (SLAB_COUNT(KEvent)), ##__VA_ARGS__) \ HANDLER(KPort, (SLAB_COUNT(KPort)), ##__VA_ARGS__) \ + HANDLER(KSessionRequest, (SLAB_COUNT(KSession) * 2), ##__VA_ARGS__) \ HANDLER(KSharedMemory, (SLAB_COUNT(KSharedMemory)), ##__VA_ARGS__) \ HANDLER(KSharedMemoryInfo, (SLAB_COUNT(KSharedMemory) * 8), ##__VA_ARGS__) \ HANDLER(KTransferMemory, (SLAB_COUNT(KTransferMemory)), ##__VA_ARGS__) \ @@ -94,8 +96,8 @@ VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAd // TODO(bunnei): Fix this once we support the kernel virtual memory layout. if (size > 0) { - void* backing_kernel_memory{ - system.DeviceMemory().GetPointer(TranslateSlabAddrToPhysical(memory_layout, start))}; + void* backing_kernel_memory{system.DeviceMemory().GetPointer<void>( + TranslateSlabAddrToPhysical(memory_layout, start))}; const KMemoryRegion* region = memory_layout.FindVirtual(start + size - 1); ASSERT(region != nullptr); @@ -181,7 +183,7 @@ void InitializeKPageBufferSlabHeap(Core::System& system) { ASSERT(slab_address != 0); // Initialize the slabheap. - KPageBuffer::InitializeSlabHeap(kernel, system.DeviceMemory().GetPointer(slab_address), + KPageBuffer::InitializeSlabHeap(kernel, system.DeviceMemory().GetPointer<void>(slab_address), slab_size); } diff --git a/src/core/hle/kernel/k_client_session.cpp b/src/core/hle/kernel/k_client_session.cpp index 8892c5b7c..b4197a8d5 100644 --- a/src/core/hle/kernel/k_client_session.cpp +++ b/src/core/hle/kernel/k_client_session.cpp @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include "common/scope_exit.h" #include "core/hle/kernel/hle_ipc.h" #include "core/hle/kernel/k_client_session.h" #include "core/hle/kernel/k_server_session.h" @@ -10,6 +11,8 @@ namespace Kernel { +static constexpr u32 MessageBufferSize = 0x100; + KClientSession::KClientSession(KernelCore& kernel_) : KAutoObjectWithSlabHeapAndContainer{kernel_} {} KClientSession::~KClientSession() = default; @@ -22,8 +25,16 @@ void KClientSession::Destroy() { void KClientSession::OnServerClosed() {} Result KClientSession::SendSyncRequest() { - // Signal the server session that new data is available - return parent->GetServerSession().OnRequest(); + // Create a session request. + KSessionRequest* request = KSessionRequest::Create(kernel); + R_UNLESS(request != nullptr, ResultOutOfResource); + SCOPE_EXIT({ request->Close(); }); + + // Initialize the request. + request->Initialize(nullptr, GetCurrentThread(kernel).GetTLSAddress(), MessageBufferSize); + + // Send the request. + return parent->GetServerSession().OnRequest(request); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_code_memory.cpp b/src/core/hle/kernel/k_code_memory.cpp index da57ceb21..4b1c134d4 100644 --- a/src/core/hle/kernel/k_code_memory.cpp +++ b/src/core/hle/kernel/k_code_memory.cpp @@ -34,7 +34,7 @@ Result KCodeMemory::Initialize(Core::DeviceMemory& device_memory, VAddr addr, si // Clear the memory. for (const auto& block : m_page_group.Nodes()) { - std::memset(device_memory.GetPointer(block.GetAddress()), 0xFF, block.GetSize()); + std::memset(device_memory.GetPointer<void>(block.GetAddress()), 0xFF, block.GetSize()); } // Set remaining tracking members. diff --git a/src/core/hle/kernel/k_dynamic_page_manager.h b/src/core/hle/kernel/k_dynamic_page_manager.h new file mode 100644 index 000000000..9076c8fa3 --- /dev/null +++ b/src/core/hle/kernel/k_dynamic_page_manager.h @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "common/alignment.h" +#include "common/common_types.h" +#include "core/hle/kernel/k_page_bitmap.h" +#include "core/hle/kernel/k_spin_lock.h" +#include "core/hle/kernel/memory_types.h" +#include "core/hle/kernel/svc_results.h" + +namespace Kernel { + +class KDynamicPageManager { +public: + class PageBuffer { + private: + u8 m_buffer[PageSize]; + }; + static_assert(sizeof(PageBuffer) == PageSize); + +public: + KDynamicPageManager() = default; + + template <typename T> + T* GetPointer(VAddr addr) { + return reinterpret_cast<T*>(m_backing_memory.data() + (addr - m_address)); + } + + template <typename T> + const T* GetPointer(VAddr addr) const { + return reinterpret_cast<T*>(m_backing_memory.data() + (addr - m_address)); + } + + Result Initialize(VAddr addr, size_t sz) { + // We need to have positive size. + R_UNLESS(sz > 0, ResultOutOfMemory); + m_backing_memory.resize(sz); + + // Calculate management overhead. + const size_t management_size = + KPageBitmap::CalculateManagementOverheadSize(sz / sizeof(PageBuffer)); + const size_t allocatable_size = sz - management_size; + + // Set tracking fields. + m_address = addr; + m_size = Common::AlignDown(allocatable_size, sizeof(PageBuffer)); + m_count = allocatable_size / sizeof(PageBuffer); + R_UNLESS(m_count > 0, ResultOutOfMemory); + + // Clear the management region. + u64* management_ptr = GetPointer<u64>(m_address + allocatable_size); + std::memset(management_ptr, 0, management_size); + + // Initialize the bitmap. + m_page_bitmap.Initialize(management_ptr, m_count); + + // Free the pages to the bitmap. + for (size_t i = 0; i < m_count; i++) { + // Ensure the freed page is all-zero. + std::memset(GetPointer<PageBuffer>(m_address) + i, 0, PageSize); + + // Set the bit for the free page. + m_page_bitmap.SetBit(i); + } + + R_SUCCEED(); + } + + VAddr GetAddress() const { + return m_address; + } + size_t GetSize() const { + return m_size; + } + size_t GetUsed() const { + return m_used; + } + size_t GetPeak() const { + return m_peak; + } + size_t GetCount() const { + return m_count; + } + + PageBuffer* Allocate() { + // Take the lock. + // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable. + KScopedSpinLock lk(m_lock); + + // Find a random free block. + s64 soffset = m_page_bitmap.FindFreeBlock(true); + if (soffset < 0) [[unlikely]] { + return nullptr; + } + + const size_t offset = static_cast<size_t>(soffset); + + // Update our tracking. + m_page_bitmap.ClearBit(offset); + m_peak = std::max(m_peak, (++m_used)); + + return GetPointer<PageBuffer>(m_address) + offset; + } + + void Free(PageBuffer* pb) { + // Ensure all pages in the heap are zero. + std::memset(pb, 0, PageSize); + + // Take the lock. + // TODO(bunnei): We should disable interrupts here via KScopedInterruptDisable. + KScopedSpinLock lk(m_lock); + + // Set the bit for the free page. + size_t offset = (reinterpret_cast<uintptr_t>(pb) - m_address) / sizeof(PageBuffer); + m_page_bitmap.SetBit(offset); + + // Decrement our used count. + --m_used; + } + +private: + KSpinLock m_lock; + KPageBitmap m_page_bitmap; + size_t m_used{}; + size_t m_peak{}; + size_t m_count{}; + VAddr m_address{}; + size_t m_size{}; + + // TODO(bunnei): Back by host memory until we emulate kernel virtual address space. + std::vector<u8> m_backing_memory; +}; + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_dynamic_resource_manager.h b/src/core/hle/kernel/k_dynamic_resource_manager.h new file mode 100644 index 000000000..1ce517e8e --- /dev/null +++ b/src/core/hle/kernel/k_dynamic_resource_manager.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "common/common_funcs.h" +#include "core/hle/kernel/k_dynamic_slab_heap.h" +#include "core/hle/kernel/k_memory_block.h" + +namespace Kernel { + +template <typename T, bool ClearNode = false> +class KDynamicResourceManager { + YUZU_NON_COPYABLE(KDynamicResourceManager); + YUZU_NON_MOVEABLE(KDynamicResourceManager); + +public: + using DynamicSlabType = KDynamicSlabHeap<T, ClearNode>; + +public: + constexpr KDynamicResourceManager() = default; + + constexpr size_t GetSize() const { + return m_slab_heap->GetSize(); + } + constexpr size_t GetUsed() const { + return m_slab_heap->GetUsed(); + } + constexpr size_t GetPeak() const { + return m_slab_heap->GetPeak(); + } + constexpr size_t GetCount() const { + return m_slab_heap->GetCount(); + } + + void Initialize(KDynamicPageManager* page_allocator, DynamicSlabType* slab_heap) { + m_page_allocator = page_allocator; + m_slab_heap = slab_heap; + } + + T* Allocate() const { + return m_slab_heap->Allocate(m_page_allocator); + } + + void Free(T* t) const { + m_slab_heap->Free(t); + } + +private: + KDynamicPageManager* m_page_allocator{}; + DynamicSlabType* m_slab_heap{}; +}; + +class KMemoryBlockSlabManager : public KDynamicResourceManager<KMemoryBlock> {}; + +using KMemoryBlockSlabHeap = typename KMemoryBlockSlabManager::DynamicSlabType; + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_dynamic_slab_heap.h b/src/core/hle/kernel/k_dynamic_slab_heap.h new file mode 100644 index 000000000..3a0ddd050 --- /dev/null +++ b/src/core/hle/kernel/k_dynamic_slab_heap.h @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include <atomic> + +#include "common/common_funcs.h" +#include "core/hle/kernel/k_dynamic_page_manager.h" +#include "core/hle/kernel/k_slab_heap.h" + +namespace Kernel { + +template <typename T, bool ClearNode = false> +class KDynamicSlabHeap : protected impl::KSlabHeapImpl { + YUZU_NON_COPYABLE(KDynamicSlabHeap); + YUZU_NON_MOVEABLE(KDynamicSlabHeap); + +public: + constexpr KDynamicSlabHeap() = default; + + constexpr VAddr GetAddress() const { + return m_address; + } + constexpr size_t GetSize() const { + return m_size; + } + constexpr size_t GetUsed() const { + return m_used.load(); + } + constexpr size_t GetPeak() const { + return m_peak.load(); + } + constexpr size_t GetCount() const { + return m_count.load(); + } + + constexpr bool IsInRange(VAddr addr) const { + return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1; + } + + void Initialize(KDynamicPageManager* page_allocator, size_t num_objects) { + ASSERT(page_allocator != nullptr); + + // Initialize members. + m_address = page_allocator->GetAddress(); + m_size = page_allocator->GetSize(); + + // Initialize the base allocator. + KSlabHeapImpl::Initialize(); + + // Allocate until we have the correct number of objects. + while (m_count.load() < num_objects) { + auto* allocated = reinterpret_cast<T*>(page_allocator->Allocate()); + ASSERT(allocated != nullptr); + + for (size_t i = 0; i < sizeof(PageBuffer) / sizeof(T); i++) { + KSlabHeapImpl::Free(allocated + i); + } + + m_count += sizeof(PageBuffer) / sizeof(T); + } + } + + T* Allocate(KDynamicPageManager* page_allocator) { + T* allocated = static_cast<T*>(KSlabHeapImpl::Allocate()); + + // If we successfully allocated and we should clear the node, do so. + if constexpr (ClearNode) { + if (allocated != nullptr) [[likely]] { + reinterpret_cast<KSlabHeapImpl::Node*>(allocated)->next = nullptr; + } + } + + // If we fail to allocate, try to get a new page from our next allocator. + if (allocated == nullptr) [[unlikely]] { + if (page_allocator != nullptr) { + allocated = reinterpret_cast<T*>(page_allocator->Allocate()); + if (allocated != nullptr) { + // If we succeeded in getting a page, free the rest to our slab. + for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) { + KSlabHeapImpl::Free(allocated + i); + } + m_count += sizeof(PageBuffer) / sizeof(T); + } + } + } + + if (allocated != nullptr) [[likely]] { + // Construct the object. + std::construct_at(allocated); + + // Update our tracking. + const size_t used = ++m_used; + size_t peak = m_peak.load(); + while (peak < used) { + if (m_peak.compare_exchange_weak(peak, used, std::memory_order_relaxed)) { + break; + } + } + } + + return allocated; + } + + void Free(T* t) { + KSlabHeapImpl::Free(t); + --m_used; + } + +private: + using PageBuffer = KDynamicPageManager::PageBuffer; + +private: + std::atomic<size_t> m_used{}; + std::atomic<size_t> m_peak{}; + std::atomic<size_t> m_count{}; + VAddr m_address{}; + size_t m_size{}; +}; + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_interrupt_manager.cpp b/src/core/hle/kernel/k_interrupt_manager.cpp index 1b577a5b3..4a6b60d26 100644 --- a/src/core/hle/kernel/k_interrupt_manager.cpp +++ b/src/core/hle/kernel/k_interrupt_manager.cpp @@ -11,29 +11,34 @@ namespace Kernel::KInterruptManager { void HandleInterrupt(KernelCore& kernel, s32 core_id) { - auto* process = kernel.CurrentProcess(); - if (!process) { - return; - } - // Acknowledge the interrupt. kernel.PhysicalCore(core_id).ClearInterrupt(); auto& current_thread = GetCurrentThread(kernel); - // If the user disable count is set, we may need to pin the current thread. - if (current_thread.GetUserDisableCount() && !process->GetPinnedThread(core_id)) { - KScopedSchedulerLock sl{kernel}; + if (auto* process = kernel.CurrentProcess(); process) { + // If the user disable count is set, we may need to pin the current thread. + if (current_thread.GetUserDisableCount() && !process->GetPinnedThread(core_id)) { + KScopedSchedulerLock sl{kernel}; - // Pin the current thread. - process->PinCurrentThread(core_id); + // Pin the current thread. + process->PinCurrentThread(core_id); - // Set the interrupt flag for the thread. - GetCurrentThread(kernel).SetInterruptFlag(); + // Set the interrupt flag for the thread. + GetCurrentThread(kernel).SetInterruptFlag(); + } } // Request interrupt scheduling. kernel.CurrentScheduler()->RequestScheduleOnInterrupt(); } +void SendInterProcessorInterrupt(KernelCore& kernel, u64 core_mask) { + for (std::size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; ++core_id) { + if (core_mask & (1ULL << core_id)) { + kernel.PhysicalCore(core_id).Interrupt(); + } + } +} + } // namespace Kernel::KInterruptManager diff --git a/src/core/hle/kernel/k_interrupt_manager.h b/src/core/hle/kernel/k_interrupt_manager.h index f103dfe3f..803dc9211 100644 --- a/src/core/hle/kernel/k_interrupt_manager.h +++ b/src/core/hle/kernel/k_interrupt_manager.h @@ -11,6 +11,8 @@ class KernelCore; namespace KInterruptManager { void HandleInterrupt(KernelCore& kernel, s32 core_id); -} +void SendInterProcessorInterrupt(KernelCore& kernel, u64 core_mask); + +} // namespace KInterruptManager } // namespace Kernel diff --git a/src/core/hle/kernel/k_linked_list.h b/src/core/hle/kernel/k_linked_list.h index 78859ced3..29ebd16b7 100644 --- a/src/core/hle/kernel/k_linked_list.h +++ b/src/core/hle/kernel/k_linked_list.h @@ -16,6 +16,7 @@ class KLinkedListNode : public boost::intrusive::list_base_hook<>, public KSlabAllocated<KLinkedListNode> { public: + explicit KLinkedListNode(KernelCore&) {} KLinkedListNode() = default; void Initialize(void* it) { diff --git a/src/core/hle/kernel/k_memory_block.h b/src/core/hle/kernel/k_memory_block.h index 18df1f836..9444f6bd2 100644 --- a/src/core/hle/kernel/k_memory_block.h +++ b/src/core/hle/kernel/k_memory_block.h @@ -6,6 +6,7 @@ #include "common/alignment.h" #include "common/assert.h" #include "common/common_types.h" +#include "common/intrusive_red_black_tree.h" #include "core/hle/kernel/memory_types.h" #include "core/hle/kernel/svc_types.h" @@ -168,9 +169,8 @@ constexpr KMemoryPermission ConvertToKMemoryPermission(Svc::MemoryPermission per enum class KMemoryAttribute : u8 { None = 0x00, - Mask = 0x7F, - All = Mask, - DontCareMask = 0x80, + All = 0xFF, + UserMask = All, Locked = static_cast<u8>(Svc::MemoryAttribute::Locked), IpcLocked = static_cast<u8>(Svc::MemoryAttribute::IpcLocked), @@ -178,76 +178,112 @@ enum class KMemoryAttribute : u8 { Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached), SetMask = Uncached, - - IpcAndDeviceMapped = IpcLocked | DeviceShared, - LockedAndIpcLocked = Locked | IpcLocked, - DeviceSharedAndUncached = DeviceShared | Uncached }; DECLARE_ENUM_FLAG_OPERATORS(KMemoryAttribute); -static_assert((static_cast<u8>(KMemoryAttribute::Mask) & - static_cast<u8>(KMemoryAttribute::DontCareMask)) == 0); +enum class KMemoryBlockDisableMergeAttribute : u8 { + None = 0, + Normal = (1u << 0), + DeviceLeft = (1u << 1), + IpcLeft = (1u << 2), + Locked = (1u << 3), + DeviceRight = (1u << 4), + + AllLeft = Normal | DeviceLeft | IpcLeft | Locked, + AllRight = DeviceRight, +}; +DECLARE_ENUM_FLAG_OPERATORS(KMemoryBlockDisableMergeAttribute); struct KMemoryInfo { - VAddr addr{}; - std::size_t size{}; - KMemoryState state{}; - KMemoryPermission perm{}; - KMemoryAttribute attribute{}; - KMemoryPermission original_perm{}; - u16 ipc_lock_count{}; - u16 device_use_count{}; + uintptr_t m_address; + size_t m_size; + KMemoryState m_state; + u16 m_device_disable_merge_left_count; + u16 m_device_disable_merge_right_count; + u16 m_ipc_lock_count; + u16 m_device_use_count; + u16 m_ipc_disable_merge_count; + KMemoryPermission m_permission; + KMemoryAttribute m_attribute; + KMemoryPermission m_original_permission; + KMemoryBlockDisableMergeAttribute m_disable_merge_attribute; constexpr Svc::MemoryInfo GetSvcMemoryInfo() const { return { - addr, - size, - static_cast<Svc::MemoryState>(state & KMemoryState::Mask), - static_cast<Svc::MemoryAttribute>(attribute & KMemoryAttribute::Mask), - static_cast<Svc::MemoryPermission>(perm & KMemoryPermission::UserMask), - ipc_lock_count, - device_use_count, + .addr = m_address, + .size = m_size, + .state = static_cast<Svc::MemoryState>(m_state & KMemoryState::Mask), + .attr = static_cast<Svc::MemoryAttribute>(m_attribute & KMemoryAttribute::UserMask), + .perm = static_cast<Svc::MemoryPermission>(m_permission & KMemoryPermission::UserMask), + .ipc_refcount = m_ipc_lock_count, + .device_refcount = m_device_use_count, + .padding = {}, }; } - constexpr VAddr GetAddress() const { - return addr; + constexpr uintptr_t GetAddress() const { + return m_address; + } + + constexpr size_t GetSize() const { + return m_size; } - constexpr std::size_t GetSize() const { - return size; + + constexpr size_t GetNumPages() const { + return this->GetSize() / PageSize; } - constexpr std::size_t GetNumPages() const { - return GetSize() / PageSize; + + constexpr uintptr_t GetEndAddress() const { + return this->GetAddress() + this->GetSize(); } - constexpr VAddr GetEndAddress() const { - return GetAddress() + GetSize(); + + constexpr uintptr_t GetLastAddress() const { + return this->GetEndAddress() - 1; } - constexpr VAddr GetLastAddress() const { - return GetEndAddress() - 1; + + constexpr u16 GetIpcLockCount() const { + return m_ipc_lock_count; } + + constexpr u16 GetIpcDisableMergeCount() const { + return m_ipc_disable_merge_count; + } + constexpr KMemoryState GetState() const { - return state; + return m_state; + } + + constexpr KMemoryPermission GetPermission() const { + return m_permission; } + + constexpr KMemoryPermission GetOriginalPermission() const { + return m_original_permission; + } + constexpr KMemoryAttribute GetAttribute() const { - return attribute; + return m_attribute; } - constexpr KMemoryPermission GetPermission() const { - return perm; + + constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const { + return m_disable_merge_attribute; } }; -class KMemoryBlock final { - friend class KMemoryBlockManager; - +class KMemoryBlock : public Common::IntrusiveRedBlackTreeBaseNode<KMemoryBlock> { private: - VAddr addr{}; - std::size_t num_pages{}; - KMemoryState state{KMemoryState::None}; - u16 ipc_lock_count{}; - u16 device_use_count{}; - KMemoryPermission perm{KMemoryPermission::None}; - KMemoryPermission original_perm{KMemoryPermission::None}; - KMemoryAttribute attribute{KMemoryAttribute::None}; + u16 m_device_disable_merge_left_count; + u16 m_device_disable_merge_right_count; + VAddr m_address; + size_t m_num_pages; + KMemoryState m_memory_state; + u16 m_ipc_lock_count; + u16 m_device_use_count; + u16 m_ipc_disable_merge_count; + KMemoryPermission m_permission; + KMemoryPermission m_original_permission; + KMemoryAttribute m_attribute; + KMemoryBlockDisableMergeAttribute m_disable_merge_attribute; public: static constexpr int Compare(const KMemoryBlock& lhs, const KMemoryBlock& rhs) { @@ -261,113 +297,349 @@ public: } public: - constexpr KMemoryBlock() = default; - constexpr KMemoryBlock(VAddr addr_, std::size_t num_pages_, KMemoryState state_, - KMemoryPermission perm_, KMemoryAttribute attribute_) - : addr{addr_}, num_pages(num_pages_), state{state_}, perm{perm_}, attribute{attribute_} {} - constexpr VAddr GetAddress() const { - return addr; + return m_address; } - constexpr std::size_t GetNumPages() const { - return num_pages; + constexpr size_t GetNumPages() const { + return m_num_pages; } - constexpr std::size_t GetSize() const { - return GetNumPages() * PageSize; + constexpr size_t GetSize() const { + return this->GetNumPages() * PageSize; } constexpr VAddr GetEndAddress() const { - return GetAddress() + GetSize(); + return this->GetAddress() + this->GetSize(); } constexpr VAddr GetLastAddress() const { - return GetEndAddress() - 1; + return this->GetEndAddress() - 1; + } + + constexpr u16 GetIpcLockCount() const { + return m_ipc_lock_count; + } + + constexpr u16 GetIpcDisableMergeCount() const { + return m_ipc_disable_merge_count; + } + + constexpr KMemoryPermission GetPermission() const { + return m_permission; + } + + constexpr KMemoryPermission GetOriginalPermission() const { + return m_original_permission; + } + + constexpr KMemoryAttribute GetAttribute() const { + return m_attribute; } constexpr KMemoryInfo GetMemoryInfo() const { return { - GetAddress(), GetSize(), state, perm, - attribute, original_perm, ipc_lock_count, device_use_count, + .m_address = this->GetAddress(), + .m_size = this->GetSize(), + .m_state = m_memory_state, + .m_device_disable_merge_left_count = m_device_disable_merge_left_count, + .m_device_disable_merge_right_count = m_device_disable_merge_right_count, + .m_ipc_lock_count = m_ipc_lock_count, + .m_device_use_count = m_device_use_count, + .m_ipc_disable_merge_count = m_ipc_disable_merge_count, + .m_permission = m_permission, + .m_attribute = m_attribute, + .m_original_permission = m_original_permission, + .m_disable_merge_attribute = m_disable_merge_attribute, }; } - void ShareToDevice(KMemoryPermission /*new_perm*/) { - ASSERT((attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared || - device_use_count == 0); - attribute |= KMemoryAttribute::DeviceShared; - const u16 new_use_count{++device_use_count}; - ASSERT(new_use_count > 0); +public: + explicit KMemoryBlock() = default; + + constexpr KMemoryBlock(VAddr addr, size_t np, KMemoryState ms, KMemoryPermission p, + KMemoryAttribute attr) + : Common::IntrusiveRedBlackTreeBaseNode<KMemoryBlock>(), + m_device_disable_merge_left_count(), m_device_disable_merge_right_count(), + m_address(addr), m_num_pages(np), m_memory_state(ms), m_ipc_lock_count(0), + m_device_use_count(0), m_ipc_disable_merge_count(), m_permission(p), + m_original_permission(KMemoryPermission::None), m_attribute(attr), + m_disable_merge_attribute() {} + + constexpr void Initialize(VAddr addr, size_t np, KMemoryState ms, KMemoryPermission p, + KMemoryAttribute attr) { + m_device_disable_merge_left_count = 0; + m_device_disable_merge_right_count = 0; + m_address = addr; + m_num_pages = np; + m_memory_state = ms; + m_ipc_lock_count = 0; + m_device_use_count = 0; + m_permission = p; + m_original_permission = KMemoryPermission::None; + m_attribute = attr; + m_disable_merge_attribute = KMemoryBlockDisableMergeAttribute::None; + } + + constexpr bool HasProperties(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) const { + constexpr auto AttributeIgnoreMask = + KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared; + return m_memory_state == s && m_permission == p && + (m_attribute | AttributeIgnoreMask) == (a | AttributeIgnoreMask); + } + + constexpr bool HasSameProperties(const KMemoryBlock& rhs) const { + return m_memory_state == rhs.m_memory_state && m_permission == rhs.m_permission && + m_original_permission == rhs.m_original_permission && + m_attribute == rhs.m_attribute && m_ipc_lock_count == rhs.m_ipc_lock_count && + m_device_use_count == rhs.m_device_use_count; + } + + constexpr bool CanMergeWith(const KMemoryBlock& rhs) const { + return this->HasSameProperties(rhs) && + (m_disable_merge_attribute & KMemoryBlockDisableMergeAttribute::AllRight) == + KMemoryBlockDisableMergeAttribute::None && + (rhs.m_disable_merge_attribute & KMemoryBlockDisableMergeAttribute::AllLeft) == + KMemoryBlockDisableMergeAttribute::None; } - void UnshareToDevice(KMemoryPermission /*new_perm*/) { - ASSERT((attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared); - const u16 prev_use_count{device_use_count--}; - ASSERT(prev_use_count > 0); - if (prev_use_count == 1) { - attribute &= ~KMemoryAttribute::DeviceShared; + constexpr bool Contains(VAddr addr) const { + return this->GetAddress() <= addr && addr <= this->GetEndAddress(); + } + + constexpr void Add(const KMemoryBlock& added_block) { + ASSERT(added_block.GetNumPages() > 0); + ASSERT(this->GetAddress() + added_block.GetSize() - 1 < + this->GetEndAddress() + added_block.GetSize() - 1); + + m_num_pages += added_block.GetNumPages(); + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute | added_block.m_disable_merge_attribute); + m_device_disable_merge_right_count = added_block.m_device_disable_merge_right_count; + } + + constexpr void Update(KMemoryState s, KMemoryPermission p, KMemoryAttribute a, + bool set_disable_merge_attr, u8 set_mask, u8 clear_mask) { + ASSERT(m_original_permission == KMemoryPermission::None); + ASSERT((m_attribute & KMemoryAttribute::IpcLocked) == KMemoryAttribute::None); + + m_memory_state = s; + m_permission = p; + m_attribute = static_cast<KMemoryAttribute>( + a | (m_attribute & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared))); + + if (set_disable_merge_attr && set_mask != 0) { + m_disable_merge_attribute = m_disable_merge_attribute | + static_cast<KMemoryBlockDisableMergeAttribute>(set_mask); + } + if (clear_mask != 0) { + m_disable_merge_attribute = m_disable_merge_attribute & + static_cast<KMemoryBlockDisableMergeAttribute>(~clear_mask); } } -private: - constexpr bool HasProperties(KMemoryState s, KMemoryPermission p, KMemoryAttribute a) const { - constexpr KMemoryAttribute AttributeIgnoreMask{KMemoryAttribute::DontCareMask | - KMemoryAttribute::IpcLocked | - KMemoryAttribute::DeviceShared}; - return state == s && perm == p && - (attribute | AttributeIgnoreMask) == (a | AttributeIgnoreMask); + constexpr void Split(KMemoryBlock* block, VAddr addr) { + ASSERT(this->GetAddress() < addr); + ASSERT(this->Contains(addr)); + ASSERT(Common::IsAligned(addr, PageSize)); + + block->m_address = m_address; + block->m_num_pages = (addr - this->GetAddress()) / PageSize; + block->m_memory_state = m_memory_state; + block->m_ipc_lock_count = m_ipc_lock_count; + block->m_device_use_count = m_device_use_count; + block->m_permission = m_permission; + block->m_original_permission = m_original_permission; + block->m_attribute = m_attribute; + block->m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute & KMemoryBlockDisableMergeAttribute::AllLeft); + block->m_ipc_disable_merge_count = m_ipc_disable_merge_count; + block->m_device_disable_merge_left_count = m_device_disable_merge_left_count; + block->m_device_disable_merge_right_count = 0; + + m_address = addr; + m_num_pages -= block->m_num_pages; + + m_ipc_disable_merge_count = 0; + m_device_disable_merge_left_count = 0; + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute & KMemoryBlockDisableMergeAttribute::AllRight); } - constexpr bool HasSameProperties(const KMemoryBlock& rhs) const { - return state == rhs.state && perm == rhs.perm && original_perm == rhs.original_perm && - attribute == rhs.attribute && ipc_lock_count == rhs.ipc_lock_count && - device_use_count == rhs.device_use_count; + constexpr void UpdateDeviceDisableMergeStateForShareLeft( + [[maybe_unused]] KMemoryPermission new_perm, bool left, [[maybe_unused]] bool right) { + if (left) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute | KMemoryBlockDisableMergeAttribute::DeviceLeft); + const u16 new_device_disable_merge_left_count = ++m_device_disable_merge_left_count; + ASSERT(new_device_disable_merge_left_count > 0); + } } - constexpr bool Contains(VAddr start) const { - return GetAddress() <= start && start <= GetEndAddress(); + constexpr void UpdateDeviceDisableMergeStateForShareRight( + [[maybe_unused]] KMemoryPermission new_perm, [[maybe_unused]] bool left, bool right) { + if (right) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute | KMemoryBlockDisableMergeAttribute::DeviceRight); + const u16 new_device_disable_merge_right_count = ++m_device_disable_merge_right_count; + ASSERT(new_device_disable_merge_right_count > 0); + } + } + + constexpr void UpdateDeviceDisableMergeStateForShare(KMemoryPermission new_perm, bool left, + bool right) { + this->UpdateDeviceDisableMergeStateForShareLeft(new_perm, left, right); + this->UpdateDeviceDisableMergeStateForShareRight(new_perm, left, right); } - constexpr void Add(std::size_t count) { - ASSERT(count > 0); - ASSERT(GetAddress() + count * PageSize - 1 < GetEndAddress() + count * PageSize - 1); + constexpr void ShareToDevice([[maybe_unused]] KMemoryPermission new_perm, bool left, + bool right) { + // We must either be shared or have a zero lock count. + ASSERT((m_attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared || + m_device_use_count == 0); - num_pages += count; + // Share. + const u16 new_count = ++m_device_use_count; + ASSERT(new_count > 0); + + m_attribute = static_cast<KMemoryAttribute>(m_attribute | KMemoryAttribute::DeviceShared); + + this->UpdateDeviceDisableMergeStateForShare(new_perm, left, right); } - constexpr void Update(KMemoryState new_state, KMemoryPermission new_perm, - KMemoryAttribute new_attribute) { - ASSERT(original_perm == KMemoryPermission::None); - ASSERT((attribute & KMemoryAttribute::IpcLocked) == KMemoryAttribute::None); + constexpr void UpdateDeviceDisableMergeStateForUnshareLeft( + [[maybe_unused]] KMemoryPermission new_perm, bool left, [[maybe_unused]] bool right) { - state = new_state; - perm = new_perm; + if (left) { + if (!m_device_disable_merge_left_count) { + return; + } + --m_device_disable_merge_left_count; + } - attribute = static_cast<KMemoryAttribute>( - new_attribute | - (attribute & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared))); + m_device_disable_merge_left_count = + std::min(m_device_disable_merge_left_count, m_device_use_count); + + if (m_device_disable_merge_left_count == 0) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute::DeviceLeft); + } } - constexpr KMemoryBlock Split(VAddr split_addr) { - ASSERT(GetAddress() < split_addr); - ASSERT(Contains(split_addr)); - ASSERT(Common::IsAligned(split_addr, PageSize)); + constexpr void UpdateDeviceDisableMergeStateForUnshareRight( + [[maybe_unused]] KMemoryPermission new_perm, [[maybe_unused]] bool left, bool right) { + if (right) { + const u16 old_device_disable_merge_right_count = m_device_disable_merge_right_count--; + ASSERT(old_device_disable_merge_right_count > 0); + if (old_device_disable_merge_right_count == 1) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute::DeviceRight); + } + } + } - KMemoryBlock block; - block.addr = addr; - block.num_pages = (split_addr - GetAddress()) / PageSize; - block.state = state; - block.ipc_lock_count = ipc_lock_count; - block.device_use_count = device_use_count; - block.perm = perm; - block.original_perm = original_perm; - block.attribute = attribute; + constexpr void UpdateDeviceDisableMergeStateForUnshare(KMemoryPermission new_perm, bool left, + bool right) { + this->UpdateDeviceDisableMergeStateForUnshareLeft(new_perm, left, right); + this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right); + } - addr = split_addr; - num_pages -= block.num_pages; + constexpr void UnshareToDevice([[maybe_unused]] KMemoryPermission new_perm, bool left, + bool right) { + // We must be shared. + ASSERT((m_attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared); + + // Unhare. + const u16 old_count = m_device_use_count--; + ASSERT(old_count > 0); + + if (old_count == 1) { + m_attribute = + static_cast<KMemoryAttribute>(m_attribute & ~KMemoryAttribute::DeviceShared); + } + + this->UpdateDeviceDisableMergeStateForUnshare(new_perm, left, right); + } + + constexpr void UnshareToDeviceRight([[maybe_unused]] KMemoryPermission new_perm, bool left, + bool right) { + + // We must be shared. + ASSERT((m_attribute & KMemoryAttribute::DeviceShared) == KMemoryAttribute::DeviceShared); + + // Unhare. + const u16 old_count = m_device_use_count--; + ASSERT(old_count > 0); + + if (old_count == 1) { + m_attribute = + static_cast<KMemoryAttribute>(m_attribute & ~KMemoryAttribute::DeviceShared); + } + + this->UpdateDeviceDisableMergeStateForUnshareRight(new_perm, left, right); + } + + constexpr void LockForIpc(KMemoryPermission new_perm, bool left, [[maybe_unused]] bool right) { + // We must either be locked or have a zero lock count. + ASSERT((m_attribute & KMemoryAttribute::IpcLocked) == KMemoryAttribute::IpcLocked || + m_ipc_lock_count == 0); + + // Lock. + const u16 new_lock_count = ++m_ipc_lock_count; + ASSERT(new_lock_count > 0); + + // If this is our first lock, update our permissions. + if (new_lock_count == 1) { + ASSERT(m_original_permission == KMemoryPermission::None); + ASSERT((m_permission | new_perm | KMemoryPermission::NotMapped) == + (m_permission | KMemoryPermission::NotMapped)); + ASSERT((m_permission & KMemoryPermission::UserExecute) != + KMemoryPermission::UserExecute || + (new_perm == KMemoryPermission::UserRead)); + m_original_permission = m_permission; + m_permission = static_cast<KMemoryPermission>( + (new_perm & KMemoryPermission::IpcLockChangeMask) | + (m_original_permission & ~KMemoryPermission::IpcLockChangeMask)); + } + m_attribute = static_cast<KMemoryAttribute>(m_attribute | KMemoryAttribute::IpcLocked); + + if (left) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute | KMemoryBlockDisableMergeAttribute::IpcLeft); + const u16 new_ipc_disable_merge_count = ++m_ipc_disable_merge_count; + ASSERT(new_ipc_disable_merge_count > 0); + } + } + + constexpr void UnlockForIpc([[maybe_unused]] KMemoryPermission new_perm, bool left, + [[maybe_unused]] bool right) { + // We must be locked. + ASSERT((m_attribute & KMemoryAttribute::IpcLocked) == KMemoryAttribute::IpcLocked); + + // Unlock. + const u16 old_lock_count = m_ipc_lock_count--; + ASSERT(old_lock_count > 0); + + // If this is our last unlock, update our permissions. + if (old_lock_count == 1) { + ASSERT(m_original_permission != KMemoryPermission::None); + m_permission = m_original_permission; + m_original_permission = KMemoryPermission::None; + m_attribute = static_cast<KMemoryAttribute>(m_attribute & ~KMemoryAttribute::IpcLocked); + } + + if (left) { + const u16 old_ipc_disable_merge_count = m_ipc_disable_merge_count--; + ASSERT(old_ipc_disable_merge_count > 0); + if (old_ipc_disable_merge_count == 1) { + m_disable_merge_attribute = static_cast<KMemoryBlockDisableMergeAttribute>( + m_disable_merge_attribute & ~KMemoryBlockDisableMergeAttribute::IpcLeft); + } + } + } - return block; + constexpr KMemoryBlockDisableMergeAttribute GetDisableMergeAttribute() const { + return m_disable_merge_attribute; } }; static_assert(std::is_trivially_destructible<KMemoryBlock>::value); diff --git a/src/core/hle/kernel/k_memory_block_manager.cpp b/src/core/hle/kernel/k_memory_block_manager.cpp index 3ddb9984f..cf4c1e371 100644 --- a/src/core/hle/kernel/k_memory_block_manager.cpp +++ b/src/core/hle/kernel/k_memory_block_manager.cpp @@ -2,221 +2,336 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "core/hle/kernel/k_memory_block_manager.h" -#include "core/hle/kernel/memory_types.h" namespace Kernel { -KMemoryBlockManager::KMemoryBlockManager(VAddr start_addr_, VAddr end_addr_) - : start_addr{start_addr_}, end_addr{end_addr_} { - const u64 num_pages{(end_addr - start_addr) / PageSize}; - memory_block_tree.emplace_back(start_addr, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None); -} +KMemoryBlockManager::KMemoryBlockManager() = default; -KMemoryBlockManager::iterator KMemoryBlockManager::FindIterator(VAddr addr) { - auto node{memory_block_tree.begin()}; - while (node != end()) { - const VAddr node_end_addr{node->GetNumPages() * PageSize + node->GetAddress()}; - if (node->GetAddress() <= addr && node_end_addr - 1 >= addr) { - return node; - } - node = std::next(node); - } - return end(); +Result KMemoryBlockManager::Initialize(VAddr st, VAddr nd, KMemoryBlockSlabManager* slab_manager) { + // Allocate a block to encapsulate the address space, insert it into the tree. + KMemoryBlock* start_block = slab_manager->Allocate(); + R_UNLESS(start_block != nullptr, ResultOutOfResource); + + // Set our start and end. + m_start_address = st; + m_end_address = nd; + ASSERT(Common::IsAligned(m_start_address, PageSize)); + ASSERT(Common::IsAligned(m_end_address, PageSize)); + + // Initialize and insert the block. + start_block->Initialize(m_start_address, (m_end_address - m_start_address) / PageSize, + KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None); + m_memory_block_tree.insert(*start_block); + + R_SUCCEED(); } -VAddr KMemoryBlockManager::FindFreeArea(VAddr region_start, std::size_t region_num_pages, - std::size_t num_pages, std::size_t align, - std::size_t offset, std::size_t guard_pages) { - if (num_pages == 0) { - return {}; +void KMemoryBlockManager::Finalize(KMemoryBlockSlabManager* slab_manager, + HostUnmapCallback&& host_unmap_callback) { + // Erase every block until we have none left. + auto it = m_memory_block_tree.begin(); + while (it != m_memory_block_tree.end()) { + KMemoryBlock* block = std::addressof(*it); + it = m_memory_block_tree.erase(it); + slab_manager->Free(block); + host_unmap_callback(block->GetAddress(), block->GetSize()); } - const VAddr region_end{region_start + region_num_pages * PageSize}; - const VAddr region_last{region_end - 1}; - for (auto it{FindIterator(region_start)}; it != memory_block_tree.cend(); it++) { - const auto info{it->GetMemoryInfo()}; - if (region_last < info.GetAddress()) { - break; - } + ASSERT(m_memory_block_tree.empty()); +} - if (info.state != KMemoryState::Free) { - continue; - } +VAddr KMemoryBlockManager::FindFreeArea(VAddr region_start, size_t region_num_pages, + size_t num_pages, size_t alignment, size_t offset, + size_t guard_pages) const { + if (num_pages > 0) { + const VAddr region_end = region_start + region_num_pages * PageSize; + const VAddr region_last = region_end - 1; + for (const_iterator it = this->FindIterator(region_start); it != m_memory_block_tree.cend(); + it++) { + const KMemoryInfo info = it->GetMemoryInfo(); + if (region_last < info.GetAddress()) { + break; + } + if (info.m_state != KMemoryState::Free) { + continue; + } - VAddr area{(info.GetAddress() <= region_start) ? region_start : info.GetAddress()}; - area += guard_pages * PageSize; + VAddr area = (info.GetAddress() <= region_start) ? region_start : info.GetAddress(); + area += guard_pages * PageSize; - const VAddr offset_area{Common::AlignDown(area, align) + offset}; - area = (area <= offset_area) ? offset_area : offset_area + align; + const VAddr offset_area = Common::AlignDown(area, alignment) + offset; + area = (area <= offset_area) ? offset_area : offset_area + alignment; - const VAddr area_end{area + num_pages * PageSize + guard_pages * PageSize}; - const VAddr area_last{area_end - 1}; + const VAddr area_end = area + num_pages * PageSize + guard_pages * PageSize; + const VAddr area_last = area_end - 1; - if (info.GetAddress() <= area && area < area_last && area_last <= region_last && - area_last <= info.GetLastAddress()) { - return area; + if (info.GetAddress() <= area && area < area_last && area_last <= region_last && + area_last <= info.GetLastAddress()) { + return area; + } } } return {}; } -void KMemoryBlockManager::Update(VAddr addr, std::size_t num_pages, KMemoryState prev_state, - KMemoryPermission prev_perm, KMemoryAttribute prev_attribute, - KMemoryState state, KMemoryPermission perm, - KMemoryAttribute attribute) { - const VAddr update_end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; +void KMemoryBlockManager::CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator* allocator, + VAddr address, size_t num_pages) { + // Find the iterator now that we've updated. + iterator it = this->FindIterator(address); + if (address != m_start_address) { + it--; + } - prev_attribute |= KMemoryAttribute::IpcAndDeviceMapped; + // Coalesce blocks that we can. + while (true) { + iterator prev = it++; + if (it == m_memory_block_tree.end()) { + break; + } - while (node != memory_block_tree.end()) { - KMemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + if (prev->CanMergeWith(*it)) { + KMemoryBlock* block = std::addressof(*it); + m_memory_block_tree.erase(it); + prev->Add(*block); + allocator->Free(block); + it = prev; + } - if (addr < cur_end_addr && cur_addr < update_end_addr) { - if (!block->HasProperties(prev_state, prev_perm, prev_attribute)) { - node = next_node; - continue; - } + if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) { + break; + } + } +} - iterator new_node{node}; - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); +void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, + size_t num_pages, KMemoryState state, KMemoryPermission perm, + KMemoryAttribute attr, + KMemoryBlockDisableMergeAttribute set_disable_attr, + KMemoryBlockDisableMergeAttribute clear_disable_attr) { + // Ensure for auditing that we never end up with an invalid tree. + KScopedMemoryBlockManagerAuditor auditor(this); + ASSERT(Common::IsAligned(address, PageSize)); + ASSERT((attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == + KMemoryAttribute::None); + + VAddr cur_address = address; + size_t remaining_pages = num_pages; + iterator it = this->FindIterator(address); + + while (remaining_pages > 0) { + const size_t remaining_size = remaining_pages * PageSize; + KMemoryInfo cur_info = it->GetMemoryInfo(); + if (it->HasProperties(state, perm, attr)) { + // If we already have the right properties, just advance. + if (cur_address + remaining_size < cur_info.GetEndAddress()) { + remaining_pages = 0; + cur_address += remaining_size; + } else { + remaining_pages = + (cur_address + remaining_size - cur_info.GetEndAddress()) / PageSize; + cur_address = cur_info.GetEndAddress(); } + } else { + // If we need to, create a new block before and insert it. + if (cur_info.GetAddress() != cur_address) { + KMemoryBlock* new_block = allocator->Allocate(); + + it->Split(new_block, cur_address); + it = m_memory_block_tree.insert(*new_block); + it++; - if (update_end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(update_end_addr)); + cur_info = it->GetMemoryInfo(); + cur_address = cur_info.GetAddress(); } - new_node->Update(state, perm, attribute); + // If we need to, create a new block after and insert it. + if (cur_info.GetSize() > remaining_size) { + KMemoryBlock* new_block = allocator->Allocate(); - MergeAdjacent(new_node, next_node); - } + it->Split(new_block, cur_address + remaining_size); + it = m_memory_block_tree.insert(*new_block); - if (cur_end_addr - 1 >= update_end_addr - 1) { - break; - } + cur_info = it->GetMemoryInfo(); + } - node = next_node; + // Update block state. + it->Update(state, perm, attr, cur_address == address, static_cast<u8>(set_disable_attr), + static_cast<u8>(clear_disable_attr)); + cur_address += cur_info.GetSize(); + remaining_pages -= cur_info.GetNumPages(); + } + it++; } + + this->CoalesceForUpdate(allocator, address, num_pages); } -void KMemoryBlockManager::Update(VAddr addr, std::size_t num_pages, KMemoryState state, - KMemoryPermission perm, KMemoryAttribute attribute) { - const VAddr update_end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; +void KMemoryBlockManager::UpdateIfMatch(KMemoryBlockManagerUpdateAllocator* allocator, + VAddr address, size_t num_pages, KMemoryState test_state, + KMemoryPermission test_perm, KMemoryAttribute test_attr, + KMemoryState state, KMemoryPermission perm, + KMemoryAttribute attr) { + // Ensure for auditing that we never end up with an invalid tree. + KScopedMemoryBlockManagerAuditor auditor(this); + ASSERT(Common::IsAligned(address, PageSize)); + ASSERT((attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) == + KMemoryAttribute::None); + + VAddr cur_address = address; + size_t remaining_pages = num_pages; + iterator it = this->FindIterator(address); + + while (remaining_pages > 0) { + const size_t remaining_size = remaining_pages * PageSize; + KMemoryInfo cur_info = it->GetMemoryInfo(); + if (it->HasProperties(test_state, test_perm, test_attr) && + !it->HasProperties(state, perm, attr)) { + // If we need to, create a new block before and insert it. + if (cur_info.GetAddress() != cur_address) { + KMemoryBlock* new_block = allocator->Allocate(); + + it->Split(new_block, cur_address); + it = m_memory_block_tree.insert(*new_block); + it++; + + cur_info = it->GetMemoryInfo(); + cur_address = cur_info.GetAddress(); + } - while (node != memory_block_tree.end()) { - KMemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + // If we need to, create a new block after and insert it. + if (cur_info.GetSize() > remaining_size) { + KMemoryBlock* new_block = allocator->Allocate(); - if (addr < cur_end_addr && cur_addr < update_end_addr) { - iterator new_node{node}; + it->Split(new_block, cur_address + remaining_size); + it = m_memory_block_tree.insert(*new_block); - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); + cur_info = it->GetMemoryInfo(); } - if (update_end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(update_end_addr)); + // Update block state. + it->Update(state, perm, attr, false, 0, 0); + cur_address += cur_info.GetSize(); + remaining_pages -= cur_info.GetNumPages(); + } else { + // If we already have the right properties, just advance. + if (cur_address + remaining_size < cur_info.GetEndAddress()) { + remaining_pages = 0; + cur_address += remaining_size; + } else { + remaining_pages = + (cur_address + remaining_size - cur_info.GetEndAddress()) / PageSize; + cur_address = cur_info.GetEndAddress(); } - - new_node->Update(state, perm, attribute); - - MergeAdjacent(new_node, next_node); - } - - if (cur_end_addr - 1 >= update_end_addr - 1) { - break; } - - node = next_node; + it++; } + + this->CoalesceForUpdate(allocator, address, num_pages); } -void KMemoryBlockManager::UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, +void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, + size_t num_pages, MemoryBlockLockFunction lock_func, KMemoryPermission perm) { - const VAddr update_end_addr{addr + num_pages * PageSize}; - iterator node{memory_block_tree.begin()}; + // Ensure for auditing that we never end up with an invalid tree. + KScopedMemoryBlockManagerAuditor auditor(this); + ASSERT(Common::IsAligned(address, PageSize)); - while (node != memory_block_tree.end()) { - KMemoryBlock* block{&(*node)}; - iterator next_node{std::next(node)}; - const VAddr cur_addr{block->GetAddress()}; - const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + VAddr cur_address = address; + size_t remaining_pages = num_pages; + iterator it = this->FindIterator(address); - if (addr < cur_end_addr && cur_addr < update_end_addr) { - iterator new_node{node}; + const VAddr end_address = address + (num_pages * PageSize); - if (addr > cur_addr) { - memory_block_tree.insert(node, block->Split(addr)); - } + while (remaining_pages > 0) { + const size_t remaining_size = remaining_pages * PageSize; + KMemoryInfo cur_info = it->GetMemoryInfo(); - if (update_end_addr < cur_end_addr) { - new_node = memory_block_tree.insert(node, block->Split(update_end_addr)); - } + // If we need to, create a new block before and insert it. + if (cur_info.m_address != cur_address) { + KMemoryBlock* new_block = allocator->Allocate(); - lock_func(new_node, perm); + it->Split(new_block, cur_address); + it = m_memory_block_tree.insert(*new_block); + it++; - MergeAdjacent(new_node, next_node); + cur_info = it->GetMemoryInfo(); + cur_address = cur_info.GetAddress(); } - if (cur_end_addr - 1 >= update_end_addr - 1) { - break; + if (cur_info.GetSize() > remaining_size) { + // If we need to, create a new block after and insert it. + KMemoryBlock* new_block = allocator->Allocate(); + + it->Split(new_block, cur_address + remaining_size); + it = m_memory_block_tree.insert(*new_block); + + cur_info = it->GetMemoryInfo(); } - node = next_node; + // Call the locked update function. + (std::addressof(*it)->*lock_func)(perm, cur_info.GetAddress() == address, + cur_info.GetEndAddress() == end_address); + cur_address += cur_info.GetSize(); + remaining_pages -= cur_info.GetNumPages(); + it++; } -} -void KMemoryBlockManager::IterateForRange(VAddr start, VAddr end, IterateFunc&& func) { - const_iterator it{FindIterator(start)}; - KMemoryInfo info{}; - do { - info = it->GetMemoryInfo(); - func(info); - it = std::next(it); - } while (info.addr + info.size - 1 < end - 1 && it != cend()); + this->CoalesceForUpdate(allocator, address, num_pages); } -void KMemoryBlockManager::MergeAdjacent(iterator it, iterator& next_it) { - KMemoryBlock* block{&(*it)}; - - auto EraseIt = [&](const iterator it_to_erase) { - if (next_it == it_to_erase) { - next_it = std::next(next_it); +// Debug. +bool KMemoryBlockManager::CheckState() const { + // Loop over every block, ensuring that we are sorted and coalesced. + auto it = m_memory_block_tree.cbegin(); + auto prev = it++; + while (it != m_memory_block_tree.cend()) { + const KMemoryInfo prev_info = prev->GetMemoryInfo(); + const KMemoryInfo cur_info = it->GetMemoryInfo(); + + // Sequential blocks which can be merged should be merged. + if (prev->CanMergeWith(*it)) { + return false; } - memory_block_tree.erase(it_to_erase); - }; - if (it != memory_block_tree.begin()) { - KMemoryBlock* prev{&(*std::prev(it))}; - - if (block->HasSameProperties(*prev)) { - const iterator prev_it{std::prev(it)}; + // Sequential blocks should be sequential. + if (prev_info.GetEndAddress() != cur_info.GetAddress()) { + return false; + } - prev->Add(block->GetNumPages()); - EraseIt(it); + // If the block is ipc locked, it must have a count. + if ((cur_info.m_attribute & KMemoryAttribute::IpcLocked) != KMemoryAttribute::None && + cur_info.m_ipc_lock_count == 0) { + return false; + } - it = prev_it; - block = prev; + // If the block is device shared, it must have a count. + if ((cur_info.m_attribute & KMemoryAttribute::DeviceShared) != KMemoryAttribute::None && + cur_info.m_device_use_count == 0) { + return false; } + + // Advance the iterator. + prev = it++; } - if (it != cend()) { - const KMemoryBlock* const next{&(*std::next(it))}; + // Our loop will miss checking the last block, potentially, so check it. + if (prev != m_memory_block_tree.cend()) { + const KMemoryInfo prev_info = prev->GetMemoryInfo(); + // If the block is ipc locked, it must have a count. + if ((prev_info.m_attribute & KMemoryAttribute::IpcLocked) != KMemoryAttribute::None && + prev_info.m_ipc_lock_count == 0) { + return false; + } - if (block->HasSameProperties(*next)) { - block->Add(next->GetNumPages()); - EraseIt(std::next(it)); + // If the block is device shared, it must have a count. + if ((prev_info.m_attribute & KMemoryAttribute::DeviceShared) != KMemoryAttribute::None && + prev_info.m_device_use_count == 0) { + return false; } } + + return true; } } // namespace Kernel diff --git a/src/core/hle/kernel/k_memory_block_manager.h b/src/core/hle/kernel/k_memory_block_manager.h index e14741b89..9b5873883 100644 --- a/src/core/hle/kernel/k_memory_block_manager.h +++ b/src/core/hle/kernel/k_memory_block_manager.h @@ -4,63 +4,154 @@ #pragma once #include <functional> -#include <list> +#include "common/common_funcs.h" #include "common/common_types.h" +#include "core/hle/kernel/k_dynamic_resource_manager.h" #include "core/hle/kernel/k_memory_block.h" namespace Kernel { +class KMemoryBlockManagerUpdateAllocator { +public: + static constexpr size_t MaxBlocks = 2; + +private: + KMemoryBlock* m_blocks[MaxBlocks]; + size_t m_index; + KMemoryBlockSlabManager* m_slab_manager; + +private: + Result Initialize(size_t num_blocks) { + // Check num blocks. + ASSERT(num_blocks <= MaxBlocks); + + // Set index. + m_index = MaxBlocks - num_blocks; + + // Allocate the blocks. + for (size_t i = 0; i < num_blocks && i < MaxBlocks; ++i) { + m_blocks[m_index + i] = m_slab_manager->Allocate(); + R_UNLESS(m_blocks[m_index + i] != nullptr, ResultOutOfResource); + } + + R_SUCCEED(); + } + +public: + KMemoryBlockManagerUpdateAllocator(Result* out_result, KMemoryBlockSlabManager* sm, + size_t num_blocks = MaxBlocks) + : m_blocks(), m_index(MaxBlocks), m_slab_manager(sm) { + *out_result = this->Initialize(num_blocks); + } + + ~KMemoryBlockManagerUpdateAllocator() { + for (const auto& block : m_blocks) { + if (block != nullptr) { + m_slab_manager->Free(block); + } + } + } + + KMemoryBlock* Allocate() { + ASSERT(m_index < MaxBlocks); + ASSERT(m_blocks[m_index] != nullptr); + KMemoryBlock* block = nullptr; + std::swap(block, m_blocks[m_index++]); + return block; + } + + void Free(KMemoryBlock* block) { + ASSERT(m_index <= MaxBlocks); + ASSERT(block != nullptr); + if (m_index == 0) { + m_slab_manager->Free(block); + } else { + m_blocks[--m_index] = block; + } + } +}; + class KMemoryBlockManager final { public: - using MemoryBlockTree = std::list<KMemoryBlock>; + using MemoryBlockTree = + Common::IntrusiveRedBlackTreeBaseTraits<KMemoryBlock>::TreeType<KMemoryBlock>; + using MemoryBlockLockFunction = void (KMemoryBlock::*)(KMemoryPermission new_perm, bool left, + bool right); using iterator = MemoryBlockTree::iterator; using const_iterator = MemoryBlockTree::const_iterator; public: - KMemoryBlockManager(VAddr start_addr_, VAddr end_addr_); + KMemoryBlockManager(); + + using HostUnmapCallback = std::function<void(VAddr, u64)>; + + Result Initialize(VAddr st, VAddr nd, KMemoryBlockSlabManager* slab_manager); + void Finalize(KMemoryBlockSlabManager* slab_manager, HostUnmapCallback&& host_unmap_callback); iterator end() { - return memory_block_tree.end(); + return m_memory_block_tree.end(); } const_iterator end() const { - return memory_block_tree.end(); + return m_memory_block_tree.end(); } const_iterator cend() const { - return memory_block_tree.cend(); + return m_memory_block_tree.cend(); } - iterator FindIterator(VAddr addr); + VAddr FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, + size_t alignment, size_t offset, size_t guard_pages) const; - VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, - std::size_t align, std::size_t offset, std::size_t guard_pages); + void Update(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, size_t num_pages, + KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, + KMemoryBlockDisableMergeAttribute set_disable_attr, + KMemoryBlockDisableMergeAttribute clear_disable_attr); + void UpdateLock(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, size_t num_pages, + MemoryBlockLockFunction lock_func, KMemoryPermission perm); - void Update(VAddr addr, std::size_t num_pages, KMemoryState prev_state, - KMemoryPermission prev_perm, KMemoryAttribute prev_attribute, KMemoryState state, - KMemoryPermission perm, KMemoryAttribute attribute); + void UpdateIfMatch(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, + size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, + KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, + KMemoryAttribute attr); - void Update(VAddr addr, std::size_t num_pages, KMemoryState state, - KMemoryPermission perm = KMemoryPermission::None, - KMemoryAttribute attribute = KMemoryAttribute::None); - - using LockFunc = std::function<void(iterator, KMemoryPermission)>; - void UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, - KMemoryPermission perm); + iterator FindIterator(VAddr address) const { + return m_memory_block_tree.find(KMemoryBlock( + address, 1, KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None)); + } - using IterateFunc = std::function<void(const KMemoryInfo&)>; - void IterateForRange(VAddr start, VAddr end, IterateFunc&& func); + const KMemoryBlock* FindBlock(VAddr address) const { + if (const_iterator it = this->FindIterator(address); it != m_memory_block_tree.end()) { + return std::addressof(*it); + } - KMemoryBlock& FindBlock(VAddr addr) { - return *FindIterator(addr); + return nullptr; } + // Debug. + bool CheckState() const; + private: - void MergeAdjacent(iterator it, iterator& next_it); + void CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address, + size_t num_pages); - [[maybe_unused]] const VAddr start_addr; - [[maybe_unused]] const VAddr end_addr; + MemoryBlockTree m_memory_block_tree; + VAddr m_start_address{}; + VAddr m_end_address{}; +}; - MemoryBlockTree memory_block_tree; +class KScopedMemoryBlockManagerAuditor { +public: + explicit KScopedMemoryBlockManagerAuditor(KMemoryBlockManager* m) : m_manager(m) { + ASSERT(m_manager->CheckState()); + } + explicit KScopedMemoryBlockManagerAuditor(KMemoryBlockManager& m) + : KScopedMemoryBlockManagerAuditor(std::addressof(m)) {} + ~KScopedMemoryBlockManagerAuditor() { + ASSERT(m_manager->CheckState()); + } + +private: + KMemoryBlockManager* m_manager; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_memory_manager.cpp b/src/core/hle/kernel/k_memory_manager.cpp index 5b0a9963a..646711505 100644 --- a/src/core/hle/kernel/k_memory_manager.cpp +++ b/src/core/hle/kernel/k_memory_manager.cpp @@ -331,7 +331,7 @@ Result KMemoryManager::AllocateAndOpenForProcess(KPageGroup* out, size_t num_pag // Set all the allocated memory. for (const auto& block : out->Nodes()) { - std::memset(system.DeviceMemory().GetPointer(block.GetAddress()), fill_pattern, + std::memset(system.DeviceMemory().GetPointer<void>(block.GetAddress()), fill_pattern, block.GetSize()); } diff --git a/src/core/hle/kernel/k_page_buffer.cpp b/src/core/hle/kernel/k_page_buffer.cpp index 1a0bf4439..0c16dded4 100644 --- a/src/core/hle/kernel/k_page_buffer.cpp +++ b/src/core/hle/kernel/k_page_buffer.cpp @@ -12,7 +12,7 @@ namespace Kernel { KPageBuffer* KPageBuffer::FromPhysicalAddress(Core::System& system, PAddr phys_addr) { ASSERT(Common::IsAligned(phys_addr, PageSize)); - return reinterpret_cast<KPageBuffer*>(system.DeviceMemory().GetPointer(phys_addr)); + return system.DeviceMemory().GetPointer<KPageBuffer>(phys_addr); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_page_buffer.h b/src/core/hle/kernel/k_page_buffer.h index 7e50dc1d1..aef06e213 100644 --- a/src/core/hle/kernel/k_page_buffer.h +++ b/src/core/hle/kernel/k_page_buffer.h @@ -13,6 +13,7 @@ namespace Kernel { class KPageBuffer final : public KSlabAllocated<KPageBuffer> { public: + explicit KPageBuffer(KernelCore&) {} KPageBuffer() = default; static KPageBuffer* FromPhysicalAddress(Core::System& system, PAddr phys_addr); diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index d975de844..307e491cb 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -25,7 +25,7 @@ namespace { using namespace Common::Literals; -constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { +constexpr size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { switch (as_type) { case FileSys::ProgramAddressSpaceType::Is32Bit: case FileSys::ProgramAddressSpaceType::Is32BitNoMap: @@ -43,27 +43,29 @@ constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceT } // namespace KPageTable::KPageTable(Core::System& system_) - : general_lock{system_.Kernel()}, map_physical_memory_lock{system_.Kernel()}, system{system_} {} + : m_general_lock{system_.Kernel()}, + m_map_physical_memory_lock{system_.Kernel()}, m_system{system_} {} KPageTable::~KPageTable() = default; Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, - VAddr code_addr, std::size_t code_size, + VAddr code_addr, size_t code_size, + KMemoryBlockSlabManager* mem_block_slab_manager, KMemoryManager::Pool pool) { const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceStart(address_space_width, type); + return KAddressSpaceInfo::GetAddressSpaceStart(m_address_space_width, type); }; const auto GetSpaceSize = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceSize(address_space_width, type); + return KAddressSpaceInfo::GetAddressSpaceSize(m_address_space_width, type); }; // Set our width and heap/alias sizes - address_space_width = GetAddressSpaceWidthFromType(as_type); + m_address_space_width = GetAddressSpaceWidthFromType(as_type); const VAddr start = 0; - const VAddr end{1ULL << address_space_width}; - std::size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; - std::size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; + const VAddr end{1ULL << m_address_space_width}; + size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; + size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; ASSERT(code_addr < code_addr + code_size); ASSERT(code_addr + code_size - 1 <= end - 1); @@ -75,66 +77,65 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type } // Set code regions and determine remaining - constexpr std::size_t RegionAlignment{2_MiB}; + constexpr size_t RegionAlignment{2_MiB}; VAddr process_code_start{}; VAddr process_code_end{}; - std::size_t stack_region_size{}; - std::size_t kernel_map_region_size{}; + size_t stack_region_size{}; + size_t kernel_map_region_size{}; - if (address_space_width == 39) { + if (m_address_space_width == 39) { alias_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Alias); heap_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Heap); stack_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Stack); kernel_map_region_size = GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); - alias_code_region_start = code_region_start; - alias_code_region_end = code_region_end; + m_code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit); + m_code_region_end = m_code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); + m_alias_code_region_start = m_code_region_start; + m_alias_code_region_end = m_code_region_end; process_code_start = Common::AlignDown(code_addr, RegionAlignment); process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment); } else { stack_region_size = 0; kernel_map_region_size = 0; - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - stack_region_start = code_region_start; - alias_code_region_start = code_region_start; - alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) + - GetSpaceSize(KAddressSpaceInfo::Type::MapLarge); - stack_region_end = code_region_end; - kernel_map_region_start = code_region_start; - kernel_map_region_end = code_region_end; - process_code_start = code_region_start; - process_code_end = code_region_end; + m_code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall); + m_code_region_end = m_code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); + m_stack_region_start = m_code_region_start; + m_alias_code_region_start = m_code_region_start; + m_alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) + + GetSpaceSize(KAddressSpaceInfo::Type::MapLarge); + m_stack_region_end = m_code_region_end; + m_kernel_map_region_start = m_code_region_start; + m_kernel_map_region_end = m_code_region_end; + process_code_start = m_code_region_start; + process_code_end = m_code_region_end; } // Set other basic fields - is_aslr_enabled = enable_aslr; - address_space_start = start; - address_space_end = end; - is_kernel = false; + m_enable_aslr = enable_aslr; + m_enable_device_address_space_merge = false; + m_address_space_start = start; + m_address_space_end = end; + m_is_kernel = false; + m_memory_block_slab_manager = mem_block_slab_manager; // Determine the region we can place our undetermineds in VAddr alloc_start{}; - std::size_t alloc_size{}; - if ((process_code_start - code_region_start) >= (end - process_code_end)) { - alloc_start = code_region_start; - alloc_size = process_code_start - code_region_start; + size_t alloc_size{}; + if ((process_code_start - m_code_region_start) >= (end - process_code_end)) { + alloc_start = m_code_region_start; + alloc_size = process_code_start - m_code_region_start; } else { alloc_start = process_code_end; alloc_size = end - process_code_end; } - const std::size_t needed_size{ - (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; - if (alloc_size < needed_size) { - ASSERT(false); - return ResultOutOfMemory; - } + const size_t needed_size = + (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size); + R_UNLESS(alloc_size >= needed_size, ResultOutOfMemory); - const std::size_t remaining_size{alloc_size - needed_size}; + const size_t remaining_size{alloc_size - needed_size}; // Determine random placements for each region - std::size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; + size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; if (enable_aslr) { alias_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * RegionAlignment; @@ -147,117 +148,130 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type } // Setup heap and alias regions - alias_region_start = alloc_start + alias_rnd; - alias_region_end = alias_region_start + alias_region_size; - heap_region_start = alloc_start + heap_rnd; - heap_region_end = heap_region_start + heap_region_size; + m_alias_region_start = alloc_start + alias_rnd; + m_alias_region_end = m_alias_region_start + alias_region_size; + m_heap_region_start = alloc_start + heap_rnd; + m_heap_region_end = m_heap_region_start + heap_region_size; if (alias_rnd <= heap_rnd) { - heap_region_start += alias_region_size; - heap_region_end += alias_region_size; + m_heap_region_start += alias_region_size; + m_heap_region_end += alias_region_size; } else { - alias_region_start += heap_region_size; - alias_region_end += heap_region_size; + m_alias_region_start += heap_region_size; + m_alias_region_end += heap_region_size; } // Setup stack region if (stack_region_size) { - stack_region_start = alloc_start + stack_rnd; - stack_region_end = stack_region_start + stack_region_size; + m_stack_region_start = alloc_start + stack_rnd; + m_stack_region_end = m_stack_region_start + stack_region_size; if (alias_rnd < stack_rnd) { - stack_region_start += alias_region_size; - stack_region_end += alias_region_size; + m_stack_region_start += alias_region_size; + m_stack_region_end += alias_region_size; } else { - alias_region_start += stack_region_size; - alias_region_end += stack_region_size; + m_alias_region_start += stack_region_size; + m_alias_region_end += stack_region_size; } if (heap_rnd < stack_rnd) { - stack_region_start += heap_region_size; - stack_region_end += heap_region_size; + m_stack_region_start += heap_region_size; + m_stack_region_end += heap_region_size; } else { - heap_region_start += stack_region_size; - heap_region_end += stack_region_size; + m_heap_region_start += stack_region_size; + m_heap_region_end += stack_region_size; } } // Setup kernel map region if (kernel_map_region_size) { - kernel_map_region_start = alloc_start + kmap_rnd; - kernel_map_region_end = kernel_map_region_start + kernel_map_region_size; + m_kernel_map_region_start = alloc_start + kmap_rnd; + m_kernel_map_region_end = m_kernel_map_region_start + kernel_map_region_size; if (alias_rnd < kmap_rnd) { - kernel_map_region_start += alias_region_size; - kernel_map_region_end += alias_region_size; + m_kernel_map_region_start += alias_region_size; + m_kernel_map_region_end += alias_region_size; } else { - alias_region_start += kernel_map_region_size; - alias_region_end += kernel_map_region_size; + m_alias_region_start += kernel_map_region_size; + m_alias_region_end += kernel_map_region_size; } if (heap_rnd < kmap_rnd) { - kernel_map_region_start += heap_region_size; - kernel_map_region_end += heap_region_size; + m_kernel_map_region_start += heap_region_size; + m_kernel_map_region_end += heap_region_size; } else { - heap_region_start += kernel_map_region_size; - heap_region_end += kernel_map_region_size; + m_heap_region_start += kernel_map_region_size; + m_heap_region_end += kernel_map_region_size; } if (stack_region_size) { if (stack_rnd < kmap_rnd) { - kernel_map_region_start += stack_region_size; - kernel_map_region_end += stack_region_size; + m_kernel_map_region_start += stack_region_size; + m_kernel_map_region_end += stack_region_size; } else { - stack_region_start += kernel_map_region_size; - stack_region_end += kernel_map_region_size; + m_stack_region_start += kernel_map_region_size; + m_stack_region_end += kernel_map_region_size; } } } // Set heap members - current_heap_end = heap_region_start; - max_heap_size = 0; - max_physical_memory_size = 0; + m_current_heap_end = m_heap_region_start; + m_max_heap_size = 0; + m_max_physical_memory_size = 0; // Ensure that we regions inside our address space auto IsInAddressSpace = [&](VAddr addr) { - return address_space_start <= addr && addr <= address_space_end; + return m_address_space_start <= addr && addr <= m_address_space_end; }; - ASSERT(IsInAddressSpace(alias_region_start)); - ASSERT(IsInAddressSpace(alias_region_end)); - ASSERT(IsInAddressSpace(heap_region_start)); - ASSERT(IsInAddressSpace(heap_region_end)); - ASSERT(IsInAddressSpace(stack_region_start)); - ASSERT(IsInAddressSpace(stack_region_end)); - ASSERT(IsInAddressSpace(kernel_map_region_start)); - ASSERT(IsInAddressSpace(kernel_map_region_end)); + ASSERT(IsInAddressSpace(m_alias_region_start)); + ASSERT(IsInAddressSpace(m_alias_region_end)); + ASSERT(IsInAddressSpace(m_heap_region_start)); + ASSERT(IsInAddressSpace(m_heap_region_end)); + ASSERT(IsInAddressSpace(m_stack_region_start)); + ASSERT(IsInAddressSpace(m_stack_region_end)); + ASSERT(IsInAddressSpace(m_kernel_map_region_start)); + ASSERT(IsInAddressSpace(m_kernel_map_region_end)); // Ensure that we selected regions that don't overlap - const VAddr alias_start{alias_region_start}; - const VAddr alias_last{alias_region_end - 1}; - const VAddr heap_start{heap_region_start}; - const VAddr heap_last{heap_region_end - 1}; - const VAddr stack_start{stack_region_start}; - const VAddr stack_last{stack_region_end - 1}; - const VAddr kmap_start{kernel_map_region_start}; - const VAddr kmap_last{kernel_map_region_end - 1}; + const VAddr alias_start{m_alias_region_start}; + const VAddr alias_last{m_alias_region_end - 1}; + const VAddr heap_start{m_heap_region_start}; + const VAddr heap_last{m_heap_region_end - 1}; + const VAddr stack_start{m_stack_region_start}; + const VAddr stack_last{m_stack_region_end - 1}; + const VAddr kmap_start{m_kernel_map_region_start}; + const VAddr kmap_last{m_kernel_map_region_end - 1}; ASSERT(alias_last < heap_start || heap_last < alias_start); ASSERT(alias_last < stack_start || stack_last < alias_start); ASSERT(alias_last < kmap_start || kmap_last < alias_start); ASSERT(heap_last < stack_start || stack_last < heap_start); ASSERT(heap_last < kmap_start || kmap_last < heap_start); - current_heap_end = heap_region_start; - max_heap_size = 0; - mapped_physical_memory_size = 0; - memory_pool = pool; + m_current_heap_end = m_heap_region_start; + m_max_heap_size = 0; + m_mapped_physical_memory_size = 0; + m_memory_pool = pool; + + m_page_table_impl = std::make_unique<Common::PageTable>(); + m_page_table_impl->Resize(m_address_space_width, PageBits); + + // Initialize our memory block manager. + R_RETURN(m_memory_block_manager.Initialize(m_address_space_start, m_address_space_end, + m_memory_block_slab_manager)); +} - page_table_impl.Resize(address_space_width, PageBits); +void KPageTable::Finalize() { + // Finalize memory blocks. + m_memory_block_manager.Finalize(m_memory_block_slab_manager, [&](VAddr addr, u64 size) { + m_system.Memory().UnmapRegion(*m_page_table_impl, addr, size); + }); - return InitializeMemoryLayout(start, end); + // Close the backing page table, as the destructor is not called for guest objects. + m_page_table_impl.reset(); } -Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, +Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) { const u64 size{num_pages * PageSize}; @@ -265,52 +279,76 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat R_UNLESS(this->CanContain(addr, size, state), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the destination memory is unmapped. R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); + + // Allocate and open. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, num_pages, - KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, allocation_option))); + KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, m_allocation_option))); R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); - block_manager->Update(addr, num_pages, state, perm); + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t size) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the source memory is normal heap. KMemoryState src_state{}; KMemoryPermission src_perm{}; - std::size_t num_src_allocator_blocks{}; + size_t num_src_allocator_blocks{}; R_TRY(this->CheckMemoryState(&src_state, &src_perm, nullptr, &num_src_allocator_blocks, src_address, size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)); // Verify that the destination memory is unmapped. - std::size_t num_dst_allocator_blocks{}; + size_t num_dst_allocator_blocks{}; R_TRY(this->CheckMemoryState(&num_dst_allocator_blocks, dst_address, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); + R_TRY(dst_allocator_result); + // Map the code memory. { // Determine the number of pages being operated on. - const std::size_t num_pages = size / PageSize; + const size_t num_pages = size / PageSize; // Create page groups for the memory being mapped. KPageGroup pg; @@ -335,33 +373,37 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size unprot_guard.Cancel(); // Apply the memory block updates. - block_manager->Update(src_address, num_pages, src_state, new_perm, - KMemoryAttribute::Locked); - block_manager->Update(dst_address, num_pages, KMemoryState::AliasCode, new_perm, - KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + src_state, new_perm, KMemoryAttribute::Locked, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::AliasCode, new_perm, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); } - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, +Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t size, ICacheInvalidationStrategy icache_invalidation_strategy) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the source memory is locked normal heap. - std::size_t num_src_allocator_blocks{}; + size_t num_src_allocator_blocks{}; R_TRY(this->CheckMemoryState(std::addressof(num_src_allocator_blocks), src_address, size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::Locked)); // Verify that the destination memory is aliasable code. - std::size_t num_dst_allocator_blocks{}; + size_t num_dst_allocator_blocks{}; R_TRY(this->CheckMemoryStateContiguous( std::addressof(num_dst_allocator_blocks), dst_address, size, KMemoryState::FlagCanCodeAlias, KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, @@ -370,7 +412,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Determine whether any pages being unmapped are code. bool any_code_pages = false; { - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(dst_address); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(dst_address); while (true) { // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -396,9 +438,9 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si SCOPE_EXIT({ if (reprotected_pages && any_code_pages) { if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { - system.InvalidateCpuInstructionCacheRange(dst_address, size); + m_system.InvalidateCpuInstructionCacheRange(dst_address, size); } else { - system.InvalidateCpuInstructionCaches(); + m_system.InvalidateCpuInstructionCaches(); } } }); @@ -406,7 +448,21 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Unmap. { // Determine the number of pages being operated on. - const std::size_t num_pages = size / PageSize; + const size_t num_pages = size / PageSize; + + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); + R_TRY(dst_allocator_result); // Unmap the aliased copy of the pages. R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); @@ -416,73 +472,34 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si OperationType::ChangePermissions)); // Apply the memory block updates. - block_manager->Update(dst_address, num_pages, KMemoryState::None); - block_manager->Update(src_address, num_pages, KMemoryState::Normal, - KMemoryPermission::UserReadWrite); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update( + std::addressof(src_allocator), src_address, num_pages, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Locked); // Note that we reprotected pages. reprotected_pages = true; } - return ResultSuccess; + R_SUCCEED(); } -VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, - std::size_t num_pages, std::size_t alignment, std::size_t offset, - std::size_t guard_pages) { +VAddr KPageTable::FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, + size_t alignment, size_t offset, size_t guard_pages) { VAddr address = 0; if (num_pages <= region_num_pages) { if (this->IsAslrEnabled()) { - // Try to directly find a free area up to 8 times. - for (std::size_t i = 0; i < 8; i++) { - const std::size_t random_offset = - KSystemControl::GenerateRandomRange( - 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * - alignment; - const VAddr candidate = - Common::AlignDown((region_start + random_offset), alignment) + offset; - - KMemoryInfo info = this->QueryInfoImpl(candidate); - - if (info.state != KMemoryState::Free) { - continue; - } - if (region_start > candidate) { - continue; - } - if (info.GetAddress() + guard_pages * PageSize > candidate) { - continue; - } - - const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; - if (candidate_end > info.GetLastAddress()) { - continue; - } - if (candidate_end > region_start + region_num_pages * PageSize - 1) { - continue; - } - - address = candidate; - break; - } - // Fall back to finding the first free area with a random offset. - if (address == 0) { - // NOTE: Nintendo does not account for guard pages here. - // This may theoretically cause an offset to be chosen that cannot be mapped. We - // will account for guard pages. - const std::size_t offset_pages = KSystemControl::GenerateRandomRange( - 0, region_num_pages - num_pages - guard_pages); - address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, - region_num_pages - offset_pages, num_pages, - alignment, offset, guard_pages); - } + UNIMPLEMENTED(); } - // Find the first free area. if (address == 0) { - address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, - alignment, offset, guard_pages); + address = m_memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); } } @@ -500,7 +517,8 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - R_UNLESS(page_table_impl.BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); + R_UNLESS(m_page_table_impl->BeginTraversal(next_entry, context, addr), + ResultInvalidCurrentMemory); // Prepare tracking variables. PAddr cur_addr = next_entry.phys_addr; @@ -508,9 +526,9 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { size_t tot_size = cur_size; // Iterate, adding to group as we go. - const auto& memory_layout = system.Kernel().MemoryLayout(); + const auto& memory_layout = m_system.Kernel().MemoryLayout(); while (tot_size < size) { - R_UNLESS(page_table_impl.ContinueTraversal(next_entry, context), + R_UNLESS(m_page_table_impl->ContinueTraversal(next_entry, context), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -538,7 +556,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); R_TRY(pg.AddBlock(cur_addr, cur_pages)); - return ResultSuccess; + R_SUCCEED(); } bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t num_pages) { @@ -546,7 +564,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu const size_t size = num_pages * PageSize; const auto& pg = pg_ll.Nodes(); - const auto& memory_layout = system.Kernel().MemoryLayout(); + const auto& memory_layout = m_system.Kernel().MemoryLayout(); // Empty groups are necessarily invalid. if (pg.empty()) { @@ -573,7 +591,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - if (!page_table_impl.BeginTraversal(next_entry, context, addr)) { + if (!m_page_table_impl->BeginTraversal(next_entry, context, addr)) { return false; } @@ -584,7 +602,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Iterate, comparing expected to actual. while (tot_size < size) { - if (!page_table_impl.ContinueTraversal(next_entry, context)) { + if (!m_page_table_impl->ContinueTraversal(next_entry, context)) { return false; } @@ -630,11 +648,11 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu return cur_block_address == cur_addr && cur_block_pages == (cur_size / PageSize); } -Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, +Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& src_page_table, VAddr src_addr) { - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); - const std::size_t num_pages{size / PageSize}; + const size_t num_pages{size / PageSize}; // Check that the memory is mapped in the destination process. size_t num_allocator_blocks; @@ -649,43 +667,51 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTab KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Apply the memory block update. - block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), dst_addr, num_pages, + KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); - system.InvalidateCpuInstructionCaches(); + m_system.InvalidateCpuInstructionCaches(); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { +Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Lock the physical memory lock. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Calculate the last address for convenience. const VAddr last_address = address + size - 1; // Define iteration variables. VAddr cur_address; - std::size_t mapped_size; + size_t mapped_size; // The entire mapping process can be retried. while (true) { // Check if the memory is already mapped. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Iterate over the memory. cur_address = address; mapped_size = 0; - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -716,20 +742,20 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { { // Reserve the memory from the process resource limit. KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), + m_system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, size - mapped_size); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the new memory. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( &pg, (size - mapped_size) / PageSize, - KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); // Map the memory. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); size_t num_allocator_blocks = 0; @@ -739,10 +765,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { size_t checked_mapped_size = 0; cur_address = address; - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -782,6 +808,14 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { } } + // Create an update allocator. + ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, + num_allocator_blocks); + R_TRY(allocator_result); + // Reset the current tracking address, and make sure we clean up on failure. cur_address = address; auto unmap_guard = detail::ScopeExit([&] { @@ -791,10 +825,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { // Iterate, unmapping the pages. cur_address = address; - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -830,10 +864,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -886,37 +920,37 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { memory_reservation.Commit(); // Increase our tracked mapped size. - mapped_physical_memory_size += (size - mapped_size); + m_mapped_physical_memory_size += (size - mapped_size); // Update the relevant memory blocks. - block_manager->Update(address, size / PageSize, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None, - KMemoryState::Normal, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None); + m_memory_block_manager.UpdateIfMatch( + std::addressof(allocator), address, size / PageSize, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None); // Cancel our guard. unmap_guard.Cancel(); - return ResultSuccess; + R_SUCCEED(); } } } } -Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { +Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { // Lock the physical memory lock. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Calculate the last address for convenience. const VAddr last_address = address + size - 1; // Define iteration variables. VAddr cur_address = 0; - std::size_t mapped_size = 0; - std::size_t num_allocator_blocks = 0; + size_t mapped_size = 0; + size_t num_allocator_blocks = 0; // Check if the memory is mapped. { @@ -924,10 +958,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; mapped_size = 0; - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1022,6 +1056,13 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { } ASSERT(pg.GetNumPages() == mapped_size / PageSize); + // Create an update allocator. + ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Reset the current tracking address, and make sure we clean up on failure. cur_address = address; auto remap_guard = detail::ScopeExit([&] { @@ -1030,7 +1071,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; // Iterate over the memory we unmapped. - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); auto pg_it = pg.Nodes().begin(); PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); @@ -1085,10 +1126,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { }); // Iterate over the memory, unmapping as we go. - auto it = block_manager->FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1115,104 +1156,159 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { } // Release the memory resource. - mapped_physical_memory_size -= mapped_size; - auto process{system.Kernel().CurrentProcess()}; + m_mapped_physical_memory_size -= mapped_size; + auto process{m_system.Kernel().CurrentProcess()}; process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); // Update memory blocks. - block_manager->Update(address, size / PageSize, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, + KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // TODO(bunnei): This is a workaround until the next set of changes, where we add reference // counting for mapped pages. Until then, we must manually close the reference to the page // group. - system.Kernel().MemoryManager().Close(pg); + m_system.Kernel().MemoryManager().Close(pg); // We succeeded. remap_guard.Cancel(); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::UserReadWrite, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); +Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) { + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Validate that the source address's state is valid. + KMemoryState src_state; + size_t num_src_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(src_state), nullptr, nullptr, + std::addressof(num_src_allocator_blocks), src_address, size, + KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, + KMemoryPermission::All, KMemoryPermission::UserReadWrite, + KMemoryAttribute::All, KMemoryAttribute::None)); - if (IsRegionMapped(dst_addr, size)) { - return ResultInvalidCurrentMemory; - } + // Validate that the dst address's state is valid. + size_t num_dst_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_dst_allocator_blocks), dst_address, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); + R_TRY(dst_allocator_result); + + // Map the memory. KPageGroup page_linked_list; - const std::size_t num_pages{size / PageSize}; - - AddRegionToPages(src_addr, num_pages, page_linked_list); + const size_t num_pages{size / PageSize}; + const KMemoryPermission new_src_perm = static_cast<KMemoryPermission>( + KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); + const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; + AddRegionToPages(src_address, num_pages, page_linked_list); { + // Reprotect the source as kernel-read/not mapped. auto block_guard = detail::ScopeExit([&] { - Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, + Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, OperationType::ChangePermissions); }); - - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, - OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::UserReadWrite)); + R_TRY(Operate(src_address, num_pages, new_src_perm, OperationType::ChangePermissions)); + R_TRY(MapPages(dst_address, page_linked_list, KMemoryPermission::UserReadWrite)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None, - KMemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, KMemoryState::Stack, - KMemoryPermission::UserReadWrite); - - return ResultSuccess; + // Apply the memory block updates. + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + new_src_perm, new_src_attr, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::Stack, KMemoryPermission::UserReadWrite, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); + + R_SUCCEED(); } -Result KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { - KScopedLightLock lk(general_lock); +Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size) { + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Validate that the source address's state is valid. + KMemoryState src_state; + size_t num_src_allocator_blocks; + R_TRY(this->CheckMemoryState( + std::addressof(src_state), nullptr, nullptr, std::addressof(num_src_allocator_blocks), + src_address, size, KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, + KMemoryPermission::All, KMemoryPermission::NotMapped | KMemoryPermission::KernelRead, + KMemoryAttribute::All, KMemoryAttribute::Locked)); + + // Validate that the dst address's state is valid. + KMemoryPermission dst_perm; + size_t num_dst_allocator_blocks; + R_TRY(this->CheckMemoryState( + nullptr, std::addressof(dst_perm), nullptr, std::addressof(num_dst_allocator_blocks), + dst_address, size, KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); + R_TRY(src_allocator_result); - KMemoryPermission dst_perm{}; - CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, nullptr, dst_addr, size, - KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); + R_TRY(dst_allocator_result); KPageGroup src_pages; KPageGroup dst_pages; - const std::size_t num_pages{size / PageSize}; + const size_t num_pages{size / PageSize}; - AddRegionToPages(src_addr, num_pages, src_pages); - AddRegionToPages(dst_addr, num_pages, dst_pages); + AddRegionToPages(src_address, num_pages, src_pages); + AddRegionToPages(dst_address, num_pages, dst_pages); - if (!dst_pages.IsEqual(src_pages)) { - return ResultInvalidMemoryRegion; - } + R_UNLESS(dst_pages.IsEqual(src_pages), ResultInvalidMemoryRegion); { - auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); + auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); - CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions)); + R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, + OperationType::ChangePermissions)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::UserReadWrite); - block_manager->Update(dst_addr, num_pages, KMemoryState::Free); - - return ResultSuccess; + // Apply the memory block updates. + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); + + R_SUCCEED(); } Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, @@ -1225,48 +1321,54 @@ Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, if (const auto result{ Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; result.IsError()) { - const std::size_t num_pages{(addr - cur_addr) / PageSize}; + const size_t num_pages{(addr - cur_addr) / PageSize}; ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) .IsSuccess()); - return result; + R_RETURN(result); } cur_addr += node.GetNumPages() * PageSize; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, KMemoryPermission perm) { // Check that the map is in range. - const std::size_t num_pages{page_linked_list.GetNumPages()}; - const std::size_t size{num_pages * PageSize}; + const size_t num_pages{page_linked_list.GetNumPages()}; + const size_t size{num_pages * PageSize}; R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. R_TRY(this->CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); + // Map the pages. R_TRY(MapPages(address, page_linked_list, perm)); // Update the blocks. - block_manager->Update(address, num_pages, state, perm); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, - PAddr phys_addr, bool is_pa_valid, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, - KMemoryPermission perm) { +Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, + bool is_pa_valid, VAddr region_start, size_t region_num_pages, + KMemoryState state, KMemoryPermission perm) { ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); // Ensure this is a valid map request. @@ -1275,7 +1377,7 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Find a random address to map at. VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, @@ -1288,6 +1390,11 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t KMemoryAttribute::None, KMemoryAttribute::None) .IsSuccess()); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); + // Perform mapping operation. if (is_pa_valid) { R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); @@ -1296,11 +1403,13 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t } // Update the blocks. - block_manager->Update(addr, num_pages, state, perm); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); // We successfully mapped the pages. *out_addr = addr; - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { @@ -1312,60 +1421,80 @@ Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { if (const auto result{Operate(cur_addr, node.GetNumPages(), KMemoryPermission::None, OperationType::Unmap)}; result.IsError()) { - return result; + R_RETURN(result); } cur_addr += node.GetNumPages() * PageSize; } - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state) { +Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { // Check that the unmap is in range. - const std::size_t num_pages{page_linked_list.GetNumPages()}; - const std::size_t size{num_pages * PageSize}; - R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + const size_t num_pages{page_linked_list.GetNumPages()}; + const size_t size{num_pages * PageSize}; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. - R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, state, KMemoryPermission::None, + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform the unmap. - R_TRY(UnmapPages(addr, page_linked_list)); + R_TRY(UnmapPages(address, page_linked_list)); // Update the blocks. - block_manager->Update(addr, num_pages, state, KMemoryPermission::None); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state) { +Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState state) { // Check that the unmap is in range. - const std::size_t size = num_pages * PageSize; + const size_t size = num_pages * PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. - std::size_t num_allocator_blocks{}; + size_t num_allocator_blocks{}; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform the unmap. R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Update the blocks. - block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages, @@ -1380,7 +1509,7 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check if state allows us to create the group. R_TRY(this->CheckMemoryState(address, size, state_mask | KMemoryState::FlagReferenceCounted, @@ -1390,15 +1519,15 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n // Create a new page group for the region. R_TRY(this->MakePageGroup(*out, address, num_pages)); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, +Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory permission. KMemoryState old_state; @@ -1435,105 +1564,101 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, // Succeed if there's nothing to do. R_SUCCEED_IF(old_perm == new_perm && old_state == new_state); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform mapping operation. const auto operation = was_x ? OperationType::ChangePermissionsAndRefresh : OperationType::ChangePermissions; R_TRY(Operate(addr, num_pages, new_perm, operation)); // Update the blocks. - block_manager->Update(addr, num_pages, new_state, new_perm, KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, new_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // Ensure cache coherency, if we're setting pages as executable. if (is_x) { - system.InvalidateCpuInstructionCacheRange(addr, size); + m_system.InvalidateCpuInstructionCacheRange(addr, size); } - return ResultSuccess; + R_SUCCEED(); } KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); - return block_manager->FindBlock(addr).GetMemoryInfo(); + return m_memory_block_manager.FindBlock(addr)->GetMemoryInfo(); } KMemoryInfo KPageTable::QueryInfo(VAddr addr) { if (!Contains(addr, 1)) { - return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible, - KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None}; + return { + .m_address = m_address_space_end, + .m_size = 0 - m_address_space_end, + .m_state = static_cast<KMemoryState>(Svc::MemoryState::Inaccessible), + .m_device_disable_merge_left_count = 0, + .m_device_disable_merge_right_count = 0, + .m_ipc_lock_count = 0, + .m_device_use_count = 0, + .m_ipc_disable_merge_count = 0, + .m_permission = KMemoryPermission::None, + .m_attribute = KMemoryAttribute::None, + .m_original_permission = KMemoryPermission::None, + .m_disable_merge_attribute = KMemoryBlockDisableMergeAttribute::None, + }; } return QueryInfoImpl(addr); } -Result KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { - KScopedLightLock lk(general_lock); - - KMemoryState state{}; - KMemoryAttribute attribute{}; - - R_TRY(CheckMemoryState(&state, nullptr, &attribute, nullptr, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryPermission::All, KMemoryPermission::UserReadWrite, - KMemoryAttribute::Mask, KMemoryAttribute::None, - KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked); - - return ResultSuccess; -} - -Result KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryState state{}; - - R_TRY(CheckMemoryState(&state, nullptr, nullptr, nullptr, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, KMemoryPermission::UserReadWrite); - return ResultSuccess; -} - -Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, - Svc::MemoryPermission svc_perm) { +Result KPageTable::SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory permission. KMemoryState old_state; KMemoryPermission old_perm; - R_TRY(this->CheckMemoryState( - std::addressof(old_state), std::addressof(old_perm), nullptr, nullptr, addr, size, - KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), nullptr, + std::addressof(num_allocator_blocks), addr, size, + KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::All, KMemoryAttribute::None)); // Determine new perm. const KMemoryPermission new_perm = ConvertToKMemoryPermission(svc_perm); R_SUCCEED_IF(old_perm == new_perm); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform mapping operation. R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); // Update the blocks. - block_manager->Update(addr, num_pages, old_state, new_perm, KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr) { +Result KPageTable::SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr) { const size_t num_pages = size / PageSize; ASSERT((static_cast<KMemoryAttribute>(mask) | KMemoryAttribute::SetMask) == KMemoryAttribute::SetMask); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory attribute. KMemoryState old_state; @@ -1548,6 +1673,12 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, AttributeTestMask, KMemoryAttribute::None, ~AttributeTestMask)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Determine the new attribute. const KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(((old_attr & static_cast<KMemoryAttribute>(~mask)) | @@ -1557,123 +1688,142 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 this->Operate(addr, num_pages, old_perm, OperationType::ChangePermissionsAndRefresh); // Update the blocks. - block_manager->Update(addr, num_pages, old_state, old_perm, new_attr); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, old_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::SetMaxHeapSize(std::size_t size) { +Result KPageTable::SetMaxHeapSize(size_t size) { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Only process page tables are allowed to set heap size. ASSERT(!this->IsKernel()); - max_heap_size = size; + m_max_heap_size = size; - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { +Result KPageTable::SetHeapSize(VAddr* out, size_t size) { // Lock the physical memory mutex. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Try to perform a reduction in heap, instead of an extension. VAddr cur_address{}; - std::size_t allocation_size{}; + size_t allocation_size{}; { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Validate that setting heap size is possible at all. - R_UNLESS(!is_kernel, ResultOutOfMemory); - R_UNLESS(size <= static_cast<std::size_t>(heap_region_end - heap_region_start), + R_UNLESS(!m_is_kernel, ResultOutOfMemory); + R_UNLESS(size <= static_cast<size_t>(m_heap_region_end - m_heap_region_start), ResultOutOfMemory); - R_UNLESS(size <= max_heap_size, ResultOutOfMemory); + R_UNLESS(size <= m_max_heap_size, ResultOutOfMemory); if (size < GetHeapSize()) { // The size being requested is less than the current size, so we need to free the end of // the heap. // Validate memory state. - std::size_t num_allocator_blocks; + size_t num_allocator_blocks; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), - heap_region_start + size, GetHeapSize() - size, + m_heap_region_start + size, GetHeapSize() - size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, + num_allocator_blocks); + R_TRY(allocator_result); + // Unmap the end of the heap. const auto num_pages = (GetHeapSize() - size) / PageSize; - R_TRY(Operate(heap_region_start + size, num_pages, KMemoryPermission::None, + R_TRY(Operate(m_heap_region_start + size, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Release the memory from the resource limit. - system.Kernel().CurrentProcess()->GetResourceLimit()->Release( + m_system.Kernel().CurrentProcess()->GetResourceLimit()->Release( LimitableResource::PhysicalMemory, num_pages * PageSize); // Apply the memory block update. - block_manager->Update(heap_region_start + size, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), m_heap_region_start + size, + num_pages, KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + size == 0 ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. - current_heap_end = heap_region_start + size; + m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = heap_region_start; - return ResultSuccess; + *out = m_heap_region_start; + R_SUCCEED(); } else if (size == GetHeapSize()) { // The size requested is exactly the current size. - *out = heap_region_start; - return ResultSuccess; + *out = m_heap_region_start; + R_SUCCEED(); } else { // We have to allocate memory. Determine how much to allocate and where while the table // is locked. - cur_address = current_heap_end; + cur_address = m_current_heap_end; allocation_size = size - GetHeapSize(); } } // Reserve memory for the heap extension. KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, + m_system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, allocation_size); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the heap extension. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, allocation_size / PageSize, - KMemoryManager::EncodeOption(memory_pool, allocation_option))); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option))); // Clear all the newly allocated pages. for (const auto& it : pg.Nodes()) { - std::memset(system.DeviceMemory().GetPointer(it.GetAddress()), heap_fill_value, + std::memset(m_system.DeviceMemory().GetPointer<void>(it.GetAddress()), m_heap_fill_value, it.GetSize()); } // Map the pages. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Ensure that the heap hasn't changed since we began executing. - ASSERT(cur_address == current_heap_end); + ASSERT(cur_address == m_current_heap_end); // Check the memory state. - std::size_t num_allocator_blocks{}; - R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), current_heap_end, + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), m_current_heap_end, allocation_size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator( + std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Map the pages. const auto num_pages = allocation_size / PageSize; - R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); + R_TRY(Operate(m_current_heap_end, num_pages, pg, OperationType::MapGroup)); // Clear all the newly allocated pages. - for (std::size_t cur_page = 0; cur_page < num_pages; ++cur_page) { - std::memset(system.Memory().GetPointer(current_heap_end + (cur_page * PageSize)), 0, + for (size_t cur_page = 0; cur_page < num_pages; ++cur_page) { + std::memset(m_system.Memory().GetPointer(m_current_heap_end + (cur_page * PageSize)), 0, PageSize); } @@ -1681,133 +1831,172 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { memory_reservation.Commit(); // Apply the memory block update. - block_manager->Update(current_heap_end, num_pages, KMemoryState::Normal, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None); + m_memory_block_manager.Update( + std::addressof(allocator), m_current_heap_end, num_pages, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + m_heap_region_start == m_current_heap_end ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. - current_heap_end = heap_region_start + size; + m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = heap_region_start; - return ResultSuccess; + *out = m_heap_region_start; + R_SUCCEED(); } } -ResultVal<VAddr> KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, +ResultVal<VAddr> KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_t align, bool is_map_only, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, + size_t region_num_pages, KMemoryState state, KMemoryPermission perm, PAddr map_addr) { - KScopedLightLock lk(general_lock); - - if (!CanContain(region_start, region_num_pages * PageSize, state)) { - return ResultInvalidCurrentMemory; - } - - if (region_num_pages <= needed_num_pages) { - return ResultOutOfMemory; - } + KScopedLightLock lk(m_general_lock); + R_UNLESS(CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(region_num_pages > needed_num_pages, ResultOutOfMemory); const VAddr addr{ AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; - if (!addr) { - return ResultOutOfMemory; - } + R_UNLESS(addr, ResultOutOfMemory); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager); if (is_map_only) { R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); } else { KPageGroup page_group; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( &page_group, needed_num_pages, - KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); } - block_manager->Update(addr, needed_num_pages, state, perm); + // Update the blocks. + m_memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return addr; } -Result KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission perm{}; - if (const Result result{CheckMemoryState( - nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } +Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMemoryPermission perm, + bool is_aligned) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->ShareToDevice(permission); - }, - perm); + // Lock the table. + KScopedLightLock lk(m_general_lock); - return ResultSuccess; + // Check the memory state. + const auto test_state = + (is_aligned ? KMemoryState::FlagCanAlignedDeviceMap : KMemoryState::FlagCanDeviceMap); + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, test_state, + test_state, perm, perm, + KMemoryAttribute::IpcLocked | KMemoryAttribute::Locked, + KMemoryAttribute::None, KMemoryAttribute::DeviceShared)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, + &KMemoryBlock::ShareToDevice, KMemoryPermission::None); + + R_SUCCEED(); } -Result KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission perm{}; - if (const Result result{CheckMemoryState( - nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } +Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->UnshareToDevice(permission); - }, - perm); + // Lock the table. + KScopedLightLock lk(m_general_lock); - return ResultSuccess; + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryStateContiguous( + std::addressof(num_allocator_blocks), address, size, + KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap, + KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + const KMemoryBlockManager::MemoryBlockLockFunction lock_func = + m_enable_device_address_space_merge + ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare + : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight; + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, + KMemoryPermission::None); + + R_SUCCEED(); } -Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size) { - return this->LockMemoryAndOpen( +Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryStateContiguous( + std::addressof(num_allocator_blocks), address, size, KMemoryState::FlagCanDeviceMap, + KMemoryState::FlagCanDeviceMap, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, + &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); + + R_SUCCEED(); +} + +Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) { + R_RETURN(this->LockMemoryAndOpen( out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None, static_cast<KMemoryPermission>(KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite), - KMemoryAttribute::Locked); + KMemoryAttribute::Locked)); } -Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg) { - return this->UnlockMemory( +Result KPageTable::UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg) { + R_RETURN(this->UnlockMemory( addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, - KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); -} - -Result KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { - block_manager = std::make_unique<KMemoryBlockManager>(start, end); - - return ResultSuccess; -} - -bool KPageTable::IsRegionMapped(VAddr address, u64 size) { - return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, - KMemoryPermission::All, KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped) - .IsError(); + KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg)); } bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { - auto start_ptr = system.Memory().GetPointer(addr); + auto start_ptr = m_system.DeviceMemory().GetPointer<u8>(addr); for (u64 offset{}; offset < size; offset += PageSize) { - if (start_ptr != system.Memory().GetPointer(addr + offset)) { + if (start_ptr != m_system.DeviceMemory().GetPointer<u8>(addr + offset)) { return false; } start_ptr += PageSize; @@ -1815,8 +2004,7 @@ bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { return true; } -void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages, - KPageGroup& page_linked_list) { +void KPageTable::AddRegionToPages(VAddr start, size_t num_pages, KPageGroup& page_linked_list) { VAddr addr{start}; while (addr < start + (num_pages * PageSize)) { const PAddr paddr{GetPhysicalAddr(addr)}; @@ -1826,16 +2014,16 @@ void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages, } } -VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, - u64 needed_num_pages, std::size_t align) { - if (is_aslr_enabled) { +VAddr KPageTable::AllocateVirtualMemory(VAddr start, size_t region_num_pages, u64 needed_num_pages, + size_t align) { + if (m_enable_aslr) { UNIMPLEMENTED(); } - return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, - IsKernel() ? 1 : 4); + return m_memory_block_manager.FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, + IsKernel() ? 1 : 4); } -Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, +Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_group, OperationType operation) { ASSERT(this->IsLockedByCurrentThread()); @@ -1844,11 +2032,11 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& ASSERT(num_pages == page_group.GetNumPages()); for (const auto& node : page_group.Nodes()) { - const std::size_t size{node.GetNumPages() * PageSize}; + const size_t size{node.GetNumPages() * PageSize}; switch (operation) { case OperationType::MapGroup: - system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress()); + m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, size, node.GetAddress()); break; default: ASSERT(false); @@ -1857,10 +2045,10 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& addr += size; } - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, +Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, OperationType operation, PAddr map_addr) { ASSERT(this->IsLockedByCurrentThread()); @@ -1870,12 +2058,12 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission switch (operation) { case OperationType::Unmap: - system.Memory().UnmapRegion(page_table_impl, addr, num_pages * PageSize); + m_system.Memory().UnmapRegion(*m_page_table_impl, addr, num_pages * PageSize); break; case OperationType::Map: { ASSERT(map_addr); ASSERT(Common::IsAligned(map_addr, PageSize)); - system.Memory().MapMemoryRegion(page_table_impl, addr, num_pages * PageSize, map_addr); + m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); break; } case OperationType::ChangePermissions: @@ -1884,25 +2072,25 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission default: ASSERT(false); } - return ResultSuccess; + R_SUCCEED(); } VAddr KPageTable::GetRegionAddress(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: - return address_space_start; + return m_address_space_start; case KMemoryState::Normal: - return heap_region_start; + return m_heap_region_start; case KMemoryState::Ipc: case KMemoryState::NonSecureIpc: case KMemoryState::NonDeviceIpc: - return alias_region_start; + return m_alias_region_start; case KMemoryState::Stack: - return stack_region_start; + return m_stack_region_start; case KMemoryState::Static: case KMemoryState::ThreadLocal: - return kernel_map_region_start; + return m_kernel_map_region_start; case KMemoryState::Io: case KMemoryState::Shared: case KMemoryState::AliasCode: @@ -1913,31 +2101,31 @@ VAddr KPageTable::GetRegionAddress(KMemoryState state) const { case KMemoryState::GeneratedCode: case KMemoryState::CodeOut: case KMemoryState::Coverage: - return alias_code_region_start; + return m_alias_code_region_start; case KMemoryState::Code: case KMemoryState::CodeData: - return code_region_start; + return m_code_region_start; default: UNREACHABLE(); } } -std::size_t KPageTable::GetRegionSize(KMemoryState state) const { +size_t KPageTable::GetRegionSize(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: - return address_space_end - address_space_start; + return m_address_space_end - m_address_space_start; case KMemoryState::Normal: - return heap_region_end - heap_region_start; + return m_heap_region_end - m_heap_region_start; case KMemoryState::Ipc: case KMemoryState::NonSecureIpc: case KMemoryState::NonDeviceIpc: - return alias_region_end - alias_region_start; + return m_alias_region_end - m_alias_region_start; case KMemoryState::Stack: - return stack_region_end - stack_region_start; + return m_stack_region_end - m_stack_region_start; case KMemoryState::Static: case KMemoryState::ThreadLocal: - return kernel_map_region_end - kernel_map_region_start; + return m_kernel_map_region_end - m_kernel_map_region_start; case KMemoryState::Io: case KMemoryState::Shared: case KMemoryState::AliasCode: @@ -1948,16 +2136,16 @@ std::size_t KPageTable::GetRegionSize(KMemoryState state) const { case KMemoryState::GeneratedCode: case KMemoryState::CodeOut: case KMemoryState::Coverage: - return alias_code_region_end - alias_code_region_start; + return m_alias_code_region_end - m_alias_code_region_start; case KMemoryState::Code: case KMemoryState::CodeData: - return code_region_end - code_region_start; + return m_code_region_end - m_code_region_start; default: UNREACHABLE(); } } -bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) const { +bool KPageTable::CanContain(VAddr addr, size_t size, KMemoryState state) const { const VAddr end = addr + size; const VAddr last = end - 1; @@ -1966,10 +2154,10 @@ bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) co const bool is_in_region = region_start <= addr && addr < end && last <= region_start + region_size - 1; - const bool is_in_heap = !(end <= heap_region_start || heap_region_end <= addr || - heap_region_start == heap_region_end); - const bool is_in_alias = !(end <= alias_region_start || alias_region_end <= addr || - alias_region_start == alias_region_end); + const bool is_in_heap = !(end <= m_heap_region_start || m_heap_region_end <= addr || + m_heap_region_start == m_heap_region_end); + const bool is_in_alias = !(end <= m_alias_region_start || m_alias_region_end <= addr || + m_alias_region_start == m_alias_region_end); switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: @@ -2008,23 +2196,23 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const { // Validate the states match expectation. - R_UNLESS((info.state & state_mask) == state, ResultInvalidCurrentMemory); - R_UNLESS((info.perm & perm_mask) == perm, ResultInvalidCurrentMemory); - R_UNLESS((info.attribute & attr_mask) == attr, ResultInvalidCurrentMemory); + R_UNLESS((info.m_state & state_mask) == state, ResultInvalidCurrentMemory); + R_UNLESS((info.m_permission & perm_mask) == perm, ResultInvalidCurrentMemory); + R_UNLESS((info.m_attribute & attr_mask) == attr, ResultInvalidCurrentMemory); - return ResultSuccess; + R_SUCCEED(); } -Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, - std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, +Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr addr, size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2042,7 +2230,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA // Advance our iterator. it++; - ASSERT(it != block_manager->cend()); + ASSERT(it != m_memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2054,12 +2242,12 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA *out_blocks_needed = blocks_for_start_align + blocks_for_end_align; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, std::size_t* out_blocks_needed, - VAddr addr, std::size_t size, KMemoryState state_mask, + KMemoryAttribute* out_attr, size_t* out_blocks_needed, + VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr) const { @@ -2067,7 +2255,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2075,14 +2263,14 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; // Validate all blocks in the range have correct state. - const KMemoryState first_state = info.state; - const KMemoryPermission first_perm = info.perm; - const KMemoryAttribute first_attr = info.attribute; + const KMemoryState first_state = info.m_state; + const KMemoryPermission first_perm = info.m_permission; + const KMemoryAttribute first_attr = info.m_attribute; while (true) { // Validate the current block. - R_UNLESS(info.state == first_state, ResultInvalidCurrentMemory); - R_UNLESS(info.perm == first_perm, ResultInvalidCurrentMemory); - R_UNLESS((info.attribute | ignore_attr) == (first_attr | ignore_attr), + R_UNLESS(info.m_state == first_state, ResultInvalidCurrentMemory); + R_UNLESS(info.m_permission == first_perm, ResultInvalidCurrentMemory); + R_UNLESS((info.m_attribute | ignore_attr) == (first_attr | ignore_attr), ResultInvalidCurrentMemory); // Validate against the provided masks. @@ -2095,7 +2283,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Advance our iterator. it++; - ASSERT(it != block_manager->cend()); + ASSERT(it != m_memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2116,7 +2304,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* if (out_blocks_needed != nullptr) { *out_blocks_needed = blocks_for_start_align + blocks_for_end_align; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr addr, size_t size, @@ -2134,7 +2322,7 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check that the output page group is empty, if it exists. if (out_pg) { @@ -2162,6 +2350,12 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); } + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Decide on new perm and attr. new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr | lock_attr); @@ -2172,9 +2366,11 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr } // Apply the memory block updates. - block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, @@ -2191,7 +2387,7 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the state. KMemoryState old_state{}; @@ -2213,15 +2409,23 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; KMemoryAttribute new_attr = static_cast<KMemoryAttribute>(old_attr & ~lock_attr); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Update permission, if we need to. if (new_perm != old_perm) { R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); } // Apply the memory block updates. - block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); - return ResultSuccess; + R_SUCCEED(); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_page_table.h b/src/core/hle/kernel/k_page_table.h index 25774f232..c6aeacd96 100644 --- a/src/core/hle/kernel/k_page_table.h +++ b/src/core/hle/kernel/k_page_table.h @@ -9,8 +9,10 @@ #include "common/common_types.h" #include "common/page_table.h" #include "core/file_sys/program_metadata.h" +#include "core/hle/kernel/k_dynamic_resource_manager.h" #include "core/hle/kernel/k_light_lock.h" #include "core/hle/kernel/k_memory_block.h" +#include "core/hle/kernel/k_memory_block_manager.h" #include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_memory_manager.h" #include "core/hle/result.h" @@ -34,58 +36,66 @@ public: ~KPageTable(); Result InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, - VAddr code_addr, std::size_t code_size, KMemoryManager::Pool pool); - Result MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, + VAddr code_addr, size_t code_size, + KMemoryBlockSlabManager* mem_block_slab_manager, + KMemoryManager::Pool pool); + + void Finalize(); + + Result MapProcessCode(VAddr addr, size_t pages_count, KMemoryState state, KMemoryPermission perm); - Result MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size); - Result UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, + Result MapCodeMemory(VAddr dst_address, VAddr src_address, size_t size); + Result UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t size, ICacheInvalidationStrategy icache_invalidation_strategy); - Result UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, + Result UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& src_page_table, VAddr src_addr); - Result MapPhysicalMemory(VAddr addr, std::size_t size); - Result UnmapPhysicalMemory(VAddr addr, std::size_t size); - Result MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); - Result UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); + Result MapPhysicalMemory(VAddr addr, size_t size); + Result UnmapPhysicalMemory(VAddr addr, size_t size); + Result MapMemory(VAddr dst_addr, VAddr src_addr, size_t size); + Result UnmapMemory(VAddr dst_addr, VAddr src_addr, size_t size); Result MapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state, KMemoryPermission perm); - Result MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, PAddr phys_addr, + Result MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, KMemoryState state, KMemoryPermission perm) { - return this->MapPages(out_addr, num_pages, alignment, phys_addr, true, - this->GetRegionAddress(state), this->GetRegionSize(state) / PageSize, - state, perm); + R_RETURN(this->MapPages(out_addr, num_pages, alignment, phys_addr, true, + this->GetRegionAddress(state), + this->GetRegionSize(state) / PageSize, state, perm)); } Result UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state); - Result UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state); - Result SetProcessMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm); + Result UnmapPages(VAddr address, size_t num_pages, KMemoryState state); + Result SetProcessMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm); KMemoryInfo QueryInfo(VAddr addr); - Result ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm); - Result ResetTransferMemory(VAddr addr, std::size_t size); - Result SetMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission perm); - Result SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr); - Result SetMaxHeapSize(std::size_t size); - Result SetHeapSize(VAddr* out, std::size_t size); - ResultVal<VAddr> AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, - bool is_map_only, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, - KMemoryPermission perm, PAddr map_addr = 0); - Result LockForDeviceAddressSpace(VAddr addr, std::size_t size); - Result UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); - Result LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size); - Result UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg); + Result SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission perm); + Result SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr); + Result SetMaxHeapSize(size_t size); + Result SetHeapSize(VAddr* out, size_t size); + ResultVal<VAddr> AllocateAndMapMemory(size_t needed_num_pages, size_t align, bool is_map_only, + VAddr region_start, size_t region_num_pages, + KMemoryState state, KMemoryPermission perm, + PAddr map_addr = 0); + + Result LockForMapDeviceAddressSpace(VAddr address, size_t size, KMemoryPermission perm, + bool is_aligned); + Result LockForUnmapDeviceAddressSpace(VAddr address, size_t size); + + Result UnlockForDeviceAddressSpace(VAddr addr, size_t size); + + Result LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size); + Result UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg); Result MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr); Common::PageTable& PageTableImpl() { - return page_table_impl; + return *m_page_table_impl; } const Common::PageTable& PageTableImpl() const { - return page_table_impl; + return *m_page_table_impl; } - bool CanContain(VAddr addr, std::size_t size, KMemoryState state) const; + bool CanContain(VAddr addr, size_t size, KMemoryState state) const; private: enum class OperationType : u32 { @@ -96,67 +106,65 @@ private: ChangePermissionsAndRefresh, }; - static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = KMemoryAttribute::DontCareMask | - KMemoryAttribute::IpcLocked | - KMemoryAttribute::DeviceShared; + static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = + KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared; - Result InitializeMemoryLayout(VAddr start, VAddr end); Result MapPages(VAddr addr, const KPageGroup& page_linked_list, KMemoryPermission perm); - Result MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, PAddr phys_addr, - bool is_pa_valid, VAddr region_start, std::size_t region_num_pages, + Result MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, + bool is_pa_valid, VAddr region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm); Result UnmapPages(VAddr addr, const KPageGroup& page_linked_list); - bool IsRegionMapped(VAddr address, u64 size); bool IsRegionContiguous(VAddr addr, u64 size) const; - void AddRegionToPages(VAddr start, std::size_t num_pages, KPageGroup& page_linked_list); + void AddRegionToPages(VAddr start, size_t num_pages, KPageGroup& page_linked_list); KMemoryInfo QueryInfoImpl(VAddr addr); - VAddr AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, u64 needed_num_pages, - std::size_t align); - Result Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, + VAddr AllocateVirtualMemory(VAddr start, size_t region_num_pages, u64 needed_num_pages, + size_t align); + Result Operate(VAddr addr, size_t num_pages, const KPageGroup& page_group, OperationType operation); - Result Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, - OperationType operation, PAddr map_addr = 0); + Result Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, OperationType operation, + PAddr map_addr = 0); VAddr GetRegionAddress(KMemoryState state) const; - std::size_t GetRegionSize(KMemoryState state) const; + size_t GetRegionSize(KMemoryState state) const; - VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, - std::size_t alignment, std::size_t offset, std::size_t guard_pages); + VAddr FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, + size_t alignment, size_t offset, size_t guard_pages); - Result CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, std::size_t size, + Result CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const; - Result CheckMemoryStateContiguous(VAddr addr, std::size_t size, KMemoryState state_mask, + Result CheckMemoryStateContiguous(VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const { - return this->CheckMemoryStateContiguous(nullptr, addr, size, state_mask, state, perm_mask, - perm, attr_mask, attr); + R_RETURN(this->CheckMemoryStateContiguous(nullptr, addr, size, state_mask, state, perm_mask, + perm, attr_mask, attr)); } Result CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const; Result CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, std::size_t* out_blocks_needed, VAddr addr, - std::size_t size, KMemoryState state_mask, KMemoryState state, + KMemoryAttribute* out_attr, size_t* out_blocks_needed, VAddr addr, + size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const; - Result CheckMemoryState(std::size_t* out_blocks_needed, VAddr addr, std::size_t size, + Result CheckMemoryState(size_t* out_blocks_needed, VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const { - return CheckMemoryState(nullptr, nullptr, nullptr, out_blocks_needed, addr, size, - state_mask, state, perm_mask, perm, attr_mask, attr, ignore_attr); + R_RETURN(CheckMemoryState(nullptr, nullptr, nullptr, out_blocks_needed, addr, size, + state_mask, state, perm_mask, perm, attr_mask, attr, + ignore_attr)); } - Result CheckMemoryState(VAddr addr, std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, + Result CheckMemoryState(VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) const { - return this->CheckMemoryState(nullptr, addr, size, state_mask, state, perm_mask, perm, - attr_mask, attr, ignore_attr); + R_RETURN(this->CheckMemoryState(nullptr, addr, size, state_mask, state, perm_mask, perm, + attr_mask, attr, ignore_attr)); } Result LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr addr, size_t size, @@ -174,13 +182,13 @@ private: bool IsValidPageGroup(const KPageGroup& pg, VAddr addr, size_t num_pages); bool IsLockedByCurrentThread() const { - return general_lock.IsLockedByCurrentThread(); + return m_general_lock.IsLockedByCurrentThread(); } bool IsHeapPhysicalAddress(const KMemoryLayout& layout, PAddr phys_addr) { ASSERT(this->IsLockedByCurrentThread()); - return layout.IsHeapPhysicalAddress(cached_physical_heap_region, phys_addr); + return layout.IsHeapPhysicalAddress(m_cached_physical_heap_region, phys_addr); } bool GetPhysicalAddressLocked(PAddr* out, VAddr virt_addr) const { @@ -191,95 +199,93 @@ private: return *out != 0; } - mutable KLightLock general_lock; - mutable KLightLock map_physical_memory_lock; - - std::unique_ptr<KMemoryBlockManager> block_manager; + mutable KLightLock m_general_lock; + mutable KLightLock m_map_physical_memory_lock; public: constexpr VAddr GetAddressSpaceStart() const { - return address_space_start; + return m_address_space_start; } constexpr VAddr GetAddressSpaceEnd() const { - return address_space_end; + return m_address_space_end; } - constexpr std::size_t GetAddressSpaceSize() const { - return address_space_end - address_space_start; + constexpr size_t GetAddressSpaceSize() const { + return m_address_space_end - m_address_space_start; } constexpr VAddr GetHeapRegionStart() const { - return heap_region_start; + return m_heap_region_start; } constexpr VAddr GetHeapRegionEnd() const { - return heap_region_end; + return m_heap_region_end; } - constexpr std::size_t GetHeapRegionSize() const { - return heap_region_end - heap_region_start; + constexpr size_t GetHeapRegionSize() const { + return m_heap_region_end - m_heap_region_start; } constexpr VAddr GetAliasRegionStart() const { - return alias_region_start; + return m_alias_region_start; } constexpr VAddr GetAliasRegionEnd() const { - return alias_region_end; + return m_alias_region_end; } - constexpr std::size_t GetAliasRegionSize() const { - return alias_region_end - alias_region_start; + constexpr size_t GetAliasRegionSize() const { + return m_alias_region_end - m_alias_region_start; } constexpr VAddr GetStackRegionStart() const { - return stack_region_start; + return m_stack_region_start; } constexpr VAddr GetStackRegionEnd() const { - return stack_region_end; + return m_stack_region_end; } - constexpr std::size_t GetStackRegionSize() const { - return stack_region_end - stack_region_start; + constexpr size_t GetStackRegionSize() const { + return m_stack_region_end - m_stack_region_start; } constexpr VAddr GetKernelMapRegionStart() const { - return kernel_map_region_start; + return m_kernel_map_region_start; } constexpr VAddr GetKernelMapRegionEnd() const { - return kernel_map_region_end; + return m_kernel_map_region_end; } constexpr VAddr GetCodeRegionStart() const { - return code_region_start; + return m_code_region_start; } constexpr VAddr GetCodeRegionEnd() const { - return code_region_end; + return m_code_region_end; } constexpr VAddr GetAliasCodeRegionStart() const { - return alias_code_region_start; + return m_alias_code_region_start; } constexpr VAddr GetAliasCodeRegionSize() const { - return alias_code_region_end - alias_code_region_start; + return m_alias_code_region_end - m_alias_code_region_start; } - std::size_t GetNormalMemorySize() { - KScopedLightLock lk(general_lock); - return GetHeapSize() + mapped_physical_memory_size; + size_t GetNormalMemorySize() { + KScopedLightLock lk(m_general_lock); + return GetHeapSize() + m_mapped_physical_memory_size; } - constexpr std::size_t GetAddressSpaceWidth() const { - return address_space_width; + constexpr size_t GetAddressSpaceWidth() const { + return m_address_space_width; } - constexpr std::size_t GetHeapSize() const { - return current_heap_end - heap_region_start; + constexpr size_t GetHeapSize() const { + return m_current_heap_end - m_heap_region_start; } - constexpr bool IsInsideAddressSpace(VAddr address, std::size_t size) const { - return address_space_start <= address && address + size - 1 <= address_space_end - 1; + constexpr bool IsInsideAddressSpace(VAddr address, size_t size) const { + return m_address_space_start <= address && address + size - 1 <= m_address_space_end - 1; } - constexpr bool IsOutsideAliasRegion(VAddr address, std::size_t size) const { - return alias_region_start > address || address + size - 1 > alias_region_end - 1; + constexpr bool IsOutsideAliasRegion(VAddr address, size_t size) const { + return m_alias_region_start > address || address + size - 1 > m_alias_region_end - 1; } - constexpr bool IsOutsideStackRegion(VAddr address, std::size_t size) const { - return stack_region_start > address || address + size - 1 > stack_region_end - 1; + constexpr bool IsOutsideStackRegion(VAddr address, size_t size) const { + return m_stack_region_start > address || address + size - 1 > m_stack_region_end - 1; } - constexpr bool IsInvalidRegion(VAddr address, std::size_t size) const { + constexpr bool IsInvalidRegion(VAddr address, size_t size) const { return address + size - 1 > GetAliasCodeRegionStart() + GetAliasCodeRegionSize() - 1; } - constexpr bool IsInsideHeapRegion(VAddr address, std::size_t size) const { - return address + size > heap_region_start && heap_region_end > address; + constexpr bool IsInsideHeapRegion(VAddr address, size_t size) const { + return address + size > m_heap_region_start && m_heap_region_end > address; } - constexpr bool IsInsideAliasRegion(VAddr address, std::size_t size) const { - return address + size > alias_region_start && alias_region_end > address; + constexpr bool IsInsideAliasRegion(VAddr address, size_t size) const { + return address + size > m_alias_region_start && m_alias_region_end > address; } - constexpr bool IsOutsideASLRRegion(VAddr address, std::size_t size) const { + constexpr bool IsOutsideASLRRegion(VAddr address, size_t size) const { if (IsInvalidRegion(address, size)) { return true; } @@ -291,73 +297,78 @@ public: } return {}; } - constexpr bool IsInsideASLRRegion(VAddr address, std::size_t size) const { + constexpr bool IsInsideASLRRegion(VAddr address, size_t size) const { return !IsOutsideASLRRegion(address, size); } - constexpr std::size_t GetNumGuardPages() const { + constexpr size_t GetNumGuardPages() const { return IsKernel() ? 1 : 4; } PAddr GetPhysicalAddr(VAddr addr) const { - const auto backing_addr = page_table_impl.backing_addr[addr >> PageBits]; + const auto backing_addr = m_page_table_impl->backing_addr[addr >> PageBits]; ASSERT(backing_addr); return backing_addr + addr; } constexpr bool Contains(VAddr addr) const { - return address_space_start <= addr && addr <= address_space_end - 1; + return m_address_space_start <= addr && addr <= m_address_space_end - 1; } - constexpr bool Contains(VAddr addr, std::size_t size) const { - return address_space_start <= addr && addr < addr + size && - addr + size - 1 <= address_space_end - 1; + constexpr bool Contains(VAddr addr, size_t size) const { + return m_address_space_start <= addr && addr < addr + size && + addr + size - 1 <= m_address_space_end - 1; } private: constexpr bool IsKernel() const { - return is_kernel; + return m_is_kernel; } constexpr bool IsAslrEnabled() const { - return is_aslr_enabled; + return m_enable_aslr; } - constexpr bool ContainsPages(VAddr addr, std::size_t num_pages) const { - return (address_space_start <= addr) && - (num_pages <= (address_space_end - address_space_start) / PageSize) && - (addr + num_pages * PageSize - 1 <= address_space_end - 1); + constexpr bool ContainsPages(VAddr addr, size_t num_pages) const { + return (m_address_space_start <= addr) && + (num_pages <= (m_address_space_end - m_address_space_start) / PageSize) && + (addr + num_pages * PageSize - 1 <= m_address_space_end - 1); } private: - VAddr address_space_start{}; - VAddr address_space_end{}; - VAddr heap_region_start{}; - VAddr heap_region_end{}; - VAddr current_heap_end{}; - VAddr alias_region_start{}; - VAddr alias_region_end{}; - VAddr stack_region_start{}; - VAddr stack_region_end{}; - VAddr kernel_map_region_start{}; - VAddr kernel_map_region_end{}; - VAddr code_region_start{}; - VAddr code_region_end{}; - VAddr alias_code_region_start{}; - VAddr alias_code_region_end{}; - - std::size_t mapped_physical_memory_size{}; - std::size_t max_heap_size{}; - std::size_t max_physical_memory_size{}; - std::size_t address_space_width{}; - - bool is_kernel{}; - bool is_aslr_enabled{}; - - u32 heap_fill_value{}; - const KMemoryRegion* cached_physical_heap_region{}; - - KMemoryManager::Pool memory_pool{KMemoryManager::Pool::Application}; - KMemoryManager::Direction allocation_option{KMemoryManager::Direction::FromFront}; - - Common::PageTable page_table_impl; - - Core::System& system; + VAddr m_address_space_start{}; + VAddr m_address_space_end{}; + VAddr m_heap_region_start{}; + VAddr m_heap_region_end{}; + VAddr m_current_heap_end{}; + VAddr m_alias_region_start{}; + VAddr m_alias_region_end{}; + VAddr m_stack_region_start{}; + VAddr m_stack_region_end{}; + VAddr m_kernel_map_region_start{}; + VAddr m_kernel_map_region_end{}; + VAddr m_code_region_start{}; + VAddr m_code_region_end{}; + VAddr m_alias_code_region_start{}; + VAddr m_alias_code_region_end{}; + + size_t m_mapped_physical_memory_size{}; + size_t m_max_heap_size{}; + size_t m_max_physical_memory_size{}; + size_t m_address_space_width{}; + + KMemoryBlockManager m_memory_block_manager; + + bool m_is_kernel{}; + bool m_enable_aslr{}; + bool m_enable_device_address_space_merge{}; + + KMemoryBlockSlabManager* m_memory_block_slab_manager{}; + + u32 m_heap_fill_value{}; + const KMemoryRegion* m_cached_physical_heap_region{}; + + KMemoryManager::Pool m_memory_pool{KMemoryManager::Pool::Application}; + KMemoryManager::Direction m_allocation_option{KMemoryManager::Direction::FromFront}; + + std::unique_ptr<Common::PageTable> m_page_table_impl; + + Core::System& m_system; }; } // namespace Kernel diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index d3e99665f..8c3495e5a 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -72,7 +72,8 @@ Result KProcess::Initialize(KProcess* process, Core::System& system, std::string process->name = std::move(process_name); process->resource_limit = res_limit; - process->status = ProcessStatus::Created; + process->system_resource_address = 0; + process->state = State::Created; process->program_id = 0; process->process_id = type == ProcessType::KernelInternal ? kernel.CreateNewKernelProcessID() : kernel.CreateNewUserProcessID(); @@ -92,11 +93,12 @@ Result KProcess::Initialize(KProcess* process, Core::System& system, std::string process->exception_thread = nullptr; process->is_suspended = false; process->schedule_count = 0; + process->is_handle_table_initialized = false; // Open a reference to the resource limit. process->resource_limit->Open(); - return ResultSuccess; + R_SUCCEED(); } void KProcess::DoWorkerTaskImpl() { @@ -121,9 +123,9 @@ void KProcess::DecrementRunningThreadCount() { } } -u64 KProcess::GetTotalPhysicalMemoryAvailable() const { +u64 KProcess::GetTotalPhysicalMemoryAvailable() { const u64 capacity{resource_limit->GetFreeValue(LimitableResource::PhysicalMemory) + - page_table->GetNormalMemorySize() + GetSystemResourceSize() + image_size + + page_table.GetNormalMemorySize() + GetSystemResourceSize() + image_size + main_thread_stack_size}; if (const auto pool_size = kernel.MemoryManager().GetSize(KMemoryManager::Pool::Application); capacity != pool_size) { @@ -135,16 +137,16 @@ u64 KProcess::GetTotalPhysicalMemoryAvailable() const { return memory_usage_capacity; } -u64 KProcess::GetTotalPhysicalMemoryAvailableWithoutSystemResource() const { +u64 KProcess::GetTotalPhysicalMemoryAvailableWithoutSystemResource() { return GetTotalPhysicalMemoryAvailable() - GetSystemResourceSize(); } -u64 KProcess::GetTotalPhysicalMemoryUsed() const { - return image_size + main_thread_stack_size + page_table->GetNormalMemorySize() + +u64 KProcess::GetTotalPhysicalMemoryUsed() { + return image_size + main_thread_stack_size + page_table.GetNormalMemorySize() + GetSystemResourceSize(); } -u64 KProcess::GetTotalPhysicalMemoryUsedWithoutSystemResource() const { +u64 KProcess::GetTotalPhysicalMemoryUsedWithoutSystemResource() { return GetTotalPhysicalMemoryUsed() - GetSystemResourceUsage(); } @@ -244,7 +246,7 @@ Result KProcess::AddSharedMemory(KSharedMemory* shmem, [[maybe_unused]] VAddr ad shmem->Open(); shemen_info->Open(); - return ResultSuccess; + R_SUCCEED(); } void KProcess::RemoveSharedMemory(KSharedMemory* shmem, [[maybe_unused]] VAddr address, @@ -289,12 +291,12 @@ Result KProcess::Reset() { KScopedSchedulerLock sl{kernel}; // Validate that we're in a state that we can reset. - R_UNLESS(status != ProcessStatus::Exited, ResultInvalidState); + R_UNLESS(state != State::Terminated, ResultInvalidState); R_UNLESS(is_signaled, ResultInvalidState); // Clear signaled. is_signaled = false; - return ResultSuccess; + R_SUCCEED(); } Result KProcess::SetActivity(ProcessActivity activity) { @@ -304,15 +306,13 @@ Result KProcess::SetActivity(ProcessActivity activity) { KScopedSchedulerLock sl{kernel}; // Validate our state. - R_UNLESS(status != ProcessStatus::Exiting, ResultInvalidState); - R_UNLESS(status != ProcessStatus::Exited, ResultInvalidState); + R_UNLESS(state != State::Terminating, ResultInvalidState); + R_UNLESS(state != State::Terminated, ResultInvalidState); // Either pause or resume. if (activity == ProcessActivity::Paused) { // Verify that we're not suspended. - if (is_suspended) { - return ResultInvalidState; - } + R_UNLESS(!is_suspended, ResultInvalidState); // Suspend all threads. for (auto* thread : GetThreadList()) { @@ -325,9 +325,7 @@ Result KProcess::SetActivity(ProcessActivity activity) { ASSERT(activity == ProcessActivity::Runnable); // Verify that we're suspended. - if (!is_suspended) { - return ResultInvalidState; - } + R_UNLESS(is_suspended, ResultInvalidState); // Resume all threads. for (auto* thread : GetThreadList()) { @@ -338,7 +336,7 @@ Result KProcess::SetActivity(ProcessActivity activity) { SetSuspended(false); } - return ResultSuccess; + R_SUCCEED(); } Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std::size_t code_size) { @@ -348,35 +346,38 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: system_resource_size = metadata.GetSystemResourceSize(); image_size = code_size; + // We currently do not support process-specific system resource + UNIMPLEMENTED_IF(system_resource_size != 0); + KScopedResourceReservation memory_reservation(resource_limit, LimitableResource::PhysicalMemory, code_size + system_resource_size); if (!memory_reservation.Succeeded()) { LOG_ERROR(Kernel, "Could not reserve process memory requirements of size {:X} bytes", code_size + system_resource_size); - return ResultLimitReached; + R_RETURN(ResultLimitReached); } // Initialize proces address space - if (const Result result{page_table->InitializeForProcess(metadata.GetAddressSpaceType(), false, - 0x8000000, code_size, - KMemoryManager::Pool::Application)}; + if (const Result result{page_table.InitializeForProcess( + metadata.GetAddressSpaceType(), false, 0x8000000, code_size, + &kernel.GetApplicationMemoryBlockManager(), KMemoryManager::Pool::Application)}; result.IsError()) { - return result; + R_RETURN(result); } // Map process code region - if (const Result result{page_table->MapProcessCode(page_table->GetCodeRegionStart(), - code_size / PageSize, KMemoryState::Code, - KMemoryPermission::None)}; + if (const Result result{page_table.MapProcessCode(page_table.GetCodeRegionStart(), + code_size / PageSize, KMemoryState::Code, + KMemoryPermission::None)}; result.IsError()) { - return result; + R_RETURN(result); } // Initialize process capabilities const auto& caps{metadata.GetKernelCapabilities()}; if (const Result result{ - capabilities.InitializeForUserProcess(caps.data(), caps.size(), *page_table)}; + capabilities.InitializeForUserProcess(caps.data(), caps.size(), page_table)}; result.IsError()) { - return result; + R_RETURN(result); } // Set memory usage capacity @@ -384,12 +385,12 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: case FileSys::ProgramAddressSpaceType::Is32Bit: case FileSys::ProgramAddressSpaceType::Is36Bit: case FileSys::ProgramAddressSpaceType::Is39Bit: - memory_usage_capacity = page_table->GetHeapRegionEnd() - page_table->GetHeapRegionStart(); + memory_usage_capacity = page_table.GetHeapRegionEnd() - page_table.GetHeapRegionStart(); break; case FileSys::ProgramAddressSpaceType::Is32BitNoMap: - memory_usage_capacity = page_table->GetHeapRegionEnd() - page_table->GetHeapRegionStart() + - page_table->GetAliasRegionEnd() - page_table->GetAliasRegionStart(); + memory_usage_capacity = page_table.GetHeapRegionEnd() - page_table.GetHeapRegionStart() + + page_table.GetAliasRegionEnd() - page_table.GetAliasRegionStart(); break; default: @@ -397,10 +398,10 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: } // Create TLS region - R_TRY(this->CreateThreadLocalRegion(std::addressof(tls_region_address))); + R_TRY(this->CreateThreadLocalRegion(std::addressof(plr_address))); memory_reservation.Commit(); - return handle_table.Initialize(capabilities.GetHandleTableSize()); + R_RETURN(handle_table.Initialize(capabilities.GetHandleTableSize())); } void KProcess::Run(s32 main_thread_priority, u64 stack_size) { @@ -409,15 +410,15 @@ void KProcess::Run(s32 main_thread_priority, u64 stack_size) { resource_limit->Reserve(LimitableResource::PhysicalMemory, main_thread_stack_size); const std::size_t heap_capacity{memory_usage_capacity - (main_thread_stack_size + image_size)}; - ASSERT(!page_table->SetMaxHeapSize(heap_capacity).IsError()); + ASSERT(!page_table.SetMaxHeapSize(heap_capacity).IsError()); - ChangeStatus(ProcessStatus::Running); + ChangeState(State::Running); SetupMainThread(kernel.System(), *this, main_thread_priority, main_thread_stack_top); } void KProcess::PrepareForTermination() { - ChangeStatus(ProcessStatus::Exiting); + ChangeState(State::Terminating); const auto stop_threads = [this](const std::vector<KThread*>& in_thread_list) { for (auto* thread : in_thread_list) { @@ -437,15 +438,15 @@ void KProcess::PrepareForTermination() { stop_threads(kernel.System().GlobalSchedulerContext().GetThreadList()); - this->DeleteThreadLocalRegion(tls_region_address); - tls_region_address = 0; + this->DeleteThreadLocalRegion(plr_address); + plr_address = 0; if (resource_limit) { resource_limit->Release(LimitableResource::PhysicalMemory, main_thread_stack_size + image_size); } - ChangeStatus(ProcessStatus::Exited); + ChangeState(State::Terminated); } void KProcess::Finalize() { @@ -474,7 +475,7 @@ void KProcess::Finalize() { } // Finalize the page table. - page_table.reset(); + page_table.Finalize(); // Perform inherited finalization. KAutoObjectWithSlabHeapAndContainer<KProcess, KWorkerTask>::Finalize(); @@ -499,7 +500,7 @@ Result KProcess::CreateThreadLocalRegion(VAddr* out) { } *out = tlr; - return ResultSuccess; + R_SUCCEED(); } } @@ -528,7 +529,7 @@ Result KProcess::CreateThreadLocalRegion(VAddr* out) { // We succeeded! tlp_guard.Cancel(); *out = tlr; - return ResultSuccess; + R_SUCCEED(); } Result KProcess::DeleteThreadLocalRegion(VAddr addr) { @@ -576,7 +577,7 @@ Result KProcess::DeleteThreadLocalRegion(VAddr addr) { KThreadLocalPage::Free(kernel, page_to_free); } - return ResultSuccess; + R_SUCCEED(); } bool KProcess::InsertWatchpoint(Core::System& system, VAddr addr, u64 size, @@ -628,7 +629,7 @@ bool KProcess::RemoveWatchpoint(Core::System& system, VAddr addr, u64 size, void KProcess::LoadModule(CodeSet code_set, VAddr base_addr) { const auto ReprotectSegment = [&](const CodeSet::Segment& segment, Svc::MemoryPermission permission) { - page_table->SetProcessMemoryPermission(segment.addr + base_addr, segment.size, permission); + page_table.SetProcessMemoryPermission(segment.addr + base_addr, segment.size, permission); }; kernel.System().Memory().WriteBlock(*this, base_addr, code_set.memory.data(), @@ -645,19 +646,18 @@ bool KProcess::IsSignaled() const { } KProcess::KProcess(KernelCore& kernel_) - : KAutoObjectWithSlabHeapAndContainer{kernel_}, page_table{std::make_unique<KPageTable>( - kernel_.System())}, + : KAutoObjectWithSlabHeapAndContainer{kernel_}, page_table{kernel_.System()}, handle_table{kernel_}, address_arbiter{kernel_.System()}, condition_var{kernel_.System()}, state_lock{kernel_}, list_lock{kernel_} {} KProcess::~KProcess() = default; -void KProcess::ChangeStatus(ProcessStatus new_status) { - if (status == new_status) { +void KProcess::ChangeState(State new_state) { + if (state == new_state) { return; } - status = new_status; + state = new_state; is_signaled = true; NotifyAvailable(); } @@ -668,17 +668,17 @@ Result KProcess::AllocateMainThreadStack(std::size_t stack_size) { // The kernel always ensures that the given stack size is page aligned. main_thread_stack_size = Common::AlignUp(stack_size, PageSize); - const VAddr start{page_table->GetStackRegionStart()}; - const std::size_t size{page_table->GetStackRegionEnd() - start}; + const VAddr start{page_table.GetStackRegionStart()}; + const std::size_t size{page_table.GetStackRegionEnd() - start}; CASCADE_RESULT(main_thread_stack_top, - page_table->AllocateAndMapMemory( + page_table.AllocateAndMapMemory( main_thread_stack_size / PageSize, PageSize, false, start, size / PageSize, KMemoryState::Stack, KMemoryPermission::UserReadWrite)); main_thread_stack_top += main_thread_stack_size; - return ResultSuccess; + R_SUCCEED(); } } // namespace Kernel diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index d56d73bab..2e0cc3d0b 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -13,6 +13,7 @@ #include "core/hle/kernel/k_auto_object.h" #include "core/hle/kernel/k_condition_variable.h" #include "core/hle/kernel/k_handle_table.h" +#include "core/hle/kernel/k_page_table.h" #include "core/hle/kernel/k_synchronization_object.h" #include "core/hle/kernel/k_thread_local_page.h" #include "core/hle/kernel/k_worker_task.h" @@ -31,7 +32,6 @@ class ProgramMetadata; namespace Kernel { class KernelCore; -class KPageTable; class KResourceLimit; class KThread; class KSharedMemoryInfo; @@ -45,24 +45,6 @@ enum class MemoryRegion : u16 { BASE = 3, }; -/** - * Indicates the status of a Process instance. - * - * @note These match the values as used by kernel, - * so new entries should only be added if RE - * shows that a new value has been introduced. - */ -enum class ProcessStatus { - Created, - CreatedWithDebuggerAttached, - Running, - WaitingForDebuggerToAttach, - DebuggerAttached, - Exiting, - Exited, - DebugBreak, -}; - enum class ProcessActivity : u32 { Runnable, Paused, @@ -89,6 +71,17 @@ public: explicit KProcess(KernelCore& kernel_); ~KProcess() override; + enum class State { + Created = static_cast<u32>(Svc::ProcessState::Created), + CreatedAttached = static_cast<u32>(Svc::ProcessState::CreatedAttached), + Running = static_cast<u32>(Svc::ProcessState::Running), + Crashed = static_cast<u32>(Svc::ProcessState::Crashed), + RunningAttached = static_cast<u32>(Svc::ProcessState::RunningAttached), + Terminating = static_cast<u32>(Svc::ProcessState::Terminating), + Terminated = static_cast<u32>(Svc::ProcessState::Terminated), + DebugBreak = static_cast<u32>(Svc::ProcessState::DebugBreak), + }; + enum : u64 { /// Lowest allowed process ID for a kernel initial process. InitialKIPIDMin = 1, @@ -114,12 +107,12 @@ public: /// Gets a reference to the process' page table. KPageTable& PageTable() { - return *page_table; + return page_table; } /// Gets const a reference to the process' page table. const KPageTable& PageTable() const { - return *page_table; + return page_table; } /// Gets a reference to the process' handle table. @@ -145,26 +138,25 @@ public: } Result WaitConditionVariable(VAddr address, u64 cv_key, u32 tag, s64 ns) { - return condition_var.Wait(address, cv_key, tag, ns); + R_RETURN(condition_var.Wait(address, cv_key, tag, ns)); } Result SignalAddressArbiter(VAddr address, Svc::SignalType signal_type, s32 value, s32 count) { - return address_arbiter.SignalToAddress(address, signal_type, value, count); + R_RETURN(address_arbiter.SignalToAddress(address, signal_type, value, count)); } Result WaitAddressArbiter(VAddr address, Svc::ArbitrationType arb_type, s32 value, s64 timeout) { - return address_arbiter.WaitForAddress(address, arb_type, value, timeout); + R_RETURN(address_arbiter.WaitForAddress(address, arb_type, value, timeout)); } - /// Gets the address to the process' dedicated TLS region. - VAddr GetTLSRegionAddress() const { - return tls_region_address; + VAddr GetProcessLocalRegionAddress() const { + return plr_address; } /// Gets the current status of the process - ProcessStatus GetStatus() const { - return status; + State GetState() const { + return state; } /// Gets the unique ID that identifies this particular process. @@ -286,18 +278,18 @@ public: } /// Retrieves the total physical memory available to this process in bytes. - u64 GetTotalPhysicalMemoryAvailable() const; + u64 GetTotalPhysicalMemoryAvailable(); /// Retrieves the total physical memory available to this process in bytes, /// without the size of the personal system resource heap added to it. - u64 GetTotalPhysicalMemoryAvailableWithoutSystemResource() const; + u64 GetTotalPhysicalMemoryAvailableWithoutSystemResource(); /// Retrieves the total physical memory used by this process in bytes. - u64 GetTotalPhysicalMemoryUsed() const; + u64 GetTotalPhysicalMemoryUsed(); /// Retrieves the total physical memory used by this process in bytes, /// without the size of the personal system resource heap added to it. - u64 GetTotalPhysicalMemoryUsedWithoutSystemResource() const; + u64 GetTotalPhysicalMemoryUsedWithoutSystemResource(); /// Gets the list of all threads created with this process as their owner. std::list<KThread*>& GetThreadList() { @@ -415,19 +407,24 @@ private: pinned_threads[core_id] = nullptr; } - /// Changes the process status. If the status is different - /// from the current process status, then this will trigger - /// a process signal. - void ChangeStatus(ProcessStatus new_status); + void FinalizeHandleTable() { + // Finalize the table. + handle_table.Finalize(); + + // Note that the table is finalized. + is_handle_table_initialized = false; + } + + void ChangeState(State new_state); /// Allocates the main thread stack for the process, given the stack size in bytes. Result AllocateMainThreadStack(std::size_t stack_size); /// Memory manager for this process - std::unique_ptr<KPageTable> page_table; + KPageTable page_table; /// Current status of the process - ProcessStatus status{}; + State state{}; /// The ID of this process u64 process_id = 0; @@ -443,6 +440,8 @@ private: /// Resource limit descriptor for this process KResourceLimit* resource_limit{}; + VAddr system_resource_address{}; + /// The ideal CPU core for this process, threads are scheduled on this core by default. u8 ideal_core = 0; @@ -469,7 +468,7 @@ private: KConditionVariable condition_var; /// Address indicating the location of the process' dedicated TLS region. - VAddr tls_region_address = 0; + VAddr plr_address = 0; /// Random values for svcGetInfo RandomEntropy std::array<u64, RANDOM_ENTROPY_SIZE> random_entropy{}; @@ -495,8 +494,12 @@ private: /// Schedule count of this process s64 schedule_count{}; + size_t memory_release_hint{}; + bool is_signaled{}; bool is_suspended{}; + bool is_immortal{}; + bool is_handle_table_initialized{}; bool is_initialized{}; std::atomic<u16> num_running_threads{}; diff --git a/src/core/hle/kernel/k_scheduler.cpp b/src/core/hle/kernel/k_scheduler.cpp index c34ce7a17..b1cabbca0 100644 --- a/src/core/hle/kernel/k_scheduler.cpp +++ b/src/core/hle/kernel/k_scheduler.cpp @@ -81,8 +81,8 @@ void KScheduler::RescheduleCurrentHLEThread(KernelCore& kernel) { // HACK: we cannot schedule from this thread, it is not a core thread ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1); - // Special case to ensure dummy threads that are waiting block - GetCurrentThread(kernel).IfDummyThreadTryWait(); + // Ensure dummy threads that are waiting block. + GetCurrentThread(kernel).DummyThreadBeginWait(); ASSERT(GetCurrentThread(kernel).GetState() != ThreadState::Waiting); GetCurrentThread(kernel).EnableDispatch(); @@ -314,6 +314,16 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) { idle_cores &= ~(1ULL << core_id); } + // HACK: any waiting dummy threads can wake up now. + kernel.GlobalSchedulerContext().WakeupWaitingDummyThreads(); + + // HACK: if we are a dummy thread, and we need to go sleep, indicate + // that for when the lock is released. + KThread* const cur_thread = GetCurrentThreadPointer(kernel); + if (cur_thread->IsDummyThread() && cur_thread->GetState() != ThreadState::Runnable) { + cur_thread->RequestDummyThreadWait(); + } + return cores_needing_scheduling; } @@ -531,11 +541,23 @@ void KScheduler::OnThreadStateChanged(KernelCore& kernel, KThread* thread, Threa GetPriorityQueue(kernel).Remove(thread); IncrementScheduledCount(thread); SetSchedulerUpdateNeeded(kernel); + + if (thread->IsDummyThread()) { + // HACK: if this is a dummy thread, it should no longer wake up when the + // scheduler lock is released. + kernel.GlobalSchedulerContext().UnregisterDummyThreadForWakeup(thread); + } } else if (cur_state == ThreadState::Runnable) { // If we're now runnable, then we weren't previously, and we should add. GetPriorityQueue(kernel).PushBack(thread); IncrementScheduledCount(thread); SetSchedulerUpdateNeeded(kernel); + + if (thread->IsDummyThread()) { + // HACK: if this is a dummy thread, it should wake up when the scheduler + // lock is released. + kernel.GlobalSchedulerContext().RegisterDummyThreadForWakeup(thread); + } } } diff --git a/src/core/hle/kernel/k_server_session.cpp b/src/core/hle/kernel/k_server_session.cpp index 4252c9adb..faf03fcc8 100644 --- a/src/core/hle/kernel/k_server_session.cpp +++ b/src/core/hle/kernel/k_server_session.cpp @@ -22,15 +22,12 @@ #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_thread_queue.h" #include "core/hle/kernel/kernel.h" -#include "core/hle/kernel/service_thread.h" #include "core/memory.h" namespace Kernel { using ThreadQueueImplForKServerSessionRequest = KThreadQueue; -static constexpr u32 MessageBufferSize = 0x100; - KServerSession::KServerSession(KernelCore& kernel_) : KSynchronizationObject{kernel_}, m_lock{kernel_} {} @@ -73,59 +70,7 @@ bool KServerSession::IsSignaled() const { } // Otherwise, we're signaled if we have a request and aren't handling one. - return !m_thread_request_list.empty() && m_current_thread_request == nullptr; -} - -void KServerSession::AppendDomainHandler(SessionRequestHandlerPtr handler) { - manager->AppendDomainHandler(std::move(handler)); -} - -std::size_t KServerSession::NumDomainRequestHandlers() const { - return manager->DomainHandlerCount(); -} - -Result KServerSession::HandleDomainSyncRequest(Kernel::HLERequestContext& context) { - if (!context.HasDomainMessageHeader()) { - return ResultSuccess; - } - - // Set domain handlers in HLE context, used for domain objects (IPC interfaces) as inputs - context.SetSessionRequestManager(manager); - - // If there is a DomainMessageHeader, then this is CommandType "Request" - const auto& domain_message_header = context.GetDomainMessageHeader(); - const u32 object_id{domain_message_header.object_id}; - switch (domain_message_header.command) { - case IPC::DomainMessageHeader::CommandType::SendMessage: - if (object_id > manager->DomainHandlerCount()) { - LOG_CRITICAL(IPC, - "object_id {} is too big! This probably means a recent service call " - "to {} needed to return a new interface!", - object_id, name); - ASSERT(false); - return ResultSuccess; // Ignore error if asserts are off - } - if (auto strong_ptr = manager->DomainHandler(object_id - 1).lock()) { - return strong_ptr->HandleSyncRequest(*this, context); - } else { - ASSERT(false); - return ResultSuccess; - } - - case IPC::DomainMessageHeader::CommandType::CloseVirtualHandle: { - LOG_DEBUG(IPC, "CloseVirtualHandle, object_id=0x{:08X}", object_id); - - manager->CloseDomainHandler(object_id - 1); - - IPC::ResponseBuilder rb{context, 2}; - rb.Push(ResultSuccess); - return ResultSuccess; - } - } - - LOG_CRITICAL(IPC, "Unknown domain command={}", domain_message_header.command.Value()); - ASSERT(false); - return ResultSuccess; + return !m_request_list.empty() && m_current_request == nullptr; } Result KServerSession::QueueSyncRequest(KThread* thread, Core::Memory::Memory& memory) { @@ -134,43 +79,11 @@ Result KServerSession::QueueSyncRequest(KThread* thread, Core::Memory::Memory& m context->PopulateFromIncomingCommandBuffer(kernel.CurrentProcess()->GetHandleTable(), cmd_buf); - // Ensure we have a session request handler - if (manager->HasSessionRequestHandler(*context)) { - if (auto strong_ptr = manager->GetServiceThread().lock()) { - strong_ptr->QueueSyncRequest(*parent, std::move(context)); - } else { - ASSERT_MSG(false, "strong_ptr is nullptr!"); - } - } else { - ASSERT_MSG(false, "handler is invalid!"); - } - - return ResultSuccess; + return manager->QueueSyncRequest(parent, std::move(context)); } Result KServerSession::CompleteSyncRequest(HLERequestContext& context) { - Result result = ResultSuccess; - - // If the session has been converted to a domain, handle the domain request - if (manager->HasSessionRequestHandler(context)) { - if (IsDomain() && context.HasDomainMessageHeader()) { - result = HandleDomainSyncRequest(context); - // If there is no domain header, the regular session handler is used - } else if (manager->HasSessionHandler()) { - // If this ServerSession has an associated HLE handler, forward the request to it. - result = manager->SessionHandler().HandleSyncRequest(*this, context); - } - } else { - ASSERT_MSG(false, "Session handler is invalid, stubbing response!"); - IPC::ResponseBuilder rb(context, 2); - rb.Push(ResultSuccess); - } - - if (convert_to_domain) { - ASSERT_MSG(!IsDomain(), "ServerSession is already a domain instance."); - manager->ConvertToDomain(); - convert_to_domain = false; - } + Result result = manager->CompleteSyncRequest(this, context); // The calling thread is waiting for this request to complete, so wake it up. context.GetThread().EndWait(result); @@ -178,7 +91,7 @@ Result KServerSession::CompleteSyncRequest(HLERequestContext& context) { return result; } -Result KServerSession::OnRequest() { +Result KServerSession::OnRequest(KSessionRequest* request) { // Create the wait queue. ThreadQueueImplForKServerSessionRequest wait_queue{kernel}; @@ -198,14 +111,13 @@ Result KServerSession::OnRequest() { this->QueueSyncRequest(GetCurrentThreadPointer(kernel), memory); } else { // Non-HLE request. - auto* thread{GetCurrentThreadPointer(kernel)}; // Get whether we're empty. - const bool was_empty = m_thread_request_list.empty(); + const bool was_empty = m_request_list.empty(); - // Add the thread to the list. - thread->Open(); - m_thread_request_list.push_back(thread); + // Add the request to the list. + request->Open(); + m_request_list.push_back(*request); // If we were empty, signal. if (was_empty) { @@ -213,6 +125,9 @@ Result KServerSession::OnRequest() { } } + // If we have a request event, this is asynchronous, and we don't need to wait. + R_SUCCEED_IF(request->GetEvent() != nullptr); + // This is a synchronous request, so we should wait for our request to complete. GetCurrentThread(kernel).SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::IPC); GetCurrentThread(kernel).BeginWait(&wait_queue); @@ -223,32 +138,32 @@ Result KServerSession::OnRequest() { Result KServerSession::SendReply() { // Lock the session. - KScopedLightLock lk(m_lock); + KScopedLightLock lk{m_lock}; // Get the request. - KThread* client_thread; + KSessionRequest* request; { KScopedSchedulerLock sl{kernel}; // Get the current request. - client_thread = m_current_thread_request; - R_UNLESS(client_thread != nullptr, ResultInvalidState); + request = m_current_request; + R_UNLESS(request != nullptr, ResultInvalidState); // Clear the current request, since we're processing it. - m_current_thread_request = nullptr; - if (!m_thread_request_list.empty()) { + m_current_request = nullptr; + if (!m_request_list.empty()) { this->NotifyAvailable(); } } // Close reference to the request once we're done processing it. - SCOPE_EXIT({ client_thread->Close(); }); + SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. - // const uintptr_t client_message = request->GetAddress(); - // const size_t client_buffer_size = request->GetSize(); - // KThread *client_thread = request->GetThread(); - // KEvent *event = request->GetEvent(); + const uintptr_t client_message = request->GetAddress(); + const size_t client_buffer_size = request->GetSize(); + KThread* client_thread = request->GetThread(); + KEvent* event = request->GetEvent(); // Check whether we're closed. const bool closed = (client_thread == nullptr || parent->IsClientClosed()); @@ -261,8 +176,8 @@ Result KServerSession::SendReply() { UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess()); auto* src_msg_buffer = memory.GetPointer(server_thread->GetTLSAddress()); - auto* dst_msg_buffer = memory.GetPointer(client_thread->GetTLSAddress()); - std::memcpy(dst_msg_buffer, src_msg_buffer, MessageBufferSize); + auto* dst_msg_buffer = memory.GetPointer(client_message); + std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); } else { result = ResultSessionClosed; } @@ -278,11 +193,30 @@ Result KServerSession::SendReply() { // If there's a client thread, update it. if (client_thread != nullptr) { - // End the client thread's wait. - KScopedSchedulerLock sl{kernel}; + if (event != nullptr) { + // // Get the client process/page table. + // KProcess *client_process = client_thread->GetOwnerProcess(); + // KPageTable *client_page_table = &client_process->PageTable(); + + // // If we need to, reply with an async error. + // if (R_FAILED(client_result)) { + // ReplyAsyncError(client_process, client_message, client_buffer_size, + // client_result); + // } + + // // Unlock the client buffer. + // // NOTE: Nintendo does not check the result of this. + // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + + // Signal the event. + event->Signal(); + } else { + // End the client thread's wait. + KScopedSchedulerLock sl{kernel}; - if (!client_thread->IsTerminationRequested()) { - client_thread->EndWait(client_result); + if (!client_thread->IsTerminationRequested()) { + client_thread->EndWait(client_result); + } } } @@ -291,10 +225,10 @@ Result KServerSession::SendReply() { Result KServerSession::ReceiveRequest() { // Lock the session. - KScopedLightLock lk(m_lock); + KScopedLightLock lk{m_lock}; // Get the request and client thread. - // KSessionRequest *request; + KSessionRequest* request; KThread* client_thread; { @@ -304,35 +238,41 @@ Result KServerSession::ReceiveRequest() { R_UNLESS(!parent->IsClientClosed(), ResultSessionClosed); // Ensure we aren't already servicing a request. - R_UNLESS(m_current_thread_request == nullptr, ResultNotFound); + R_UNLESS(m_current_request == nullptr, ResultNotFound); // Ensure we have a request to service. - R_UNLESS(!m_thread_request_list.empty(), ResultNotFound); + R_UNLESS(!m_request_list.empty(), ResultNotFound); // Pop the first request from the list. - client_thread = m_thread_request_list.front(); - m_thread_request_list.pop_front(); + request = &m_request_list.front(); + m_request_list.pop_front(); // Get the thread for the request. + client_thread = request->GetThread(); R_UNLESS(client_thread != nullptr, ResultSessionClosed); // Open the client thread. client_thread->Open(); } - // SCOPE_EXIT({ client_thread->Close(); }); + SCOPE_EXIT({ client_thread->Close(); }); // Set the request as our current. - m_current_thread_request = client_thread; + m_current_request = request; + + // Get the client address. + uintptr_t client_message = request->GetAddress(); + size_t client_buffer_size = request->GetSize(); + // bool recv_list_broken = false; // Receive the message. Core::Memory::Memory& memory{kernel.System().Memory()}; KThread* server_thread{GetCurrentThreadPointer(kernel)}; UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess()); - auto* src_msg_buffer = memory.GetPointer(client_thread->GetTLSAddress()); + auto* src_msg_buffer = memory.GetPointer(client_message); auto* dst_msg_buffer = memory.GetPointer(server_thread->GetTLSAddress()); - std::memcpy(dst_msg_buffer, src_msg_buffer, MessageBufferSize); + std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); // We succeeded. return ResultSuccess; @@ -344,35 +284,34 @@ void KServerSession::CleanupRequests() { // Clean up any pending requests. while (true) { // Get the next request. - // KSessionRequest *request = nullptr; - KThread* client_thread = nullptr; + KSessionRequest* request = nullptr; { KScopedSchedulerLock sl{kernel}; - if (m_current_thread_request) { + if (m_current_request) { // Choose the current request if we have one. - client_thread = m_current_thread_request; - m_current_thread_request = nullptr; - } else if (!m_thread_request_list.empty()) { + request = m_current_request; + m_current_request = nullptr; + } else if (!m_request_list.empty()) { // Pop the request from the front of the list. - client_thread = m_thread_request_list.front(); - m_thread_request_list.pop_front(); + request = &m_request_list.front(); + m_request_list.pop_front(); } } // If there's no request, we're done. - if (client_thread == nullptr) { + if (request == nullptr) { break; } // Close a reference to the request once it's cleaned up. - SCOPE_EXIT({ client_thread->Close(); }); + SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. // const uintptr_t client_message = request->GetAddress(); // const size_t client_buffer_size = request->GetSize(); - // KThread *client_thread = request->GetThread(); - // KEvent *event = request->GetEvent(); + KThread* client_thread = request->GetThread(); + KEvent* event = request->GetEvent(); // KProcess *server_process = request->GetServerProcess(); // KProcess *client_process = (client_thread != nullptr) ? @@ -385,11 +324,24 @@ void KServerSession::CleanupRequests() { // If there's a client thread, update it. if (client_thread != nullptr) { - // End the client thread's wait. - KScopedSchedulerLock sl{kernel}; - - if (!client_thread->IsTerminationRequested()) { - client_thread->EndWait(ResultSessionClosed); + if (event != nullptr) { + // // We need to reply async. + // ReplyAsyncError(client_process, client_message, client_buffer_size, + // (R_SUCCEEDED(result) ? ResultSessionClosed : result)); + + // // Unlock the client buffer. + // NOTE: Nintendo does not check the result of this. + // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + + // Signal the event. + event->Signal(); + } else { + // End the client thread's wait. + KScopedSchedulerLock sl{kernel}; + + if (!client_thread->IsTerminationRequested()) { + client_thread->EndWait(ResultSessionClosed); + } } } } diff --git a/src/core/hle/kernel/k_server_session.h b/src/core/hle/kernel/k_server_session.h index 748d52826..188aef4af 100644 --- a/src/core/hle/kernel/k_server_session.h +++ b/src/core/hle/kernel/k_server_session.h @@ -12,6 +12,7 @@ #include "core/hle/kernel/hle_ipc.h" #include "core/hle/kernel/k_light_lock.h" +#include "core/hle/kernel/k_session_request.h" #include "core/hle/kernel/k_synchronization_object.h" #include "core/hle/result.h" @@ -57,44 +58,15 @@ public: } bool IsSignaled() const override; - void OnClientClosed(); - void ClientConnected(SessionRequestHandlerPtr handler) { - if (manager) { - manager->SetSessionHandler(std::move(handler)); - } - } - - void ClientDisconnected() { - manager = nullptr; - } - - /// Adds a new domain request handler to the collection of request handlers within - /// this ServerSession instance. - void AppendDomainHandler(SessionRequestHandlerPtr handler); - - /// Retrieves the total number of domain request handlers that have been - /// appended to this ServerSession instance. - std::size_t NumDomainRequestHandlers() const; - - /// Returns true if the session has been converted to a domain, otherwise False - bool IsDomain() const { - return manager && manager->IsDomain(); - } - - /// Converts the session to a domain at the end of the current command - void ConvertToDomain() { - convert_to_domain = true; - } - /// Gets the session request manager, which forwards requests to the underlying service std::shared_ptr<SessionRequestManager>& GetSessionRequestManager() { return manager; } /// TODO: flesh these out to match the real kernel - Result OnRequest(); + Result OnRequest(KSessionRequest* request); Result SendReply(); Result ReceiveRequest(); @@ -108,10 +80,6 @@ private: /// Completes a sync request from the emulated application. Result CompleteSyncRequest(HLERequestContext& context); - /// Handles a SyncRequest to a domain, forwarding the request to the proper object or closing an - /// object handle. - Result HandleDomainSyncRequest(Kernel::HLERequestContext& context); - /// This session's HLE request handlers; if nullptr, this is not an HLE server std::shared_ptr<SessionRequestManager> manager; @@ -122,9 +90,8 @@ private: KSession* parent{}; /// List of threads which are pending a reply. - /// FIXME: KSessionRequest - std::list<KThread*> m_thread_request_list; - KThread* m_current_thread_request{}; + boost::intrusive::list<KSessionRequest> m_request_list; + KSessionRequest* m_current_request{}; KLightLock m_lock; }; diff --git a/src/core/hle/kernel/k_session_request.cpp b/src/core/hle/kernel/k_session_request.cpp new file mode 100644 index 000000000..520da6aa7 --- /dev/null +++ b/src/core/hle/kernel/k_session_request.cpp @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "core/hle/kernel/k_page_buffer.h" +#include "core/hle/kernel/k_session_request.h" + +namespace Kernel { + +Result KSessionRequest::SessionMappings::PushMap(VAddr client, VAddr server, size_t size, + KMemoryState state, size_t index) { + // At most 15 buffers of each type (4-bit descriptor counts). + ASSERT(index < ((1ul << 4) - 1) * 3); + + // Get the mapping. + Mapping* mapping; + if (index < NumStaticMappings) { + mapping = &m_static_mappings[index]; + } else { + // Allocate a page for the extra mappings. + if (m_mappings == nullptr) { + KPageBuffer* page_buffer = KPageBuffer::Allocate(kernel); + R_UNLESS(page_buffer != nullptr, ResultOutOfMemory); + + m_mappings = reinterpret_cast<Mapping*>(page_buffer); + } + + mapping = &m_mappings[index - NumStaticMappings]; + } + + // Set the mapping. + mapping->Set(client, server, size, state); + + return ResultSuccess; +} + +Result KSessionRequest::SessionMappings::PushSend(VAddr client, VAddr server, size_t size, + KMemoryState state) { + ASSERT(m_num_recv == 0); + ASSERT(m_num_exch == 0); + return this->PushMap(client, server, size, state, m_num_send++); +} + +Result KSessionRequest::SessionMappings::PushReceive(VAddr client, VAddr server, size_t size, + KMemoryState state) { + ASSERT(m_num_exch == 0); + return this->PushMap(client, server, size, state, m_num_send + m_num_recv++); +} + +Result KSessionRequest::SessionMappings::PushExchange(VAddr client, VAddr server, size_t size, + KMemoryState state) { + return this->PushMap(client, server, size, state, m_num_send + m_num_recv + m_num_exch++); +} + +void KSessionRequest::SessionMappings::Finalize() { + if (m_mappings) { + KPageBuffer::Free(kernel, reinterpret_cast<KPageBuffer*>(m_mappings)); + m_mappings = nullptr; + } +} + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_session_request.h b/src/core/hle/kernel/k_session_request.h new file mode 100644 index 000000000..e5558bc2c --- /dev/null +++ b/src/core/hle/kernel/k_session_request.h @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include <array> + +#include "core/hle/kernel/k_auto_object.h" +#include "core/hle/kernel/k_event.h" +#include "core/hle/kernel/k_memory_block.h" +#include "core/hle/kernel/k_process.h" +#include "core/hle/kernel/k_thread.h" +#include "core/hle/kernel/slab_helpers.h" + +namespace Kernel { + +class KSessionRequest final : public KSlabAllocated<KSessionRequest>, + public KAutoObject, + public boost::intrusive::list_base_hook<> { + KERNEL_AUTOOBJECT_TRAITS(KSessionRequest, KAutoObject); + +public: + class SessionMappings { + private: + static constexpr size_t NumStaticMappings = 8; + + class Mapping { + public: + constexpr void Set(VAddr c, VAddr s, size_t sz, KMemoryState st) { + m_client_address = c; + m_server_address = s; + m_size = sz; + m_state = st; + } + + constexpr VAddr GetClientAddress() const { + return m_client_address; + } + constexpr VAddr GetServerAddress() const { + return m_server_address; + } + constexpr size_t GetSize() const { + return m_size; + } + constexpr KMemoryState GetMemoryState() const { + return m_state; + } + + private: + VAddr m_client_address; + VAddr m_server_address; + size_t m_size; + KMemoryState m_state; + }; + + public: + explicit SessionMappings(KernelCore& kernel_) : kernel(kernel_) {} + + void Initialize() {} + void Finalize(); + + size_t GetSendCount() const { + return m_num_send; + } + size_t GetReceiveCount() const { + return m_num_recv; + } + size_t GetExchangeCount() const { + return m_num_exch; + } + + Result PushSend(VAddr client, VAddr server, size_t size, KMemoryState state); + Result PushReceive(VAddr client, VAddr server, size_t size, KMemoryState state); + Result PushExchange(VAddr client, VAddr server, size_t size, KMemoryState state); + + VAddr GetSendClientAddress(size_t i) const { + return GetSendMapping(i).GetClientAddress(); + } + VAddr GetSendServerAddress(size_t i) const { + return GetSendMapping(i).GetServerAddress(); + } + size_t GetSendSize(size_t i) const { + return GetSendMapping(i).GetSize(); + } + KMemoryState GetSendMemoryState(size_t i) const { + return GetSendMapping(i).GetMemoryState(); + } + + VAddr GetReceiveClientAddress(size_t i) const { + return GetReceiveMapping(i).GetClientAddress(); + } + VAddr GetReceiveServerAddress(size_t i) const { + return GetReceiveMapping(i).GetServerAddress(); + } + size_t GetReceiveSize(size_t i) const { + return GetReceiveMapping(i).GetSize(); + } + KMemoryState GetReceiveMemoryState(size_t i) const { + return GetReceiveMapping(i).GetMemoryState(); + } + + VAddr GetExchangeClientAddress(size_t i) const { + return GetExchangeMapping(i).GetClientAddress(); + } + VAddr GetExchangeServerAddress(size_t i) const { + return GetExchangeMapping(i).GetServerAddress(); + } + size_t GetExchangeSize(size_t i) const { + return GetExchangeMapping(i).GetSize(); + } + KMemoryState GetExchangeMemoryState(size_t i) const { + return GetExchangeMapping(i).GetMemoryState(); + } + + private: + Result PushMap(VAddr client, VAddr server, size_t size, KMemoryState state, size_t index); + + const Mapping& GetSendMapping(size_t i) const { + ASSERT(i < m_num_send); + + const size_t index = i; + if (index < NumStaticMappings) { + return m_static_mappings[index]; + } else { + return m_mappings[index - NumStaticMappings]; + } + } + + const Mapping& GetReceiveMapping(size_t i) const { + ASSERT(i < m_num_recv); + + const size_t index = m_num_send + i; + if (index < NumStaticMappings) { + return m_static_mappings[index]; + } else { + return m_mappings[index - NumStaticMappings]; + } + } + + const Mapping& GetExchangeMapping(size_t i) const { + ASSERT(i < m_num_exch); + + const size_t index = m_num_send + m_num_recv + i; + if (index < NumStaticMappings) { + return m_static_mappings[index]; + } else { + return m_mappings[index - NumStaticMappings]; + } + } + + private: + KernelCore& kernel; + std::array<Mapping, NumStaticMappings> m_static_mappings; + Mapping* m_mappings{}; + u8 m_num_send{}; + u8 m_num_recv{}; + u8 m_num_exch{}; + }; + +public: + explicit KSessionRequest(KernelCore& kernel_) : KAutoObject(kernel_), m_mappings(kernel_) {} + + static KSessionRequest* Create(KernelCore& kernel) { + KSessionRequest* req = KSessionRequest::Allocate(kernel); + if (req != nullptr) [[likely]] { + KAutoObject::Create(req); + } + return req; + } + + void Destroy() override { + this->Finalize(); + KSessionRequest::Free(kernel, this); + } + + void Initialize(KEvent* event, uintptr_t address, size_t size) { + m_mappings.Initialize(); + + m_thread = GetCurrentThreadPointer(kernel); + m_event = event; + m_address = address; + m_size = size; + + m_thread->Open(); + if (m_event != nullptr) { + m_event->Open(); + } + } + + static void PostDestroy(uintptr_t arg) {} + + KThread* GetThread() const { + return m_thread; + } + KEvent* GetEvent() const { + return m_event; + } + uintptr_t GetAddress() const { + return m_address; + } + size_t GetSize() const { + return m_size; + } + KProcess* GetServerProcess() const { + return m_server; + } + + void SetServerProcess(KProcess* process) { + m_server = process; + m_server->Open(); + } + + void ClearThread() { + m_thread = nullptr; + } + void ClearEvent() { + m_event = nullptr; + } + + size_t GetSendCount() const { + return m_mappings.GetSendCount(); + } + size_t GetReceiveCount() const { + return m_mappings.GetReceiveCount(); + } + size_t GetExchangeCount() const { + return m_mappings.GetExchangeCount(); + } + + Result PushSend(VAddr client, VAddr server, size_t size, KMemoryState state) { + return m_mappings.PushSend(client, server, size, state); + } + + Result PushReceive(VAddr client, VAddr server, size_t size, KMemoryState state) { + return m_mappings.PushReceive(client, server, size, state); + } + + Result PushExchange(VAddr client, VAddr server, size_t size, KMemoryState state) { + return m_mappings.PushExchange(client, server, size, state); + } + + VAddr GetSendClientAddress(size_t i) const { + return m_mappings.GetSendClientAddress(i); + } + VAddr GetSendServerAddress(size_t i) const { + return m_mappings.GetSendServerAddress(i); + } + size_t GetSendSize(size_t i) const { + return m_mappings.GetSendSize(i); + } + KMemoryState GetSendMemoryState(size_t i) const { + return m_mappings.GetSendMemoryState(i); + } + + VAddr GetReceiveClientAddress(size_t i) const { + return m_mappings.GetReceiveClientAddress(i); + } + VAddr GetReceiveServerAddress(size_t i) const { + return m_mappings.GetReceiveServerAddress(i); + } + size_t GetReceiveSize(size_t i) const { + return m_mappings.GetReceiveSize(i); + } + KMemoryState GetReceiveMemoryState(size_t i) const { + return m_mappings.GetReceiveMemoryState(i); + } + + VAddr GetExchangeClientAddress(size_t i) const { + return m_mappings.GetExchangeClientAddress(i); + } + VAddr GetExchangeServerAddress(size_t i) const { + return m_mappings.GetExchangeServerAddress(i); + } + size_t GetExchangeSize(size_t i) const { + return m_mappings.GetExchangeSize(i); + } + KMemoryState GetExchangeMemoryState(size_t i) const { + return m_mappings.GetExchangeMemoryState(i); + } + +private: + // NOTE: This is public and virtual in Nintendo's kernel. + void Finalize() override { + m_mappings.Finalize(); + + if (m_thread) { + m_thread->Close(); + } + if (m_event) { + m_event->Close(); + } + if (m_server) { + m_server->Close(); + } + } + +private: + SessionMappings m_mappings; + KThread* m_thread{}; + KProcess* m_server{}; + KEvent* m_event{}; + uintptr_t m_address{}; + size_t m_size{}; +}; + +} // namespace Kernel diff --git a/src/core/hle/kernel/k_shared_memory.cpp b/src/core/hle/kernel/k_shared_memory.cpp index 8ff1545b6..a039cc591 100644 --- a/src/core/hle/kernel/k_shared_memory.cpp +++ b/src/core/hle/kernel/k_shared_memory.cpp @@ -50,7 +50,7 @@ Result KSharedMemory::Initialize(Core::DeviceMemory& device_memory_, KProcess* o is_initialized = true; // Clear all pages in the memory. - std::memset(device_memory_.GetPointer(physical_address_), 0, size_); + std::memset(device_memory_.GetPointer<void>(physical_address_), 0, size_); return ResultSuccess; } diff --git a/src/core/hle/kernel/k_shared_memory.h b/src/core/hle/kernel/k_shared_memory.h index 34cb98456..5620c3660 100644 --- a/src/core/hle/kernel/k_shared_memory.h +++ b/src/core/hle/kernel/k_shared_memory.h @@ -54,7 +54,7 @@ public: * @return A pointer to the shared memory block from the specified offset */ u8* GetPointer(std::size_t offset = 0) { - return device_memory->GetPointer(physical_address + offset); + return device_memory->GetPointer<u8>(physical_address + offset); } /** @@ -63,7 +63,7 @@ public: * @return A pointer to the shared memory block from the specified offset */ const u8* GetPointer(std::size_t offset = 0) const { - return device_memory->GetPointer(physical_address + offset); + return device_memory->GetPointer<u8>(physical_address + offset); } void Finalize() override; diff --git a/src/core/hle/kernel/k_shared_memory_info.h b/src/core/hle/kernel/k_shared_memory_info.h index e43db8515..2bb6b6d08 100644 --- a/src/core/hle/kernel/k_shared_memory_info.h +++ b/src/core/hle/kernel/k_shared_memory_info.h @@ -15,7 +15,8 @@ class KSharedMemoryInfo final : public KSlabAllocated<KSharedMemoryInfo>, public boost::intrusive::list_base_hook<> { public: - explicit KSharedMemoryInfo() = default; + explicit KSharedMemoryInfo(KernelCore&) {} + KSharedMemoryInfo() = default; constexpr void Initialize(KSharedMemory* shmem) { shared_memory = shmem; diff --git a/src/core/hle/kernel/k_slab_heap.h b/src/core/hle/kernel/k_slab_heap.h index 2b303537e..a8c77a7d4 100644 --- a/src/core/hle/kernel/k_slab_heap.h +++ b/src/core/hle/kernel/k_slab_heap.h @@ -8,6 +8,7 @@ #include "common/assert.h" #include "common/common_funcs.h" #include "common/common_types.h" +#include "common/spin_lock.h" namespace Kernel { @@ -36,28 +37,34 @@ public: } void* Allocate() { - Node* ret = m_head.load(); + // KScopedInterruptDisable di; - do { - if (ret == nullptr) { - break; - } - } while (!m_head.compare_exchange_weak(ret, ret->next)); + m_lock.lock(); + + Node* ret = m_head; + if (ret != nullptr) [[likely]] { + m_head = ret->next; + } + m_lock.unlock(); return ret; } void Free(void* obj) { + // KScopedInterruptDisable di; + + m_lock.lock(); + Node* node = static_cast<Node*>(obj); + node->next = m_head; + m_head = node; - Node* cur_head = m_head.load(); - do { - node->next = cur_head; - } while (!m_head.compare_exchange_weak(cur_head, node)); + m_lock.unlock(); } private: std::atomic<Node*> m_head{}; + Common::SpinLock m_lock; }; } // namespace impl diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 174afc80d..d57b42fdf 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -30,6 +30,7 @@ #include "core/hle/kernel/k_worker_task_manager.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/svc_results.h" +#include "core/hle/kernel/svc_types.h" #include "core/hle/result.h" #include "core/memory.h" @@ -38,6 +39,9 @@ #endif namespace { + +constexpr inline s32 TerminatingThreadPriority = Kernel::Svc::SystemThreadPriorityHighest - 1; + static void ResetThreadContext32(Core::ARM_Interface::ThreadContext32& context, u32 stack_top, u32 entry_point, u32 arg) { context = {}; @@ -144,7 +148,9 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack physical_affinity_mask.SetAffinity(phys_core, true); // Set the thread state. - thread_state = (type == ThreadType::Main) ? ThreadState::Runnable : ThreadState::Initialized; + thread_state = (type == ThreadType::Main || type == ThreadType::Dummy) + ? ThreadState::Runnable + : ThreadState::Initialized; // Set TLS address. tls_address = 0; @@ -241,7 +247,7 @@ Result KThread::Initialize(KThreadFunction func, uintptr_t arg, VAddr user_stack } } - return ResultSuccess; + R_SUCCEED(); } Result KThread::InitializeThread(KThread* thread, KThreadFunction func, uintptr_t arg, @@ -254,7 +260,7 @@ Result KThread::InitializeThread(KThread* thread, KThreadFunction func, uintptr_ thread->host_context = std::make_shared<Common::Fiber>(std::move(init_func)); thread->is_single_core = !Settings::values.use_multi_core.GetValue(); - return ResultSuccess; + R_SUCCEED(); } Result KThread::InitializeDummyThread(KThread* thread) { @@ -264,31 +270,32 @@ Result KThread::InitializeDummyThread(KThread* thread) { // Initialize emulation parameters. thread->stack_parameters.disable_count = 0; - return ResultSuccess; + R_SUCCEED(); } Result KThread::InitializeMainThread(Core::System& system, KThread* thread, s32 virt_core) { - return InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, ThreadType::Main, - system.GetCpuManager().GetGuestActivateFunc()); + R_RETURN(InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, + ThreadType::Main, system.GetCpuManager().GetGuestActivateFunc())); } Result KThread::InitializeIdleThread(Core::System& system, KThread* thread, s32 virt_core) { - return InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, ThreadType::Main, - system.GetCpuManager().GetIdleThreadStartFunc()); + R_RETURN(InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, + ThreadType::Main, system.GetCpuManager().GetIdleThreadStartFunc())); } Result KThread::InitializeHighPriorityThread(Core::System& system, KThread* thread, KThreadFunction func, uintptr_t arg, s32 virt_core) { - return InitializeThread(thread, func, arg, {}, {}, virt_core, nullptr, ThreadType::HighPriority, - system.GetCpuManager().GetShutdownThreadStartFunc()); + R_RETURN(InitializeThread(thread, func, arg, {}, {}, virt_core, nullptr, + ThreadType::HighPriority, + system.GetCpuManager().GetShutdownThreadStartFunc())); } Result KThread::InitializeUserThread(Core::System& system, KThread* thread, KThreadFunction func, uintptr_t arg, VAddr user_stack_top, s32 prio, s32 virt_core, KProcess* owner) { system.Kernel().GlobalSchedulerContext().AddThread(thread); - return InitializeThread(thread, func, arg, user_stack_top, prio, virt_core, owner, - ThreadType::User, system.GetCpuManager().GetGuestThreadFunc()); + R_RETURN(InitializeThread(thread, func, arg, user_stack_top, prio, virt_core, owner, + ThreadType::User, system.GetCpuManager().GetGuestThreadFunc())); } void KThread::PostDestroy(uintptr_t arg) { @@ -538,7 +545,7 @@ Result KThread::GetCoreMask(s32* out_ideal_core, u64* out_affinity_mask) { *out_ideal_core = virtual_ideal_core_id; *out_affinity_mask = virtual_affinity_mask; - return ResultSuccess; + R_SUCCEED(); } Result KThread::GetPhysicalCoreMask(s32* out_ideal_core, u64* out_affinity_mask) { @@ -554,7 +561,7 @@ Result KThread::GetPhysicalCoreMask(s32* out_ideal_core, u64* out_affinity_mask) *out_affinity_mask = original_physical_affinity_mask.GetAffinityMask(); } - return ResultSuccess; + R_SUCCEED(); } Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { @@ -666,7 +673,7 @@ Result KThread::SetCoreMask(s32 core_id_, u64 v_affinity_mask) { } while (retry_update); } - return ResultSuccess; + R_SUCCEED(); } void KThread::SetBasePriority(s32 value) { @@ -839,7 +846,7 @@ Result KThread::SetActivity(Svc::ThreadActivity activity) { } while (thread_is_current); } - return ResultSuccess; + R_SUCCEED(); } Result KThread::GetThreadContext3(std::vector<u8>& out) { @@ -874,7 +881,7 @@ Result KThread::GetThreadContext3(std::vector<u8>& out) { } } - return ResultSuccess; + R_SUCCEED(); } void KThread::AddWaiterImpl(KThread* thread) { @@ -1038,7 +1045,7 @@ Result KThread::Run() { // Set our state and finish. SetState(ThreadState::Runnable); - return ResultSuccess; + R_SUCCEED(); } } @@ -1073,6 +1080,78 @@ void KThread::Exit() { UNREACHABLE_MSG("KThread::Exit() would return"); } +Result KThread::Terminate() { + ASSERT(this != GetCurrentThreadPointer(kernel)); + + // Request the thread terminate if it hasn't already. + if (const auto new_state = this->RequestTerminate(); new_state != ThreadState::Terminated) { + // If the thread isn't terminated, wait for it to terminate. + s32 index; + KSynchronizationObject* objects[] = {this}; + R_TRY(KSynchronizationObject::Wait(kernel, std::addressof(index), objects, 1, + Svc::WaitInfinite)); + } + + R_SUCCEED(); +} + +ThreadState KThread::RequestTerminate() { + ASSERT(this != GetCurrentThreadPointer(kernel)); + + KScopedSchedulerLock sl{kernel}; + + // Determine if this is the first termination request. + const bool first_request = [&]() -> bool { + // Perform an atomic compare-and-swap from false to true. + bool expected = false; + return termination_requested.compare_exchange_strong(expected, true); + }(); + + // If this is the first request, start termination procedure. + if (first_request) { + // If the thread is in initialized state, just change state to terminated. + if (this->GetState() == ThreadState::Initialized) { + thread_state = ThreadState::Terminated; + return ThreadState::Terminated; + } + + // Register the terminating dpc. + this->RegisterDpc(DpcFlag::Terminating); + + // If the thread is pinned, unpin it. + if (this->GetStackParameters().is_pinned) { + this->GetOwnerProcess()->UnpinThread(this); + } + + // If the thread is suspended, continue it. + if (this->IsSuspended()) { + suspend_allowed_flags = 0; + this->UpdateState(); + } + + // Change the thread's priority to be higher than any system thread's. + if (this->GetBasePriority() >= Svc::SystemThreadPriorityHighest) { + this->SetBasePriority(TerminatingThreadPriority); + } + + // If the thread is runnable, send a termination interrupt to other cores. + if (this->GetState() == ThreadState::Runnable) { + if (const u64 core_mask = + physical_affinity_mask.GetAffinityMask() & ~(1ULL << GetCurrentCoreId(kernel)); + core_mask != 0) { + Kernel::KInterruptManager::SendInterProcessorInterrupt(kernel, core_mask); + } + } + + // Wake up the thread. + if (this->GetState() == ThreadState::Waiting) { + wait_queue->CancelWait(this, ResultTerminationRequested, true); + } + } + + return this->GetState(); +} + Result KThread::Sleep(s64 timeout) { ASSERT(!kernel.GlobalSchedulerContext().IsLocked()); ASSERT(this == GetCurrentThreadPointer(kernel)); @@ -1086,7 +1165,7 @@ Result KThread::Sleep(s64 timeout) { // Check if the thread should terminate. if (this->IsTerminationRequested()) { slp.CancelSleep(); - return ResultTerminationRequested; + R_THROW(ResultTerminationRequested); } // Wait for the sleep to end. @@ -1094,33 +1173,32 @@ Result KThread::Sleep(s64 timeout) { SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::Sleep); } - return ResultSuccess; + R_SUCCEED(); } -void KThread::IfDummyThreadTryWait() { - if (!IsDummyThread()) { - return; - } +void KThread::RequestDummyThreadWait() { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(this->IsDummyThread()); - if (GetState() != ThreadState::Waiting) { - return; - } + // We will block when the scheduler lock is released. + dummy_thread_runnable.store(false); +} +void KThread::DummyThreadBeginWait() { + ASSERT(this->IsDummyThread()); ASSERT(!kernel.IsPhantomModeForSingleCore()); - // Block until we are no longer waiting. - std::unique_lock lk(dummy_wait_lock); - dummy_wait_cv.wait( - lk, [&] { return GetState() != ThreadState::Waiting || kernel.IsShuttingDown(); }); + // Block until runnable is no longer false. + dummy_thread_runnable.wait(false); } -void KThread::IfDummyThreadEndWait() { - if (!IsDummyThread()) { - return; - } +void KThread::DummyThreadEndWait() { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(kernel)); + ASSERT(this->IsDummyThread()); // Wake up the waiting thread. - dummy_wait_cv.notify_one(); + dummy_thread_runnable.store(true); + dummy_thread_runnable.notify_one(); } void KThread::BeginWait(KThreadQueue* queue) { @@ -1154,9 +1232,6 @@ void KThread::EndWait(Result wait_result_) { } wait_queue->EndWait(this, wait_result_); - - // Special case for dummy threads to wakeup if necessary. - IfDummyThreadEndWait(); } } diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index 9ee20208e..30aa10c9a 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -180,6 +180,10 @@ public: void Exit(); + Result Terminate(); + + ThreadState RequestTerminate(); + [[nodiscard]] u32 GetSuspendFlags() const { return suspend_allowed_flags & suspend_request_flags; } @@ -639,8 +643,9 @@ public: // therefore will not block on guest kernel synchronization primitives. These methods handle // blocking as needed. - void IfDummyThreadTryWait(); - void IfDummyThreadEndWait(); + void RequestDummyThreadWait(); + void DummyThreadBeginWait(); + void DummyThreadEndWait(); [[nodiscard]] uintptr_t GetArgument() const { return argument; @@ -773,8 +778,7 @@ private: bool is_single_core{}; ThreadType thread_type{}; StepState step_state{}; - std::mutex dummy_wait_lock; - std::condition_variable dummy_wait_cv; + std::atomic<bool> dummy_thread_runnable{true}; // For debugging std::vector<KSynchronizationObject*> wait_objects_for_debugging; diff --git a/src/core/hle/kernel/k_thread_local_page.h b/src/core/hle/kernel/k_thread_local_page.h index 0a7f22680..5d466ace7 100644 --- a/src/core/hle/kernel/k_thread_local_page.h +++ b/src/core/hle/kernel/k_thread_local_page.h @@ -26,7 +26,7 @@ public: static_assert(RegionsPerPage > 0); public: - constexpr explicit KThreadLocalPage(VAddr addr = {}) : m_virt_addr(addr) { + constexpr explicit KThreadLocalPage(KernelCore&, VAddr addr = {}) : m_virt_addr(addr) { m_is_region_free.fill(true); } diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 9251f29ad..fdc774e30 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -24,6 +24,7 @@ #include "core/hardware_properties.h" #include "core/hle/kernel/init/init_slab_setup.h" #include "core/hle/kernel/k_client_port.h" +#include "core/hle/kernel/k_dynamic_resource_manager.h" #include "core/hle/kernel/k_handle_table.h" #include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_memory_manager.h" @@ -47,8 +48,8 @@ namespace Kernel { struct KernelCore::Impl { explicit Impl(Core::System& system_, KernelCore& kernel_) - : time_manager{system_}, - service_threads_manager{1, "ServiceThreadsManager"}, system{system_} {} + : time_manager{system_}, service_threads_manager{1, "ServiceThreadsManager"}, + service_thread_barrier{2}, system{system_} {} void SetMulticore(bool is_multi) { is_multicore = is_multi; @@ -73,8 +74,16 @@ struct KernelCore::Impl { InitializeMemoryLayout(); Init::InitializeKPageBufferSlabHeap(system); InitializeShutdownThreads(); - InitializePreemption(kernel); InitializePhysicalCores(); + InitializePreemption(kernel); + + // Initialize the Dynamic Slab Heaps. + { + const auto& pt_heap_region = memory_layout->GetPageTableHeapRegion(); + ASSERT(pt_heap_region.GetEndAddress() != 0); + + InitializeResourceManagers(pt_heap_region.GetAddress(), pt_heap_region.GetSize()); + } RegisterHostThread(); } @@ -86,6 +95,15 @@ struct KernelCore::Impl { } } + void CloseCurrentProcess() { + (*current_process).Finalize(); + // current_process->Close(); + // TODO: The current process should be destroyed based on accurate ref counting after + // calling Close(). Adding a manual Destroy() call instead to avoid a memory leak. + (*current_process).Destroy(); + current_process = nullptr; + } + void Shutdown() { is_shutting_down.store(true, std::memory_order_relaxed); SCOPE_EXIT({ is_shutting_down.store(false, std::memory_order_relaxed); }); @@ -99,10 +117,6 @@ struct KernelCore::Impl { next_user_process_id = KProcess::ProcessIDMin; next_thread_id = 1; - for (auto& core : cores) { - core = nullptr; - } - global_handle_table->Finalize(); global_handle_table.reset(); @@ -152,15 +166,7 @@ struct KernelCore::Impl { } } - // Shutdown all processes. - if (current_process) { - (*current_process).Finalize(); - // current_process->Close(); - // TODO: The current process should be destroyed based on accurate ref counting after - // calling Close(). Adding a manual Destroy() call instead to avoid a memory leak. - (*current_process).Destroy(); - current_process = nullptr; - } + CloseCurrentProcess(); // Track kernel objects that were not freed on shutdown { @@ -257,6 +263,18 @@ struct KernelCore::Impl { system.CoreTiming().ScheduleLoopingEvent(time_interval, time_interval, preemption_event); } + void InitializeResourceManagers(VAddr address, size_t size) { + dynamic_page_manager = std::make_unique<KDynamicPageManager>(); + memory_block_heap = std::make_unique<KMemoryBlockSlabHeap>(); + app_memory_block_manager = std::make_unique<KMemoryBlockSlabManager>(); + + dynamic_page_manager->Initialize(address, size); + static constexpr size_t ApplicationMemoryBlockSlabHeapSize = 20000; + memory_block_heap->Initialize(dynamic_page_manager.get(), + ApplicationMemoryBlockSlabHeapSize); + app_memory_block_manager->Initialize(nullptr, memory_block_heap.get()); + } + void InitializeShutdownThreads() { for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { shutdown_threads[core_id] = KThread::Create(system.Kernel()); @@ -344,11 +362,6 @@ struct KernelCore::Impl { static inline thread_local KThread* current_thread{nullptr}; KThread* GetCurrentEmuThread() { - // If we are shutting down the kernel, none of this is relevant anymore. - if (IsShuttingDown()) { - return {}; - } - const auto thread_id = GetCurrentHostThreadID(); if (thread_id >= Core::Hardware::NUM_CPU_CORES) { return GetHostDummyThread(); @@ -724,7 +737,12 @@ struct KernelCore::Impl { } void ClearServiceThreads() { - service_threads_manager.QueueWork([this]() { service_threads.clear(); }); + service_threads_manager.QueueWork([this] { + service_threads.clear(); + default_service_thread.reset(); + service_thread_barrier.Sync(); + }); + service_thread_barrier.Sync(); } std::mutex server_objects_lock; @@ -770,6 +788,11 @@ struct KernelCore::Impl { // Kernel memory management std::unique_ptr<KMemoryManager> memory_manager; + // Dynamic slab managers + std::unique_ptr<KDynamicPageManager> dynamic_page_manager; + std::unique_ptr<KMemoryBlockSlabHeap> memory_block_heap; + std::unique_ptr<KMemoryBlockSlabManager> app_memory_block_manager; + // Shared memory for services Kernel::KSharedMemory* hid_shared_mem{}; Kernel::KSharedMemory* font_shared_mem{}; @@ -784,6 +807,7 @@ struct KernelCore::Impl { std::unordered_set<std::shared_ptr<ServiceThread>> service_threads; std::weak_ptr<ServiceThread> default_service_thread; Common::ThreadWorker service_threads_manager; + Common::Barrier service_thread_barrier; std::array<KThread*, Core::Hardware::NUM_CPU_CORES> shutdown_threads; std::array<std::unique_ptr<Kernel::KScheduler>, Core::Hardware::NUM_CPU_CORES> schedulers{}; @@ -853,6 +877,10 @@ const KProcess* KernelCore::CurrentProcess() const { return impl->current_process; } +void KernelCore::CloseCurrentProcess() { + impl->CloseCurrentProcess(); +} + const std::vector<KProcess*>& KernelCore::GetProcessList() const { return impl->process_list; } @@ -1041,6 +1069,14 @@ const KMemoryManager& KernelCore::MemoryManager() const { return *impl->memory_manager; } +KMemoryBlockSlabManager& KernelCore::GetApplicationMemoryBlockManager() { + return *impl->app_memory_block_manager; +} + +const KMemoryBlockSlabManager& KernelCore::GetApplicationMemoryBlockManager() const { + return *impl->app_memory_block_manager; +} + Kernel::KSharedMemory& KernelCore::GetHidSharedMem() { return *impl->hid_shared_mem; } diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index 0847cbcbf..266be2bc4 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -37,6 +37,7 @@ class KClientSession; class KEvent; class KHandleTable; class KLinkedListNode; +class KMemoryBlockSlabManager; class KMemoryLayout; class KMemoryManager; class KPageBuffer; @@ -46,6 +47,7 @@ class KResourceLimit; class KScheduler; class KServerSession; class KSession; +class KSessionRequest; class KSharedMemory; class KSharedMemoryInfo; class KThread; @@ -130,6 +132,9 @@ public: /// Retrieves a const pointer to the current process. const KProcess* CurrentProcess() const; + /// Closes the current process. + void CloseCurrentProcess(); + /// Retrieves the list of processes. const std::vector<KProcess*>& GetProcessList() const; @@ -238,6 +243,12 @@ public: /// Gets the virtual memory manager for the kernel. const KMemoryManager& MemoryManager() const; + /// Gets the application memory block manager for the kernel. + KMemoryBlockSlabManager& GetApplicationMemoryBlockManager(); + + /// Gets the application memory block manager for the kernel. + const KMemoryBlockSlabManager& GetApplicationMemoryBlockManager() const; + /// Gets the shared memory object for HID services. Kernel::KSharedMemory& GetHidSharedMem(); @@ -350,6 +361,8 @@ public: return slab_heap_container->page_buffer; } else if constexpr (std::is_same_v<T, KThreadLocalPage>) { return slab_heap_container->thread_local_page; + } else if constexpr (std::is_same_v<T, KSessionRequest>) { + return slab_heap_container->session_request; } } @@ -412,6 +425,7 @@ private: KSlabHeap<KCodeMemory> code_memory; KSlabHeap<KPageBuffer> page_buffer; KSlabHeap<KThreadLocalPage> thread_local_page; + KSlabHeap<KSessionRequest> session_request; }; std::unique_ptr<SlabHeapContainer> slab_heap_container; diff --git a/src/core/hle/kernel/slab_helpers.h b/src/core/hle/kernel/slab_helpers.h index 299a981a8..06b51e919 100644 --- a/src/core/hle/kernel/slab_helpers.h +++ b/src/core/hle/kernel/slab_helpers.h @@ -24,7 +24,7 @@ public: } static Derived* Allocate(KernelCore& kernel) { - return kernel.SlabHeap<Derived>().Allocate(); + return kernel.SlabHeap<Derived>().Allocate(kernel); } static void Free(KernelCore& kernel, Derived* obj) { diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 1d145ea91..4aca5b27d 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -751,8 +751,8 @@ static void Break(Core::System& system, u32 reason, u64 info1, u64 info2) { } system.GetReporter().SaveSvcBreakReport( - static_cast<u32>(break_reason.break_type.Value()), break_reason.signal_debugger, info1, - info2, has_dumped_buffer ? std::make_optional(debug_buffer) : std::nullopt); + static_cast<u32>(break_reason.break_type.Value()), break_reason.signal_debugger.As<bool>(), + info1, info2, has_dumped_buffer ? std::make_optional(debug_buffer) : std::nullopt); if (!break_reason.signal_debugger) { LOG_CRITICAL( @@ -933,7 +933,7 @@ static Result GetInfo(Core::System& system, u64* result, u64 info_id, Handle han return ResultSuccess; case GetInfoType::UserExceptionContextAddr: - *result = process->GetTLSRegionAddress(); + *result = process->GetProcessLocalRegionAddress(); return ResultSuccess; case GetInfoType::TotalPhysicalMemoryAvailableWithoutSystemResource: @@ -1888,7 +1888,7 @@ static void ExitProcess(Core::System& system) { auto* current_process = system.Kernel().CurrentProcess(); LOG_INFO(Kernel_SVC, "Process {} exiting", current_process->GetProcessID()); - ASSERT_MSG(current_process->GetStatus() == ProcessStatus::Running, + ASSERT_MSG(current_process->GetState() == KProcess::State::Running, "Process has already exited"); system.Exit(); @@ -2557,7 +2557,7 @@ static Result GetProcessInfo(Core::System& system, u64* out, Handle process_hand return ResultInvalidEnumValue; } - *out = static_cast<u64>(process->GetStatus()); + *out = static_cast<u64>(process->GetState()); return ResultSuccess; } diff --git a/src/core/hle/kernel/svc_common.h b/src/core/hle/kernel/svc_common.h index 95750c3eb..85506710e 100644 --- a/src/core/hle/kernel/svc_common.h +++ b/src/core/hle/kernel/svc_common.h @@ -14,8 +14,11 @@ namespace Kernel::Svc { using namespace Common::Literals; -constexpr s32 ArgumentHandleCountMax = 0x40; -constexpr u32 HandleWaitMask{1u << 30}; +constexpr inline s32 ArgumentHandleCountMax = 0x40; + +constexpr inline u32 HandleWaitMask = 1u << 30; + +constexpr inline s64 WaitInfinite = -1; constexpr inline std::size_t HeapSizeAlignment = 2_MiB; diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h index 79e15183a..abb9847fe 100644 --- a/src/core/hle/kernel/svc_types.h +++ b/src/core/hle/kernel/svc_types.h @@ -95,6 +95,19 @@ constexpr inline s32 IdealCoreNoUpdate = -3; constexpr inline s32 LowestThreadPriority = 63; constexpr inline s32 HighestThreadPriority = 0; +constexpr inline s32 SystemThreadPriorityHighest = 16; + +enum class ProcessState : u32 { + Created = 0, + CreatedAttached = 1, + Running = 2, + Crashed = 3, + RunningAttached = 4, + Terminating = 5, + Terminated = 6, + DebugBreak = 7, +}; + constexpr inline size_t ThreadLocalRegionSize = 0x200; } // namespace Kernel::Svc diff --git a/src/core/hle/result.h b/src/core/hle/result.h index d67e68bae..ef4b2d417 100644 --- a/src/core/hle/result.h +++ b/src/core/hle/result.h @@ -135,6 +135,14 @@ union Result { [[nodiscard]] constexpr bool IsFailure() const { return !IsSuccess(); } + + [[nodiscard]] constexpr u32 GetInnerValue() const { + return static_cast<u32>(module.Value()) | (description << module.bits); + } + + [[nodiscard]] constexpr bool Includes(Result result) const { + return GetInnerValue() == result.GetInnerValue(); + } }; static_assert(std::is_trivial_v<Result>); @@ -462,9 +470,6 @@ constexpr inline Result __TmpCurrentResultReference = ResultSuccess; #define R_UNLESS(expr, res) \ { \ if (!(expr)) { \ - if (res.IsError()) { \ - LOG_ERROR(Kernel, "Failed with result: {}", res.raw); \ - } \ R_THROW(res); \ } \ } diff --git a/src/core/hle/service/am/am.cpp b/src/core/hle/service/am/am.cpp index e55233054..8ea7fd760 100644 --- a/src/core/hle/service/am/am.cpp +++ b/src/core/hle/service/am/am.cpp @@ -299,7 +299,7 @@ ISelfController::ISelfController(Core::System& system_, NVFlinger::NVFlinger& nv {100, &ISelfController::SetAlbumImageTakenNotificationEnabled, "SetAlbumImageTakenNotificationEnabled"}, {110, nullptr, "SetApplicationAlbumUserData"}, {120, &ISelfController::SaveCurrentScreenshot, "SaveCurrentScreenshot"}, - {130, nullptr, "SetRecordVolumeMuted"}, + {130, &ISelfController::SetRecordVolumeMuted, "SetRecordVolumeMuted"}, {1000, nullptr, "GetDebugStorageChannel"}, }; // clang-format on @@ -597,6 +597,17 @@ void ISelfController::SaveCurrentScreenshot(Kernel::HLERequestContext& ctx) { rb.Push(ResultSuccess); } +void ISelfController::SetRecordVolumeMuted(Kernel::HLERequestContext& ctx) { + IPC::RequestParser rp{ctx}; + + const auto is_record_volume_muted = rp.Pop<bool>(); + + LOG_WARNING(Service_AM, "(STUBBED) called. is_record_volume_muted={}", is_record_volume_muted); + + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); +} + AppletMessageQueue::AppletMessageQueue(Core::System& system) : service_context{system, "AppletMessageQueue"} { on_new_message = service_context.CreateEvent("AMMessageQueue:OnMessageReceived"); diff --git a/src/core/hle/service/am/am.h b/src/core/hle/service/am/am.h index bb75c6281..a0fbfcfc5 100644 --- a/src/core/hle/service/am/am.h +++ b/src/core/hle/service/am/am.h @@ -182,6 +182,7 @@ private: void GetAccumulatedSuspendedTickChangedEvent(Kernel::HLERequestContext& ctx); void SetAlbumImageTakenNotificationEnabled(Kernel::HLERequestContext& ctx); void SaveCurrentScreenshot(Kernel::HLERequestContext& ctx); + void SetRecordVolumeMuted(Kernel::HLERequestContext& ctx); enum class ScreenshotPermission : u32 { Inherit = 0, diff --git a/src/core/hle/service/am/applets/applets.h b/src/core/hle/service/am/applets/applets.h index e78a57657..12c6a5b1a 100644 --- a/src/core/hle/service/am/applets/applets.h +++ b/src/core/hle/service/am/applets/applets.h @@ -164,7 +164,7 @@ protected: u32_le size; u32_le library_version; u32_le theme_color; - u8 play_startup_sound; + bool play_startup_sound; u64_le system_tick; }; static_assert(sizeof(CommonArguments) == 0x20, "CommonArguments has incorrect size."); diff --git a/src/core/hle/service/audio/audctl.cpp b/src/core/hle/service/audio/audctl.cpp index 4a2ae5f88..5abf22ba4 100644 --- a/src/core/hle/service/audio/audctl.cpp +++ b/src/core/hle/service/audio/audctl.cpp @@ -45,9 +45,25 @@ AudCtl::AudCtl(Core::System& system_) : ServiceFramework{system_, "audctl"} { {32, nullptr, "GetActiveOutputTarget"}, {33, nullptr, "GetTargetDeviceInfo"}, {34, nullptr, "AcquireTargetNotification"}, + {35, nullptr, "SetHearingProtectionSafeguardTimerRemainingTimeForDebug"}, + {36, nullptr, "GetHearingProtectionSafeguardTimerRemainingTimeForDebug"}, + {37, nullptr, "SetHearingProtectionSafeguardEnabled"}, + {38, nullptr, "IsHearingProtectionSafeguardEnabled"}, + {39, nullptr, "IsHearingProtectionSafeguardMonitoringOutputForDebug"}, + {40, nullptr, "GetSystemInformationForDebug"}, + {41, nullptr, "SetVolumeButtonLongPressTime"}, + {42, nullptr, "SetNativeVolumeForDebug"}, {10000, nullptr, "NotifyAudioOutputTargetForPlayReport"}, {10001, nullptr, "NotifyAudioOutputChannelCountForPlayReport"}, {10002, nullptr, "NotifyUnsupportedUsbOutputDeviceAttachedForPlayReport"}, + {10100, nullptr, "GetAudioVolumeDataForPlayReport"}, + {10101, nullptr, "BindAudioVolumeUpdateEventForPlayReport"}, + {10102, nullptr, "BindAudioOutputTargetUpdateEventForPlayReport"}, + {10103, nullptr, "GetAudioOutputTargetForPlayReport"}, + {10104, nullptr, "GetAudioOutputChannelCountForPlayReport"}, + {10105, nullptr, "BindAudioOutputChannelCountUpdateEventForPlayReport"}, + {10106, nullptr, "GetDefaultAudioOutputTargetForPlayReport"}, + {50000, nullptr, "SetAnalogInputBoostGainForPrototyping"}, }; // clang-format on diff --git a/src/core/hle/service/audio/audin_u.cpp b/src/core/hle/service/audio/audin_u.cpp index 48a9a73a0..608925dfc 100644 --- a/src/core/hle/service/audio/audin_u.cpp +++ b/src/core/hle/service/audio/audin_u.cpp @@ -17,7 +17,7 @@ using namespace AudioCore::AudioIn; class IAudioIn final : public ServiceFramework<IAudioIn> { public: explicit IAudioIn(Core::System& system_, Manager& manager, size_t session_id, - std::string& device_name, const AudioInParameter& in_params, u32 handle, + const std::string& device_name, const AudioInParameter& in_params, u32 handle, u64 applet_resource_user_id) : ServiceFramework{system_, "IAudioIn"}, service_context{system_, "IAudioIn"}, event{service_context.CreateEvent("AudioInEvent")}, diff --git a/src/core/hle/service/audio/audout_u.cpp b/src/core/hle/service/audio/audout_u.cpp index 49c092301..122290c6a 100644 --- a/src/core/hle/service/audio/audout_u.cpp +++ b/src/core/hle/service/audio/audout_u.cpp @@ -24,7 +24,7 @@ using namespace AudioCore::AudioOut; class IAudioOut final : public ServiceFramework<IAudioOut> { public: explicit IAudioOut(Core::System& system_, AudioCore::AudioOut::Manager& manager, - size_t session_id, std::string& device_name, + size_t session_id, const std::string& device_name, const AudioOutParameter& in_params, u32 handle, u64 applet_resource_user_id) : ServiceFramework{system_, "IAudioOut", ServiceThreadType::CreateNew}, service_context{system_, "IAudioOut"}, event{service_context.CreateEvent( diff --git a/src/core/hle/service/audio/audren_u.cpp b/src/core/hle/service/audio/audren_u.cpp index 60c30cd5b..13423dca6 100644 --- a/src/core/hle/service/audio/audren_u.cpp +++ b/src/core/hle/service/audio/audren_u.cpp @@ -52,6 +52,8 @@ public: {9, &IAudioRenderer::GetRenderingTimeLimit, "GetRenderingTimeLimit"}, {10, &IAudioRenderer::RequestUpdate, "RequestUpdateAuto"}, {11, nullptr, "ExecuteAudioRendererRendering"}, + {12, &IAudioRenderer::SetVoiceDropParameter, "SetVoiceDropParameter"}, + {13, &IAudioRenderer::GetVoiceDropParameter, "GetVoiceDropParameter"}, }; // clang-format on RegisterHandlers(functions); @@ -205,6 +207,30 @@ private: LOG_DEBUG(Service_Audio, "called"); } + void SetVoiceDropParameter(Kernel::HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + + IPC::RequestParser rp{ctx}; + auto voice_drop_param{rp.Pop<f32>()}; + + auto& system_ = impl->GetSystem(); + system_.SetVoiceDropParameter(voice_drop_param); + + IPC::ResponseBuilder rb{ctx, 2}; + rb.Push(ResultSuccess); + } + + void GetVoiceDropParameter(Kernel::HLERequestContext& ctx) { + LOG_DEBUG(Service_Audio, "called"); + + auto& system_ = impl->GetSystem(); + auto voice_drop_param{system_.GetVoiceDropParameter()}; + + IPC::ResponseBuilder rb{ctx, 3}; + rb.Push(ResultSuccess); + rb.Push(voice_drop_param); + } + KernelHelpers::ServiceContext service_context; Kernel::KEvent* rendered_event; Manager& manager; diff --git a/src/core/hle/service/hid/controllers/npad.cpp b/src/core/hle/service/hid/controllers/npad.cpp index 98e4f2af7..2f871de31 100644 --- a/src/core/hle/service/hid/controllers/npad.cpp +++ b/src/core/hle/service/hid/controllers/npad.cpp @@ -745,8 +745,9 @@ void Controller_NPad::SetSupportedNpadIdTypes(u8* data, std::size_t length) { } void Controller_NPad::GetSupportedNpadIdTypes(u32* data, std::size_t max_length) { - ASSERT(max_length < supported_npad_id_types.size()); - std::memcpy(data, supported_npad_id_types.data(), supported_npad_id_types.size()); + const auto copy_amount = supported_npad_id_types.size() * sizeof(u32); + ASSERT(max_length <= copy_amount); + std::memcpy(data, supported_npad_id_types.data(), copy_amount); } std::size_t Controller_NPad::GetSupportedNpadIdTypesSize() const { @@ -867,7 +868,7 @@ bool Controller_NPad::VibrateControllerAtIndex(Core::HID::NpadIdType npad_id, return false; } - if (!controller.device->IsVibrationEnabled()) { + if (!controller.device->IsVibrationEnabled(device_index)) { if (controller.vibration[device_index].latest_vibration_value.low_amplitude != 0.0f || controller.vibration[device_index].latest_vibration_value.high_amplitude != 0.0f) { // Send an empty vibration to stop any vibrations. @@ -1000,7 +1001,7 @@ void Controller_NPad::InitializeVibrationDeviceAtIndex(Core::HID::NpadIdType npa } controller.vibration[device_index].device_mounted = - controller.device->TestVibration(device_index); + controller.device->IsVibrationEnabled(device_index); } void Controller_NPad::SetPermitVibrationSession(bool permit_vibration_session) { @@ -1501,25 +1502,25 @@ bool Controller_NPad::IsControllerSupported(Core::HID::NpadStyleIndex controller Core::HID::NpadStyleTag style = GetSupportedStyleSet(); switch (controller) { case Core::HID::NpadStyleIndex::ProController: - return style.fullkey; + return style.fullkey.As<bool>(); case Core::HID::NpadStyleIndex::JoyconDual: - return style.joycon_dual; + return style.joycon_dual.As<bool>(); case Core::HID::NpadStyleIndex::JoyconLeft: - return style.joycon_left; + return style.joycon_left.As<bool>(); case Core::HID::NpadStyleIndex::JoyconRight: - return style.joycon_right; + return style.joycon_right.As<bool>(); case Core::HID::NpadStyleIndex::GameCube: - return style.gamecube; + return style.gamecube.As<bool>(); case Core::HID::NpadStyleIndex::Pokeball: - return style.palma; + return style.palma.As<bool>(); case Core::HID::NpadStyleIndex::NES: - return style.lark; + return style.lark.As<bool>(); case Core::HID::NpadStyleIndex::SNES: - return style.lucia; + return style.lucia.As<bool>(); case Core::HID::NpadStyleIndex::N64: - return style.lagoon; + return style.lagoon.As<bool>(); case Core::HID::NpadStyleIndex::SegaGenesis: - return style.lager; + return style.lager.As<bool>(); default: return false; } diff --git a/src/core/hle/service/ldr/ldr.cpp b/src/core/hle/service/ldr/ldr.cpp index becd6d1b9..652441bc2 100644 --- a/src/core/hle/service/ldr/ldr.cpp +++ b/src/core/hle/service/ldr/ldr.cpp @@ -290,7 +290,7 @@ public: const std::size_t padding_size{page_table.GetNumGuardPages() * Kernel::PageSize}; const auto start_info{page_table.QueryInfo(start - 1)}; - if (start_info.state != Kernel::KMemoryState::Free) { + if (start_info.GetState() != Kernel::KMemoryState::Free) { return {}; } @@ -300,7 +300,7 @@ public: const auto end_info{page_table.QueryInfo(start + size)}; - if (end_info.state != Kernel::KMemoryState::Free) { + if (end_info.GetState() != Kernel::KMemoryState::Free) { return {}; } diff --git a/src/core/hle/service/nfp/amiibo_crypto.cpp b/src/core/hle/service/nfp/amiibo_crypto.cpp index c32a6816b..167e29572 100644 --- a/src/core/hle/service/nfp/amiibo_crypto.cpp +++ b/src/core/hle/service/nfp/amiibo_crypto.cpp @@ -9,6 +9,7 @@ #include <mbedtls/hmac_drbg.h> #include "common/fs/file.h" +#include "common/fs/fs.h" #include "common/fs/path_util.h" #include "common/logging/log.h" #include "core/hle/service/mii/mii_manager.h" @@ -279,7 +280,7 @@ bool LoadKeys(InternalKey& locked_secret, InternalKey& unfixed_info) { Common::FS::FileType::BinaryFile}; if (!keys_file.IsOpen()) { - LOG_ERROR(Service_NFP, "No keys detected"); + LOG_ERROR(Service_NFP, "Failed to open key file"); return false; } @@ -295,6 +296,11 @@ bool LoadKeys(InternalKey& locked_secret, InternalKey& unfixed_info) { return true; } +bool IsKeyAvailable() { + const auto yuzu_keys_dir = Common::FS::GetYuzuPath(Common::FS::YuzuPath::KeysDir); + return Common::FS::Exists(yuzu_keys_dir / "key_retail.bin"); +} + bool DecodeAmiibo(const EncryptedNTAG215File& encrypted_tag_data, NTAG215File& tag_data) { InternalKey locked_secret{}; InternalKey unfixed_info{}; diff --git a/src/core/hle/service/nfp/amiibo_crypto.h b/src/core/hle/service/nfp/amiibo_crypto.h index 0175ced91..1fa61174e 100644 --- a/src/core/hle/service/nfp/amiibo_crypto.h +++ b/src/core/hle/service/nfp/amiibo_crypto.h @@ -91,6 +91,9 @@ void Cipher(const DerivedKeys& keys, const NTAG215File& in_data, NTAG215File& ou /// Loads both amiibo keys from key_retail.bin bool LoadKeys(InternalKey& locked_secret, InternalKey& unfixed_info); +/// Returns true if key_retail.bin exist +bool IsKeyAvailable(); + /// Decodes encripted amiibo data returns true if output is valid bool DecodeAmiibo(const EncryptedNTAG215File& encrypted_tag_data, NTAG215File& tag_data); diff --git a/src/core/hle/service/nfp/nfp_device.cpp b/src/core/hle/service/nfp/nfp_device.cpp index 76f8a267a..b19672560 100644 --- a/src/core/hle/service/nfp/nfp_device.cpp +++ b/src/core/hle/service/nfp/nfp_device.cpp @@ -17,6 +17,7 @@ #include "core/hle/ipc_helpers.h" #include "core/hle/kernel/k_event.h" #include "core/hle/service/mii/mii_manager.h" +#include "core/hle/service/mii/types.h" #include "core/hle/service/nfp/amiibo_crypto.h" #include "core/hle/service/nfp/nfp.h" #include "core/hle/service/nfp/nfp_device.h" @@ -233,6 +234,14 @@ Result NfpDevice::Mount(MountTarget mount_target_) { return NotAnAmiibo; } + // Mark amiibos as read only when keys are missing + if (!AmiiboCrypto::IsKeyAvailable()) { + LOG_ERROR(Service_NFP, "No keys detected"); + device_state = DeviceState::TagMounted; + mount_target = MountTarget::Rom; + return ResultSuccess; + } + if (!AmiiboCrypto::DecodeAmiibo(encrypted_tag_data, tag_data)) { LOG_ERROR(Service_NFP, "Can't decode amiibo {}", device_state); return CorruptedData; diff --git a/src/core/hle/service/nfp/nfp_device.h b/src/core/hle/service/nfp/nfp_device.h index a5b72cf19..76d0e9ae4 100644 --- a/src/core/hle/service/nfp/nfp_device.h +++ b/src/core/hle/service/nfp/nfp_device.h @@ -8,7 +8,6 @@ #include "common/common_funcs.h" #include "core/hle/service/kernel_helpers.h" -#include "core/hle/service/mii/types.h" #include "core/hle/service/nfp/nfp_types.h" #include "core/hle/service/service.h" diff --git a/src/core/hle/service/nfp/nfp_types.h b/src/core/hle/service/nfp/nfp_types.h index c09f9ddb6..63d5917cb 100644 --- a/src/core/hle/service/nfp/nfp_types.h +++ b/src/core/hle/service/nfp/nfp_types.h @@ -17,11 +17,6 @@ enum class ServiceType : u32 { System, }; -enum class State : u32 { - NonInitialized, - Initialized, -}; - enum class DeviceState : u32 { Initialized, SearchingForTag, diff --git a/src/core/hle/service/nfp/nfp_user.cpp b/src/core/hle/service/nfp/nfp_user.cpp index 4ed53b534..33e2ef518 100644 --- a/src/core/hle/service/nfp/nfp_user.cpp +++ b/src/core/hle/service/nfp/nfp_user.cpp @@ -6,12 +6,9 @@ #include "common/logging/log.h" #include "core/core.h" -#include "core/hid/emulated_controller.h" -#include "core/hid/hid_core.h" #include "core/hid/hid_types.h" #include "core/hle/ipc_helpers.h" #include "core/hle/kernel/k_event.h" -#include "core/hle/service/mii/mii_manager.h" #include "core/hle/service/nfp/nfp_device.h" #include "core/hle/service/nfp/nfp_result.h" #include "core/hle/service/nfp/nfp_user.h" diff --git a/src/core/hle/service/nfp/nfp_user.h b/src/core/hle/service/nfp/nfp_user.h index 68c60ae82..47aff3695 100644 --- a/src/core/hle/service/nfp/nfp_user.h +++ b/src/core/hle/service/nfp/nfp_user.h @@ -4,8 +4,7 @@ #pragma once #include "core/hle/service/kernel_helpers.h" -#include "core/hle/service/nfp/nfp.h" -#include "core/hle/service/nfp/nfp_types.h" +#include "core/hle/service/service.h" namespace Service::NFP { class NfpDevice; @@ -15,6 +14,11 @@ public: explicit IUser(Core::System& system_); private: + enum class State : u32 { + NonInitialized, + Initialized, + }; + void Initialize(Kernel::HLERequestContext& ctx); void Finalize(Kernel::HLERequestContext& ctx); void ListDevices(Kernel::HLERequestContext& ctx); diff --git a/src/core/hle/service/nvdrv/core/nvmap.cpp b/src/core/hle/service/nvdrv/core/nvmap.cpp index fbd8a74a5..a51ca5444 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.cpp +++ b/src/core/hle/service/nvdrv/core/nvmap.cpp @@ -255,15 +255,16 @@ std::optional<NvMap::FreeInfo> NvMap::FreeHandle(Handle::Id handle, bool interna .address = handle_description->address, .size = handle_description->size, .was_uncached = handle_description->flags.map_uncached.Value() != 0, + .can_unlock = true, }; } else { return std::nullopt; } - // Handle hasn't been freed from memory, set address to 0 to mark that the handle wasn't freed + // If the handle hasn't been freed from memory, mark that if (!hWeak.expired()) { LOG_DEBUG(Service_NVDRV, "nvmap handle: {} wasn't freed as it is still in use", handle); - freeInfo.address = 0; + freeInfo.can_unlock = false; } return freeInfo; diff --git a/src/core/hle/service/nvdrv/core/nvmap.h b/src/core/hle/service/nvdrv/core/nvmap.h index b9dd3801f..a8e573890 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.h +++ b/src/core/hle/service/nvdrv/core/nvmap.h @@ -105,6 +105,7 @@ public: u64 address; //!< Address the handle referred to before deletion u64 size; //!< Page-aligned handle size bool was_uncached; //!< If the handle was allocated as uncached + bool can_unlock; //!< If the address region is ready to be unlocked }; explicit NvMap(Tegra::Host1x::Host1x& host1x); diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 6411dbf43..b635e6ed1 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -311,7 +311,8 @@ NvResult nvhost_as_gpu::Remap(const std::vector<u8>& input, std::vector<u8>& out handle->address + (static_cast<u64>(entry.handle_offset_big_pages) << vm.big_page_size_bits))}; - gmmu->Map(virtual_address, cpu_address, size, use_big_pages); + gmmu->Map(virtual_address, cpu_address, size, static_cast<Tegra::PTEKind>(entry.kind), + use_big_pages); } } @@ -350,7 +351,8 @@ NvResult nvhost_as_gpu::MapBufferEx(const std::vector<u8>& input, std::vector<u8 u64 gpu_address{static_cast<u64>(params.offset + params.buffer_offset)}; VAddr cpu_address{mapping->ptr + params.buffer_offset}; - gmmu->Map(gpu_address, cpu_address, params.mapping_size, mapping->big_page); + gmmu->Map(gpu_address, cpu_address, params.mapping_size, + static_cast<Tegra::PTEKind>(params.kind), mapping->big_page); return NvResult::Success; } catch (const std::out_of_range&) { @@ -389,7 +391,8 @@ NvResult nvhost_as_gpu::MapBufferEx(const std::vector<u8>& input, std::vector<u8 } const bool use_big_pages = alloc->second.big_pages && big_page; - gmmu->Map(params.offset, cpu_address, size, use_big_pages); + gmmu->Map(params.offset, cpu_address, size, static_cast<Tegra::PTEKind>(params.kind), + use_big_pages); auto mapping{std::make_shared<Mapping>(cpu_address, params.offset, size, true, use_big_pages, alloc->second.sparse)}; @@ -409,7 +412,8 @@ NvResult nvhost_as_gpu::MapBufferEx(const std::vector<u8>& input, std::vector<u8 return NvResult::InsufficientMemory; } - gmmu->Map(params.offset, cpu_address, Common::AlignUp(size, page_size), big_page); + gmmu->Map(params.offset, cpu_address, Common::AlignUp(size, page_size), + static_cast<Tegra::PTEKind>(params.kind), big_page); auto mapping{ std::make_shared<Mapping>(cpu_address, params.offset, size, false, big_page, false)}; diff --git a/src/core/hle/service/nvdrv/devices/nvmap.cpp b/src/core/hle/service/nvdrv/devices/nvmap.cpp index ddf273b5e..44388655d 100644 --- a/src/core/hle/service/nvdrv/devices/nvmap.cpp +++ b/src/core/hle/service/nvdrv/devices/nvmap.cpp @@ -128,7 +128,8 @@ NvResult nvmap::IocAlloc(const std::vector<u8>& input, std::vector<u8>& output) } ASSERT(system.CurrentProcess() ->PageTable() - .LockForDeviceAddressSpace(handle_description->address, handle_description->size) + .LockForMapDeviceAddressSpace(handle_description->address, handle_description->size, + Kernel::KMemoryPermission::None, true) .IsSuccess()); std::memcpy(output.data(), ¶ms, sizeof(params)); return result; @@ -250,10 +251,12 @@ NvResult nvmap::IocFree(const std::vector<u8>& input, std::vector<u8>& output) { } if (auto freeInfo{file.FreeHandle(params.handle, false)}) { - ASSERT(system.CurrentProcess() - ->PageTable() - .UnlockForDeviceAddressSpace(freeInfo->address, freeInfo->size) - .IsSuccess()); + if (freeInfo->can_unlock) { + ASSERT(system.CurrentProcess() + ->PageTable() + .UnlockForDeviceAddressSpace(freeInfo->address, freeInfo->size) + .IsSuccess()); + } params.address = freeInfo->address; params.size = static_cast<u32>(freeInfo->size); params.flags.raw = 0; diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index 9d9924395..9f4c7c99a 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -53,7 +53,7 @@ void InstallInterfaces(SM::ServiceManager& service_manager, NVFlinger::NVFlinger } Module::Module(Core::System& system) - : service_context{system, "nvdrv"}, events_interface{*this}, container{system.Host1x()} { + : container{system.Host1x()}, service_context{system, "nvdrv"}, events_interface{*this} { builders["/dev/nvhost-as-gpu"] = [this, &system](DeviceFD fd) { std::shared_ptr<Devices::nvdevice> device = std::make_shared<Devices::nvhost_as_gpu>(system, *this, container); diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h index 146d046a9..f3c81bd88 100644 --- a/src/core/hle/service/nvdrv/nvdrv.h +++ b/src/core/hle/service/nvdrv/nvdrv.h @@ -97,6 +97,9 @@ private: friend class EventInterface; friend class Service::NVFlinger::NVFlinger; + /// Manages syncpoints on the host + NvCore::Container container; + /// Id to use for the next open file descriptor. DeviceFD next_fd = 1; @@ -108,9 +111,6 @@ private: EventInterface events_interface; - /// Manages syncpoints on the host - NvCore::Container container; - std::unordered_map<std::string, std::function<FilesContainerType::iterator(DeviceFD)>> builders; }; diff --git a/src/core/hle/service/nvflinger/buffer_queue_producer.cpp b/src/core/hle/service/nvflinger/buffer_queue_producer.cpp index 77ddbb6ef..41ba44b21 100644 --- a/src/core/hle/service/nvflinger/buffer_queue_producer.cpp +++ b/src/core/hle/service/nvflinger/buffer_queue_producer.cpp @@ -742,6 +742,13 @@ Status BufferQueueProducer::Disconnect(NativeWindowApi api) { return Status::NoError; } + // HACK: We are not Android. Remove handle for items in queue, and clear queue. + // Allows synchronous destruction of nvmap handles. + for (auto& item : core->queue) { + nvmap.FreeHandle(item.graphic_buffer->BufferId(), true); + } + core->queue.clear(); + switch (api) { case NativeWindowApi::Egl: case NativeWindowApi::Cpu: diff --git a/src/core/hle/service/nvflinger/nvflinger.cpp b/src/core/hle/service/nvflinger/nvflinger.cpp index aa14d2cbc..c3af12c90 100644 --- a/src/core/hle/service/nvflinger/nvflinger.cpp +++ b/src/core/hle/service/nvflinger/nvflinger.cpp @@ -102,15 +102,19 @@ NVFlinger::~NVFlinger() { system.CoreTiming().UnscheduleEvent(single_composition_event, {}); } + ShutdownLayers(); + + if (nvdrv) { + nvdrv->Close(disp_fd); + } +} + +void NVFlinger::ShutdownLayers() { for (auto& display : displays) { for (size_t layer = 0; layer < display.GetNumLayers(); ++layer) { display.GetLayer(layer).Core().NotifyShutdown(); } } - - if (nvdrv) { - nvdrv->Close(disp_fd); - } } void NVFlinger::SetNVDrvInstance(std::shared_ptr<Nvidia::Module> instance) { @@ -134,6 +138,19 @@ std::optional<u64> NVFlinger::OpenDisplay(std::string_view name) { return itr->GetID(); } +bool NVFlinger::CloseDisplay(u64 display_id) { + const auto lock_guard = Lock(); + auto* const display = FindDisplay(display_id); + + if (display == nullptr) { + return false; + } + + display->Reset(); + + return true; +} + std::optional<u64> NVFlinger::CreateLayer(u64 display_id) { const auto lock_guard = Lock(); auto* const display = FindDisplay(display_id); diff --git a/src/core/hle/service/nvflinger/nvflinger.h b/src/core/hle/service/nvflinger/nvflinger.h index 99509bc5b..460bef976 100644 --- a/src/core/hle/service/nvflinger/nvflinger.h +++ b/src/core/hle/service/nvflinger/nvflinger.h @@ -48,6 +48,8 @@ public: explicit NVFlinger(Core::System& system_, HosBinderDriverServer& hos_binder_driver_server_); ~NVFlinger(); + void ShutdownLayers(); + /// Sets the NVDrv module instance to use to send buffers to the GPU. void SetNVDrvInstance(std::shared_ptr<Nvidia::Module> instance); @@ -56,6 +58,11 @@ public: /// If an invalid display name is provided, then an empty optional is returned. [[nodiscard]] std::optional<u64> OpenDisplay(std::string_view name); + /// Closes the specified display by its ID. + /// + /// Returns false if an invalid display ID is provided. + [[nodiscard]] bool CloseDisplay(u64 display_id); + /// Creates a layer on the specified display and returns the layer ID. /// /// If an invalid display ID is specified, then an empty optional is returned. diff --git a/src/core/hle/service/service.cpp b/src/core/hle/service/service.cpp index dadaf897f..5db6588e4 100644 --- a/src/core/hle/service/service.cpp +++ b/src/core/hle/service/service.cpp @@ -303,4 +303,8 @@ Services::Services(std::shared_ptr<SM::ServiceManager>& sm, Core::System& system Services::~Services() = default; +void Services::KillNVNFlinger() { + nv_flinger->ShutdownLayers(); +} + } // namespace Service diff --git a/src/core/hle/service/service.h b/src/core/hle/service/service.h index 5bf197c51..ec9deeee4 100644 --- a/src/core/hle/service/service.h +++ b/src/core/hle/service/service.h @@ -238,6 +238,8 @@ public: explicit Services(std::shared_ptr<SM::ServiceManager>& sm, Core::System& system); ~Services(); + void KillNVNFlinger(); + private: std::unique_ptr<NVFlinger::HosBinderDriverServer> hos_binder_driver_server; std::unique_ptr<NVFlinger::NVFlinger> nv_flinger; diff --git a/src/core/hle/service/sm/sm.cpp b/src/core/hle/service/sm/sm.cpp index 48e70f93c..e2b8d8720 100644 --- a/src/core/hle/service/sm/sm.cpp +++ b/src/core/hle/service/sm/sm.cpp @@ -80,7 +80,6 @@ ResultVal<Kernel::KPort*> ServiceManager::GetServicePort(const std::string& name } auto* port = Kernel::KPort::Create(kernel); - SCOPE_EXIT({ port->Close(); }); port->Initialize(ServerSessionCountMax, false, name); auto handler = it->second; diff --git a/src/core/hle/service/sm/sm_controller.cpp b/src/core/hle/service/sm/sm_controller.cpp index 2a4bd64ab..273f79568 100644 --- a/src/core/hle/service/sm/sm_controller.cpp +++ b/src/core/hle/service/sm/sm_controller.cpp @@ -15,9 +15,10 @@ namespace Service::SM { void Controller::ConvertCurrentObjectToDomain(Kernel::HLERequestContext& ctx) { - ASSERT_MSG(!ctx.Session()->IsDomain(), "Session is already a domain"); + ASSERT_MSG(!ctx.Session()->GetSessionRequestManager()->IsDomain(), + "Session is already a domain"); LOG_DEBUG(Service, "called, server_session={}", ctx.Session()->GetId()); - ctx.Session()->ConvertToDomain(); + ctx.Session()->GetSessionRequestManager()->ConvertToDomainOnRequestEnd(); IPC::ResponseBuilder rb{ctx, 3}; rb.Push(ResultSuccess); diff --git a/src/core/hle/service/vi/display/vi_display.h b/src/core/hle/service/vi/display/vi_display.h index 33d5f398c..0b65a65da 100644 --- a/src/core/hle/service/vi/display/vi_display.h +++ b/src/core/hle/service/vi/display/vi_display.h @@ -106,6 +106,12 @@ public: /// void CloseLayer(u64 layer_id); + /// Resets the display for a new connection. + void Reset() { + layers.clear(); + got_vsync_event = false; + } + /// Attempts to find a layer with the given ID. /// /// @param layer_id The layer ID. diff --git a/src/core/hle/service/vi/vi.cpp b/src/core/hle/service/vi/vi.cpp index 9c917cacf..bb283e74e 100644 --- a/src/core/hle/service/vi/vi.cpp +++ b/src/core/hle/service/vi/vi.cpp @@ -324,10 +324,10 @@ private: IPC::RequestParser rp{ctx}; const u64 display = rp.Pop<u64>(); - LOG_WARNING(Service_VI, "(STUBBED) called. display=0x{:016X}", display); + const Result rc = nv_flinger.CloseDisplay(display) ? ResultSuccess : ResultUnknown; IPC::ResponseBuilder rb{ctx, 2}; - rb.Push(ResultSuccess); + rb.Push(rc); } void CreateManagedLayer(Kernel::HLERequestContext& ctx) { @@ -508,10 +508,10 @@ private: IPC::RequestParser rp{ctx}; const u64 display_id = rp.Pop<u64>(); - LOG_WARNING(Service_VI, "(STUBBED) called. display_id=0x{:016X}", display_id); + const Result rc = nv_flinger.CloseDisplay(display_id) ? ResultSuccess : ResultUnknown; IPC::ResponseBuilder rb{ctx, 2}; - rb.Push(ResultSuccess); + rb.Push(rc); } // This literally does nothing internally in the actual service itself, diff --git a/src/core/hle/service/vi/vi_results.h b/src/core/hle/service/vi/vi_results.h index a46c247d2..22bac799f 100644 --- a/src/core/hle/service/vi/vi_results.h +++ b/src/core/hle/service/vi/vi_results.h @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#pragma once + #include "core/hle/result.h" namespace Service::VI { diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 2ac792566..3ca80c8ff 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -65,7 +65,7 @@ struct Memory::Impl { return {}; } - return system.DeviceMemory().GetPointer(paddr) + vaddr; + return system.DeviceMemory().GetPointer<u8>(paddr) + vaddr; } [[nodiscard]] u8* GetPointerFromDebugMemory(VAddr vaddr) const { @@ -75,7 +75,7 @@ struct Memory::Impl { return {}; } - return system.DeviceMemory().GetPointer(paddr) + vaddr; + return system.DeviceMemory().GetPointer<u8>(paddr) + vaddr; } u8 Read8(const VAddr addr) { @@ -233,18 +233,17 @@ struct Memory::Impl { current_vaddr, src_addr, size); std::memset(dest_buffer, 0, copy_amount); }, - [&dest_buffer](const std::size_t copy_amount, const u8* const src_ptr) { + [&](const std::size_t copy_amount, const u8* const src_ptr) { std::memcpy(dest_buffer, src_ptr, copy_amount); }, - [&system = system, &dest_buffer](const VAddr current_vaddr, - const std::size_t copy_amount, - const u8* const host_ptr) { + [&](const VAddr current_vaddr, const std::size_t copy_amount, + const u8* const host_ptr) { if constexpr (!UNSAFE) { system.GPU().FlushRegion(current_vaddr, copy_amount); } std::memcpy(dest_buffer, host_ptr, copy_amount); }, - [&dest_buffer](const std::size_t copy_amount) { + [&](const std::size_t copy_amount) { dest_buffer = static_cast<u8*>(dest_buffer) + copy_amount; }); } @@ -267,17 +266,16 @@ struct Memory::Impl { "Unmapped WriteBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", current_vaddr, dest_addr, size); }, - [&src_buffer](const std::size_t copy_amount, u8* const dest_ptr) { + [&](const std::size_t copy_amount, u8* const dest_ptr) { std::memcpy(dest_ptr, src_buffer, copy_amount); }, - [&system = system, &src_buffer](const VAddr current_vaddr, - const std::size_t copy_amount, u8* const host_ptr) { + [&](const VAddr current_vaddr, const std::size_t copy_amount, u8* const host_ptr) { if constexpr (!UNSAFE) { system.GPU().InvalidateRegion(current_vaddr, copy_amount); } std::memcpy(host_ptr, src_buffer, copy_amount); }, - [&src_buffer](const std::size_t copy_amount) { + [&](const std::size_t copy_amount) { src_buffer = static_cast<const u8*>(src_buffer) + copy_amount; }); } @@ -301,8 +299,7 @@ struct Memory::Impl { [](const std::size_t copy_amount, u8* const dest_ptr) { std::memset(dest_ptr, 0, copy_amount); }, - [&system = system](const VAddr current_vaddr, const std::size_t copy_amount, - u8* const host_ptr) { + [&](const VAddr current_vaddr, const std::size_t copy_amount, u8* const host_ptr) { system.GPU().InvalidateRegion(current_vaddr, copy_amount); std::memset(host_ptr, 0, copy_amount); }, @@ -313,22 +310,20 @@ struct Memory::Impl { const std::size_t size) { WalkBlock( process, dest_addr, size, - [this, &process, &dest_addr, &src_addr, size](const std::size_t copy_amount, - const VAddr current_vaddr) { + [&](const std::size_t copy_amount, const VAddr current_vaddr) { LOG_ERROR(HW_Memory, "Unmapped CopyBlock @ 0x{:016X} (start address = 0x{:016X}, size = {})", current_vaddr, src_addr, size); ZeroBlock(process, dest_addr, copy_amount); }, - [this, &process, &dest_addr](const std::size_t copy_amount, const u8* const src_ptr) { + [&](const std::size_t copy_amount, const u8* const src_ptr) { WriteBlockImpl<false>(process, dest_addr, src_ptr, copy_amount); }, - [this, &system = system, &process, &dest_addr]( - const VAddr current_vaddr, const std::size_t copy_amount, u8* const host_ptr) { + [&](const VAddr current_vaddr, const std::size_t copy_amount, u8* const host_ptr) { system.GPU().FlushRegion(current_vaddr, copy_amount); WriteBlockImpl<false>(process, dest_addr, host_ptr, copy_amount); }, - [&dest_addr, &src_addr](const std::size_t copy_amount) { + [&](const std::size_t copy_amount) { dest_addr += static_cast<VAddr>(copy_amount); src_addr += static_cast<VAddr>(copy_amount); }); @@ -499,7 +494,7 @@ struct Memory::Impl { } else { while (base != end) { page_table.pointers[base].Store( - system.DeviceMemory().GetPointer(target) - (base << YUZU_PAGEBITS), type); + system.DeviceMemory().GetPointer<u8>(target) - (base << YUZU_PAGEBITS), type); page_table.backing_addr[base] = target - (base << YUZU_PAGEBITS); ASSERT_MSG(page_table.pointers[base].Pointer(), @@ -575,7 +570,7 @@ struct Memory::Impl { [vaddr]() { LOG_ERROR(HW_Memory, "Unmapped Read{} @ 0x{:016X}", sizeof(T) * 8, vaddr); }, - [&system = system, vaddr]() { system.GPU().FlushRegion(vaddr, sizeof(T)); }); + [&]() { system.GPU().FlushRegion(vaddr, sizeof(T)); }); if (ptr) { std::memcpy(&result, ptr, sizeof(T)); } @@ -599,7 +594,7 @@ struct Memory::Impl { LOG_ERROR(HW_Memory, "Unmapped Write{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8, vaddr, static_cast<u64>(data)); }, - [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); }); + [&]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); }); if (ptr) { std::memcpy(ptr, &data, sizeof(T)); } @@ -613,7 +608,7 @@ struct Memory::Impl { LOG_ERROR(HW_Memory, "Unmapped WriteExclusive{} @ 0x{:016X} = 0x{:016X}", sizeof(T) * 8, vaddr, static_cast<u64>(data)); }, - [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); }); + [&]() { system.GPU().InvalidateRegion(vaddr, sizeof(T)); }); if (ptr) { const auto volatile_pointer = reinterpret_cast<volatile T*>(ptr); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); @@ -628,7 +623,7 @@ struct Memory::Impl { LOG_ERROR(HW_Memory, "Unmapped WriteExclusive128 @ 0x{:016X} = 0x{:016X}{:016X}", vaddr, static_cast<u64>(data[1]), static_cast<u64>(data[0])); }, - [&system = system, vaddr]() { system.GPU().InvalidateRegion(vaddr, sizeof(u128)); }); + [&]() { system.GPU().InvalidateRegion(vaddr, sizeof(u128)); }); if (ptr) { const auto volatile_pointer = reinterpret_cast<volatile u64*>(ptr); return Common::AtomicCompareAndSwap(volatile_pointer, data, expected); diff --git a/src/input_common/CMakeLists.txt b/src/input_common/CMakeLists.txt index 2cf9eb97f..cc6f0ffc0 100644 --- a/src/input_common/CMakeLists.txt +++ b/src/input_common/CMakeLists.txt @@ -39,21 +39,14 @@ add_library(input_common STATIC if (MSVC) target_compile_options(input_common PRIVATE /W4 - /WX /we4242 # 'identifier': conversion from 'type1' to 'type2', possible loss of data - /we4244 # 'conversion': conversion from 'type1' to 'type2', possible loss of data - /we4245 # 'conversion': conversion from 'type1' to 'type2', signed/unsigned mismatch /we4254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data + /we4800 # Implicit conversion from 'type' to bool. Possible information loss ) else() target_compile_options(input_common PRIVATE - -Werror -Werror=conversion - -Werror=ignored-qualifiers - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable> - -Werror=unused-variable ) endif() diff --git a/src/input_common/drivers/gc_adapter.cpp b/src/input_common/drivers/gc_adapter.cpp index f4dd24e7d..826fa2109 100644 --- a/src/input_common/drivers/gc_adapter.cpp +++ b/src/input_common/drivers/gc_adapter.cpp @@ -324,7 +324,7 @@ bool GCAdapter::GetGCEndpoint(libusb_device* device) { return true; } -Common::Input::VibrationError GCAdapter::SetRumble( +Common::Input::VibrationError GCAdapter::SetVibration( const PadIdentifier& identifier, const Common::Input::VibrationStatus& vibration) { const auto mean_amplitude = (vibration.low_amplitude + vibration.high_amplitude) * 0.5f; const auto processed_amplitude = @@ -338,6 +338,10 @@ Common::Input::VibrationError GCAdapter::SetRumble( return Common::Input::VibrationError::None; } +bool GCAdapter::IsVibrationEnabled([[maybe_unused]] const PadIdentifier& identifier) { + return rumble_enabled; +} + void GCAdapter::UpdateVibrations() { // Use 8 states to keep the switching between on/off fast enough for // a human to feel different vibration strenght diff --git a/src/input_common/drivers/gc_adapter.h b/src/input_common/drivers/gc_adapter.h index 8682da847..7f81767f7 100644 --- a/src/input_common/drivers/gc_adapter.h +++ b/src/input_common/drivers/gc_adapter.h @@ -25,9 +25,11 @@ public: explicit GCAdapter(std::string input_engine_); ~GCAdapter() override; - Common::Input::VibrationError SetRumble( + Common::Input::VibrationError SetVibration( const PadIdentifier& identifier, const Common::Input::VibrationStatus& vibration) override; + bool IsVibrationEnabled(const PadIdentifier& identifier) override; + /// Used for automapping features std::vector<Common::ParamPackage> GetInputDevices() const override; ButtonMapping GetButtonMappingForDevice(const Common::ParamPackage& params) override; diff --git a/src/input_common/drivers/sdl_driver.cpp b/src/input_common/drivers/sdl_driver.cpp index b72e4b397..45ce588f0 100644 --- a/src/input_common/drivers/sdl_driver.cpp +++ b/src/input_common/drivers/sdl_driver.cpp @@ -40,8 +40,8 @@ public: void EnableMotion() { if (sdl_controller) { SDL_GameController* controller = sdl_controller.get(); - has_accel = SDL_GameControllerHasSensor(controller, SDL_SENSOR_ACCEL); - has_gyro = SDL_GameControllerHasSensor(controller, SDL_SENSOR_GYRO); + has_accel = SDL_GameControllerHasSensor(controller, SDL_SENSOR_ACCEL) == SDL_TRUE; + has_gyro = SDL_GameControllerHasSensor(controller, SDL_SENSOR_GYRO) == SDL_TRUE; if (has_accel) { SDL_GameControllerSetSensorEnabled(controller, SDL_SENSOR_ACCEL, SDL_TRUE); } @@ -114,6 +114,20 @@ public: } return false; } + + void EnableVibration(bool is_enabled) { + has_vibration = is_enabled; + is_vibration_tested = true; + } + + bool HasVibration() const { + return has_vibration; + } + + bool IsVibrationTested() const { + return is_vibration_tested; + } + /** * The Pad identifier of the joystick */ @@ -236,6 +250,8 @@ private: u64 last_motion_update{}; bool has_gyro{false}; bool has_accel{false}; + bool has_vibration{false}; + bool is_vibration_tested{false}; BasicMotion motion; }; @@ -517,7 +533,7 @@ std::vector<Common::ParamPackage> SDLDriver::GetInputDevices() const { return devices; } -Common::Input::VibrationError SDLDriver::SetRumble( +Common::Input::VibrationError SDLDriver::SetVibration( const PadIdentifier& identifier, const Common::Input::VibrationStatus& vibration) { const auto joystick = GetSDLJoystickByGUID(identifier.guid.RawString(), static_cast<int>(identifier.port)); @@ -546,13 +562,6 @@ Common::Input::VibrationError SDLDriver::SetRumble( .type = Common::Input::VibrationAmplificationType::Exponential, }; - if (vibration.type == Common::Input::VibrationAmplificationType::Test) { - if (!joystick->RumblePlay(new_vibration)) { - return Common::Input::VibrationError::Unknown; - } - return Common::Input::VibrationError::None; - } - vibration_queue.Push(VibrationRequest{ .identifier = identifier, .vibration = new_vibration, @@ -561,6 +570,45 @@ Common::Input::VibrationError SDLDriver::SetRumble( return Common::Input::VibrationError::None; } +bool SDLDriver::IsVibrationEnabled(const PadIdentifier& identifier) { + const auto joystick = + GetSDLJoystickByGUID(identifier.guid.RawString(), static_cast<int>(identifier.port)); + + constexpr Common::Input::VibrationStatus test_vibration{ + .low_amplitude = 1, + .low_frequency = 160.0f, + .high_amplitude = 1, + .high_frequency = 320.0f, + .type = Common::Input::VibrationAmplificationType::Exponential, + }; + + constexpr Common::Input::VibrationStatus zero_vibration{ + .low_amplitude = 0, + .low_frequency = 160.0f, + .high_amplitude = 0, + .high_frequency = 320.0f, + .type = Common::Input::VibrationAmplificationType::Exponential, + }; + + if (joystick->IsVibrationTested()) { + return joystick->HasVibration(); + } + + // First vibration might fail + joystick->RumblePlay(test_vibration); + + // Wait for about 15ms to ensure the controller is ready for the stop command + std::this_thread::sleep_for(std::chrono::milliseconds(15)); + + if (!joystick->RumblePlay(zero_vibration)) { + joystick->EnableVibration(false); + return false; + } + + joystick->EnableVibration(true); + return true; +} + void SDLDriver::SendVibrations() { while (!vibration_queue.Empty()) { VibrationRequest request; diff --git a/src/input_common/drivers/sdl_driver.h b/src/input_common/drivers/sdl_driver.h index fc3a44572..d1b4471cf 100644 --- a/src/input_common/drivers/sdl_driver.h +++ b/src/input_common/drivers/sdl_driver.h @@ -61,9 +61,11 @@ public: bool IsStickInverted(const Common::ParamPackage& params) override; - Common::Input::VibrationError SetRumble( + Common::Input::VibrationError SetVibration( const PadIdentifier& identifier, const Common::Input::VibrationStatus& vibration) override; + bool IsVibrationEnabled(const PadIdentifier& identifier) override; + private: struct VibrationRequest { PadIdentifier identifier; diff --git a/src/input_common/input_engine.h b/src/input_common/input_engine.h index cfbdb26bd..d4c264a8e 100644 --- a/src/input_common/input_engine.h +++ b/src/input_common/input_engine.h @@ -108,12 +108,17 @@ public: [[maybe_unused]] const Common::Input::LedStatus& led_status) {} // Sets rumble to a controller - virtual Common::Input::VibrationError SetRumble( + virtual Common::Input::VibrationError SetVibration( [[maybe_unused]] const PadIdentifier& identifier, [[maybe_unused]] const Common::Input::VibrationStatus& vibration) { return Common::Input::VibrationError::NotSupported; } + // Returns true if device supports vibrations + virtual bool IsVibrationEnabled([[maybe_unused]] const PadIdentifier& identifier) { + return false; + } + // Sets polling mode to a controller virtual Common::Input::PollingError SetPollingMode( [[maybe_unused]] const PadIdentifier& identifier, diff --git a/src/input_common/input_poller.cpp b/src/input_common/input_poller.cpp index ca33fb4eb..4ac182147 100644 --- a/src/input_common/input_poller.cpp +++ b/src/input_common/input_poller.cpp @@ -763,7 +763,11 @@ public: Common::Input::VibrationError SetVibration( const Common::Input::VibrationStatus& vibration_status) override { - return input_engine->SetRumble(identifier, vibration_status); + return input_engine->SetVibration(identifier, vibration_status); + } + + bool IsVibrationEnabled() override { + return input_engine->IsVibrationEnabled(identifier); } Common::Input::PollingError SetPollingMode(Common::Input::PollingMode polling_mode) override { @@ -797,8 +801,8 @@ std::unique_ptr<Common::Input::InputDevice> InputFactory::CreateButtonDevice( const auto button_id = params.Get("button", 0); const auto keyboard_key = params.Get("code", 0); - const auto toggle = params.Get("toggle", false); - const auto inverted = params.Get("inverted", false); + const auto toggle = params.Get("toggle", false) != 0; + const auto inverted = params.Get("inverted", false) != 0; input_engine->PreSetController(identifier); input_engine->PreSetButton(identifier, button_id); input_engine->PreSetButton(identifier, keyboard_key); @@ -820,8 +824,8 @@ std::unique_ptr<Common::Input::InputDevice> InputFactory::CreateHatButtonDevice( const auto button_id = params.Get("hat", 0); const auto direction = input_engine->GetHatButtonId(params.Get("direction", "")); - const auto toggle = params.Get("toggle", false); - const auto inverted = params.Get("inverted", false); + const auto toggle = params.Get("toggle", false) != 0; + const auto inverted = params.Get("inverted", false) != 0; input_engine->PreSetController(identifier); input_engine->PreSetHatButton(identifier, button_id); @@ -879,7 +883,7 @@ std::unique_ptr<Common::Input::InputDevice> InputFactory::CreateAnalogDevice( .threshold = std::clamp(params.Get("threshold", 0.5f), 0.0f, 1.0f), .offset = std::clamp(params.Get("offset", 0.0f), -1.0f, 1.0f), .inverted = params.Get("invert", "+") == "-", - .toggle = static_cast<bool>(params.Get("toggle", false)), + .toggle = params.Get("toggle", false) != 0, }; input_engine->PreSetController(identifier); input_engine->PreSetAxis(identifier, axis); @@ -895,8 +899,8 @@ std::unique_ptr<Common::Input::InputDevice> InputFactory::CreateTriggerDevice( }; const auto button = params.Get("button", 0); - const auto toggle = params.Get("toggle", false); - const auto inverted = params.Get("inverted", false); + const auto toggle = params.Get("toggle", false) != 0; + const auto inverted = params.Get("inverted", false) != 0; const auto axis = params.Get("axis", 0); const Common::Input::AnalogProperties properties = { @@ -926,8 +930,8 @@ std::unique_ptr<Common::Input::InputDevice> InputFactory::CreateTouchDevice( }; const auto button = params.Get("button", 0); - const auto toggle = params.Get("toggle", false); - const auto inverted = params.Get("inverted", false); + const auto toggle = params.Get("toggle", false) != 0; + const auto inverted = params.Get("inverted", false) != 0; const auto axis_x = params.Get("axis_x", 0); const Common::Input::AnalogProperties properties_x = { diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index af8e51fe8..bcdd60db9 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -241,24 +241,14 @@ target_link_libraries(shader_recompiler PUBLIC common fmt::fmt sirit) if (MSVC) target_compile_options(shader_recompiler PRIVATE /W4 - /WX - /we4018 # 'expression' : signed/unsigned mismatch - /we4244 # 'argument' : conversion from 'type1' to 'type2', possible loss of data (floating-point) - /we4245 # 'conversion' : conversion from 'type1' to 'type2', signed/unsigned mismatch + + /we4242 # 'identifier': conversion from 'type1' to 'type2', possible loss of data /we4254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data - /we4267 # 'var' : conversion from 'size_t' to 'type', possible loss of data - /we4305 # 'context' : truncation from 'type1' to 'type2' /we4800 # Implicit conversion from 'type' to bool. Possible information loss - /we4826 # Conversion from 'type1' to 'type2' is sign-extended. This may cause unexpected runtime behavior. ) else() target_compile_options(shader_recompiler PRIVATE - -Werror -Werror=conversion - -Werror=ignored-qualifiers - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable> - -Werror=unused-variable # Bracket depth determines maximum size of a fold expression in Clang since 9c9974c3ccb6. # And this in turns limits the size of a std::array. diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp index 7094d8e42..1f4ffdd62 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_not_implemented.cpp @@ -5,10 +5,6 @@ #include "shader_recompiler/backend/glasm/glasm_emit_context.h" #include "shader_recompiler/frontend/ir/value.h" -#ifdef _MSC_VER -#pragma warning(disable : 4100) -#endif - namespace Shader::Backend::GLASM { #define NotImplemented() throw NotImplementedException("GLASM instruction {}", __LINE__) diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp index b03a8ba1e..9f1ed95a4 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp +++ b/src/shader_recompiler/backend/glsl/emit_glsl_not_implemented.cpp @@ -7,10 +7,6 @@ #include "shader_recompiler/backend/glsl/glsl_emit_context.h" #include "shader_recompiler/frontend/ir/value.h" -#ifdef _MSC_VER -#pragma warning(disable : 4100) -#endif - namespace Shader::Backend::GLSL { void EmitGetRegister(EmitContext& ctx) { diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp index 468782eb1..84417980b 100644 --- a/src/shader_recompiler/frontend/ir/microinstruction.cpp +++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp @@ -325,11 +325,6 @@ void Inst::AddPhiOperand(Block* predecessor, const Value& value) { phi_args.emplace_back(predecessor, value); } -void Inst::ErasePhiOperand(size_t index) { - const auto operand_it{phi_args.begin() + static_cast<ptrdiff_t>(index)}; - phi_args.erase(operand_it); -} - void Inst::OrderPhiArgs() { if (op != Opcode::Phi) { throw LogicError("{} is not a Phi instruction", op); diff --git a/src/shader_recompiler/frontend/ir/value.h b/src/shader_recompiler/frontend/ir/value.h index 1a2e4ccb6..6a673ca05 100644 --- a/src/shader_recompiler/frontend/ir/value.h +++ b/src/shader_recompiler/frontend/ir/value.h @@ -178,13 +178,9 @@ public: /// Get a pointer to the block of a phi argument. [[nodiscard]] Block* PhiBlock(size_t index) const; - /// Add phi operand to a phi instruction. void AddPhiOperand(Block* predecessor, const Value& value); - // Erase the phi operand at the given index. - void ErasePhiOperand(size_t index); - /// Orders the Phi arguments from farthest away to nearest. void OrderPhiArgs(); diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index 77efb4f57..b58741d4d 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -137,28 +137,35 @@ bool IsLegacyAttribute(IR::Attribute attribute) { } std::map<IR::Attribute, IR::Attribute> GenerateLegacyToGenericMappings( - const VaryingState& state, std::queue<IR::Attribute> ununsed_generics) { + const VaryingState& state, std::queue<IR::Attribute> unused_generics, + const std::map<IR::Attribute, IR::Attribute>& previous_stage_mapping) { std::map<IR::Attribute, IR::Attribute> mapping; + auto update_mapping = [&mapping, &unused_generics, previous_stage_mapping](IR::Attribute attr, + size_t count) { + if (previous_stage_mapping.find(attr) != previous_stage_mapping.end()) { + for (size_t i = 0; i < count; ++i) { + mapping.insert({attr + i, previous_stage_mapping.at(attr + i)}); + } + } else { + for (size_t i = 0; i < count; ++i) { + mapping.insert({attr + i, unused_generics.front() + i}); + } + unused_generics.pop(); + } + }; for (size_t index = 0; index < 4; ++index) { auto attr = IR::Attribute::ColorFrontDiffuseR + index * 4; if (state.AnyComponent(attr)) { - for (size_t i = 0; i < 4; ++i) { - mapping.insert({attr + i, ununsed_generics.front() + i}); - } - ununsed_generics.pop(); + update_mapping(attr, 4); } } if (state[IR::Attribute::FogCoordinate]) { - mapping.insert({IR::Attribute::FogCoordinate, ununsed_generics.front()}); - ununsed_generics.pop(); + update_mapping(IR::Attribute::FogCoordinate, 1); } for (size_t index = 0; index < IR::NUM_FIXEDFNCTEXTURE; ++index) { auto attr = IR::Attribute::FixedFncTexture0S + index * 4; if (state.AnyComponent(attr)) { - for (size_t i = 0; i < 4; ++i) { - mapping.insert({attr + i, ununsed_generics.front() + i}); - } - ununsed_generics.pop(); + update_mapping(attr, 4); } } return mapping; @@ -265,21 +272,22 @@ IR::Program MergeDualVertexPrograms(IR::Program& vertex_a, IR::Program& vertex_b void ConvertLegacyToGeneric(IR::Program& program, const Shader::RuntimeInfo& runtime_info) { auto& stores = program.info.stores; if (stores.Legacy()) { - std::queue<IR::Attribute> ununsed_output_generics{}; + std::queue<IR::Attribute> unused_output_generics{}; for (size_t index = 0; index < IR::NUM_GENERICS; ++index) { if (!stores.Generic(index)) { - ununsed_output_generics.push(IR::Attribute::Generic0X + index * 4); + unused_output_generics.push(IR::Attribute::Generic0X + index * 4); } } - auto mappings = GenerateLegacyToGenericMappings(stores, ununsed_output_generics); + program.info.legacy_stores_mapping = + GenerateLegacyToGenericMappings(stores, unused_output_generics, {}); for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { switch (inst.GetOpcode()) { case IR::Opcode::SetAttribute: { const auto attr = inst.Arg(0).Attribute(); if (IsLegacyAttribute(attr)) { - stores.Set(mappings[attr], true); - inst.SetArg(0, Shader::IR::Value(mappings[attr])); + stores.Set(program.info.legacy_stores_mapping[attr], true); + inst.SetArg(0, Shader::IR::Value(program.info.legacy_stores_mapping[attr])); } break; } @@ -292,15 +300,16 @@ void ConvertLegacyToGeneric(IR::Program& program, const Shader::RuntimeInfo& run auto& loads = program.info.loads; if (loads.Legacy()) { - std::queue<IR::Attribute> ununsed_input_generics{}; + std::queue<IR::Attribute> unused_input_generics{}; for (size_t index = 0; index < IR::NUM_GENERICS; ++index) { const AttributeType input_type{runtime_info.generic_input_types[index]}; if (!runtime_info.previous_stage_stores.Generic(index) || !loads.Generic(index) || input_type == AttributeType::Disabled) { - ununsed_input_generics.push(IR::Attribute::Generic0X + index * 4); + unused_input_generics.push(IR::Attribute::Generic0X + index * 4); } } - auto mappings = GenerateLegacyToGenericMappings(loads, ununsed_input_generics); + auto mappings = GenerateLegacyToGenericMappings( + loads, unused_input_generics, runtime_info.previous_stage_legacy_stores_mapping); for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { switch (inst.GetOpcode()) { diff --git a/src/shader_recompiler/ir_opt/dead_code_elimination_pass.cpp b/src/shader_recompiler/ir_opt/dead_code_elimination_pass.cpp index 9a7d47344..1bd8afd6f 100644 --- a/src/shader_recompiler/ir_opt/dead_code_elimination_pass.cpp +++ b/src/shader_recompiler/ir_opt/dead_code_elimination_pass.cpp @@ -1,104 +1,24 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include <algorithm> - -#include <boost/container/small_vector.hpp> - #include "shader_recompiler/frontend/ir/basic_block.h" #include "shader_recompiler/frontend/ir/value.h" #include "shader_recompiler/ir_opt/passes.h" namespace Shader::Optimization { -namespace { -template <bool TEST_USES> -void DeadInstElimination(IR::Block* const block) { + +void DeadCodeEliminationPass(IR::Program& program) { // We iterate over the instructions in reverse order. // This is because removing an instruction reduces the number of uses for earlier instructions. - auto it{block->end()}; - while (it != block->begin()) { - --it; - if constexpr (TEST_USES) { - if (it->HasUses() || it->MayHaveSideEffects()) { - continue; - } - } - it->Invalidate(); - it = block->Instructions().erase(it); - } -} - -void DeletedPhiArgElimination(IR::Program& program, std::span<const IR::Block*> dead_blocks) { - for (IR::Block* const block : program.blocks) { - for (IR::Inst& phi : *block) { - if (!IR::IsPhi(phi)) { - continue; - } - for (size_t i = 0; i < phi.NumArgs(); ++i) { - if (std::ranges::find(dead_blocks, phi.PhiBlock(i)) == dead_blocks.end()) { - continue; - } - // Phi operand at this index is an unreachable block - phi.ErasePhiOperand(i); - --i; - } - } - } -} - -void DeadBranchElimination(IR::Program& program) { - boost::container::small_vector<const IR::Block*, 3> dead_blocks; - const auto begin_it{program.syntax_list.begin()}; - for (auto node_it = begin_it; node_it != program.syntax_list.end(); ++node_it) { - if (node_it->type != IR::AbstractSyntaxNode::Type::If) { - continue; - } - IR::Inst* const cond_ref{node_it->data.if_node.cond.Inst()}; - const IR::U1 cond{cond_ref->Arg(0)}; - if (!cond.IsImmediate()) { - continue; - } - if (cond.U1()) { - continue; - } - // False immediate condition. Remove condition ref, erase the entire branch. - cond_ref->Invalidate(); - // Account for nested if-statements within the if(false) branch - u32 nested_ifs{1u}; - while (node_it->type != IR::AbstractSyntaxNode::Type::EndIf || nested_ifs > 0) { - node_it = program.syntax_list.erase(node_it); - switch (node_it->type) { - case IR::AbstractSyntaxNode::Type::If: - ++nested_ifs; - break; - case IR::AbstractSyntaxNode::Type::EndIf: - --nested_ifs; - break; - case IR::AbstractSyntaxNode::Type::Block: { - IR::Block* const block{node_it->data.block}; - DeadInstElimination<false>(block); - dead_blocks.push_back(block); - break; - } - default: - break; + for (IR::Block* const block : program.post_order_blocks) { + auto it{block->end()}; + while (it != block->begin()) { + --it; + if (!it->HasUses() && !it->MayHaveSideEffects()) { + it->Invalidate(); + it = block->Instructions().erase(it); } } - // Erase EndIf node of the if(false) branch - node_it = program.syntax_list.erase(node_it); - // Account for loop increment - --node_it; - } - if (!dead_blocks.empty()) { - DeletedPhiArgElimination(program, std::span(dead_blocks.data(), dead_blocks.size())); - } -} -} // namespace - -void DeadCodeEliminationPass(IR::Program& program) { - DeadBranchElimination(program); - for (IR::Block* const block : program.post_order_blocks) { - DeadInstElimination<true>(block); } } diff --git a/src/shader_recompiler/runtime_info.h b/src/shader_recompiler/runtime_info.h index dcb5ab158..549b81ef7 100644 --- a/src/shader_recompiler/runtime_info.h +++ b/src/shader_recompiler/runtime_info.h @@ -4,6 +4,7 @@ #pragma once #include <array> +#include <map> #include <optional> #include <vector> @@ -60,6 +61,7 @@ struct TransformFeedbackVarying { struct RuntimeInfo { std::array<AttributeType, 32> generic_input_types{}; VaryingState previous_stage_stores; + std::map<IR::Attribute, IR::Attribute> previous_stage_legacy_stores_mapping; bool convert_depth_mode{}; bool force_early_z{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index cc596da4f..81097bf1a 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -5,6 +5,7 @@ #include <array> #include <bitset> +#include <map> #include "common/common_types.h" #include "shader_recompiler/frontend/ir/type.h" @@ -127,6 +128,8 @@ struct Info { VaryingState stores; VaryingState passthrough; + std::map<IR::Attribute, IR::Attribute> legacy_stores_mapping; + bool loads_indexed_attributes{}; std::array<bool, 8> stores_frag_color{}; diff --git a/src/tests/core/core_timing.cpp b/src/tests/core/core_timing.cpp index 7c432a63c..284b2ae66 100644 --- a/src/tests/core/core_timing.cpp +++ b/src/tests/core/core_timing.cpp @@ -40,9 +40,6 @@ struct ScopeInit final { core_timing.SetMulticore(true); core_timing.Initialize([]() {}); } - ~ScopeInit() { - core_timing.Shutdown(); - } Core::Timing::CoreTiming core_timing; }; diff --git a/src/tests/video_core/buffer_base.cpp b/src/tests/video_core/buffer_base.cpp index 71121e42a..f7236afab 100644 --- a/src/tests/video_core/buffer_base.cpp +++ b/src/tests/video_core/buffer_base.cpp @@ -44,7 +44,7 @@ public: [[nodiscard]] unsigned Count() const noexcept { unsigned count = 0; - for (const auto [index, value] : page_table) { + for (const auto& [index, value] : page_table) { count += value; } return count; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 40e6d1ec4..106991969 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -82,6 +82,7 @@ add_library(video_core STATIC gpu_thread.h memory_manager.cpp memory_manager.h + pte_kind.h query_cache.h rasterizer_accelerated.cpp rasterizer_accelerated.h @@ -278,14 +279,8 @@ if (MSVC) else() target_compile_options(video_core PRIVATE -Werror=conversion - -Wno-error=sign-conversion - -Werror=pessimizing-move - -Werror=redundant-move - -Werror=type-limits - $<$<CXX_COMPILER_ID:GNU>:-Werror=class-memaccess> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-parameter> - $<$<CXX_COMPILER_ID:GNU>:-Werror=unused-but-set-variable> + -Wno-sign-conversion ) endif() diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index 89a9d1f5a..f9794dfe4 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -117,10 +117,15 @@ void Maxwell3D::InitializeRegisterDefaults() { shadow_state = regs; - mme_inline[MAXWELL3D_REG_INDEX(draw.end)] = true; - mme_inline[MAXWELL3D_REG_INDEX(draw.begin)] = true; - mme_inline[MAXWELL3D_REG_INDEX(vertex_buffer.count)] = true; - mme_inline[MAXWELL3D_REG_INDEX(index_buffer.count)] = true; + draw_command[MAXWELL3D_REG_INDEX(draw.end)] = true; + draw_command[MAXWELL3D_REG_INDEX(draw.begin)] = true; + draw_command[MAXWELL3D_REG_INDEX(vertex_buffer.first)] = true; + draw_command[MAXWELL3D_REG_INDEX(vertex_buffer.count)] = true; + draw_command[MAXWELL3D_REG_INDEX(index_buffer.first)] = true; + draw_command[MAXWELL3D_REG_INDEX(index_buffer.count)] = true; + draw_command[MAXWELL3D_REG_INDEX(draw_inline_index)] = true; + draw_command[MAXWELL3D_REG_INDEX(inline_index_2x16.even)] = true; + draw_command[MAXWELL3D_REG_INDEX(inline_index_4x8.index0)] = true; } void Maxwell3D::ProcessMacro(u32 method, const u32* base_start, u32 amount, bool is_last_call) { @@ -208,25 +213,21 @@ void Maxwell3D::ProcessMethodCall(u32 method, u32 argument, u32 nonshadow_argume return ProcessCBBind(3); case MAXWELL3D_REG_INDEX(bind_groups[4].raw_config): return ProcessCBBind(4); - case MAXWELL3D_REG_INDEX(draw.end): - return DrawArrays(); case MAXWELL3D_REG_INDEX(index_buffer32_first): regs.index_buffer.count = regs.index_buffer32_first.count; regs.index_buffer.first = regs.index_buffer32_first.first; dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; - return DrawArrays(); + return ProcessDraw(); case MAXWELL3D_REG_INDEX(index_buffer16_first): regs.index_buffer.count = regs.index_buffer16_first.count; regs.index_buffer.first = regs.index_buffer16_first.first; dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; - return DrawArrays(); + return ProcessDraw(); case MAXWELL3D_REG_INDEX(index_buffer8_first): regs.index_buffer.count = regs.index_buffer8_first.count; regs.index_buffer.first = regs.index_buffer8_first.first; dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; - // a macro calls this one over and over, should it increase instancing? - // Used by Hades and likely other Vulkan games. - return DrawArrays(); + return ProcessDraw(); case MAXWELL3D_REG_INDEX(topology_override): use_topology_override = true; return; @@ -261,14 +262,13 @@ void Maxwell3D::CallMacroMethod(u32 method, const std::vector<u32>& parameters) // Execute the current macro. macro_engine->Execute(macro_positions[entry], parameters); - if (mme_draw.current_mode != MMEDrawMode::Undefined) { - FlushMMEInlineDraw(); - } + + ProcessDeferredDraw(); } void Maxwell3D::CallMethod(u32 method, u32 method_argument, bool is_last_call) { - // It is an error to write to a register other than the current macro's ARG register before it - // has finished execution. + // It is an error to write to a register other than the current macro's ARG register before + // it has finished execution. if (executing_macro != 0) { ASSERT(method == executing_macro + 1); } @@ -283,9 +283,33 @@ void Maxwell3D::CallMethod(u32 method, u32 method_argument, bool is_last_call) { ASSERT_MSG(method < Regs::NUM_REGS, "Invalid Maxwell3D register, increase the size of the Regs structure"); - const u32 argument = ProcessShadowRam(method, method_argument); - ProcessDirtyRegisters(method, argument); - ProcessMethodCall(method, argument, method_argument, is_last_call); + if (draw_command[method]) { + regs.reg_array[method] = method_argument; + deferred_draw_method.push_back(method); + auto u32_to_u8 = [&](const u32 argument) { + inline_index_draw_indexes.push_back(static_cast<u8>(argument & 0x000000ff)); + inline_index_draw_indexes.push_back(static_cast<u8>((argument & 0x0000ff00) >> 8)); + inline_index_draw_indexes.push_back(static_cast<u8>((argument & 0x00ff0000) >> 16)); + inline_index_draw_indexes.push_back(static_cast<u8>((argument & 0xff000000) >> 24)); + }; + if (MAXWELL3D_REG_INDEX(draw_inline_index) == method) { + u32_to_u8(method_argument); + } else if (MAXWELL3D_REG_INDEX(inline_index_2x16.even) == method) { + u32_to_u8(regs.inline_index_2x16.even); + u32_to_u8(regs.inline_index_2x16.odd); + } else if (MAXWELL3D_REG_INDEX(inline_index_4x8.index0) == method) { + u32_to_u8(regs.inline_index_4x8.index0); + u32_to_u8(regs.inline_index_4x8.index1); + u32_to_u8(regs.inline_index_4x8.index2); + u32_to_u8(regs.inline_index_4x8.index3); + } + } else { + ProcessDeferredDraw(); + + const u32 argument = ProcessShadowRam(method, method_argument); + ProcessDirtyRegisters(method, argument); + ProcessMethodCall(method, argument, method_argument, is_last_call); + } } void Maxwell3D::CallMultiMethod(u32 method, const u32* base_start, u32 amount, @@ -326,55 +350,6 @@ void Maxwell3D::CallMultiMethod(u32 method, const u32* base_start, u32 amount, } } -void Maxwell3D::StepInstance(const MMEDrawMode expected_mode, const u32 count) { - if (mme_draw.current_mode == MMEDrawMode::Undefined) { - if (mme_draw.gl_begin_consume) { - mme_draw.current_mode = expected_mode; - mme_draw.current_count = count; - mme_draw.instance_count = 1; - mme_draw.gl_begin_consume = false; - mme_draw.gl_end_count = 0; - } - return; - } else { - if (mme_draw.current_mode == expected_mode && count == mme_draw.current_count && - mme_draw.instance_mode && mme_draw.gl_begin_consume) { - mme_draw.instance_count++; - mme_draw.gl_begin_consume = false; - return; - } else { - FlushMMEInlineDraw(); - } - } - // Tail call in case it needs to retry. - StepInstance(expected_mode, count); -} - -void Maxwell3D::CallMethodFromMME(u32 method, u32 method_argument) { - if (mme_inline[method]) { - regs.reg_array[method] = method_argument; - if (method == MAXWELL3D_REG_INDEX(vertex_buffer.count) || - method == MAXWELL3D_REG_INDEX(index_buffer.count)) { - const MMEDrawMode expected_mode = method == MAXWELL3D_REG_INDEX(vertex_buffer.count) - ? MMEDrawMode::Array - : MMEDrawMode::Indexed; - StepInstance(expected_mode, method_argument); - } else if (method == MAXWELL3D_REG_INDEX(draw.begin)) { - mme_draw.instance_mode = - (regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::Subsequent) || - (regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::Unchanged); - mme_draw.gl_begin_consume = true; - } else { - mme_draw.gl_end_count++; - } - } else { - if (mme_draw.current_mode != MMEDrawMode::Undefined) { - FlushMMEInlineDraw(); - } - CallMethod(method, method_argument, true); - } -} - void Maxwell3D::ProcessTopologyOverride() { using PrimitiveTopology = Maxwell3D::Regs::PrimitiveTopology; using PrimitiveTopologyOverride = Maxwell3D::Regs::PrimitiveTopologyOverride; @@ -404,41 +379,6 @@ void Maxwell3D::ProcessTopologyOverride() { } } -void Maxwell3D::FlushMMEInlineDraw() { - LOG_TRACE(HW_GPU, "called, topology={}, count={}", regs.draw.topology.Value(), - regs.vertex_buffer.count); - ASSERT_MSG(!(regs.index_buffer.count && regs.vertex_buffer.count), "Both indexed and direct?"); - ASSERT(mme_draw.instance_count == mme_draw.gl_end_count); - - // Both instance configuration registers can not be set at the same time. - ASSERT_MSG(regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::First || - regs.draw.instance_id != Maxwell3D::Regs::Draw::InstanceId::Unchanged, - "Illegal combination of instancing parameters"); - - ProcessTopologyOverride(); - - const bool is_indexed = mme_draw.current_mode == MMEDrawMode::Indexed; - if (ShouldExecute()) { - rasterizer->Draw(is_indexed, true); - } - - // TODO(bunnei): Below, we reset vertex count so that we can use these registers to determine if - // the game is trying to draw indexed or direct mode. This needs to be verified on HW still - - // it's possible that it is incorrect and that there is some other register used to specify the - // drawing mode. - if (is_indexed) { - regs.index_buffer.count = 0; - } else { - regs.vertex_buffer.count = 0; - } - mme_draw.current_mode = MMEDrawMode::Undefined; - mme_draw.current_count = 0; - mme_draw.instance_count = 0; - mme_draw.instance_mode = false; - mme_draw.gl_begin_consume = false; - mme_draw.gl_end_count = 0; -} - void Maxwell3D::ProcessMacroUpload(u32 data) { macro_engine->AddCode(regs.load_mme.instruction_ptr++, data); } @@ -473,9 +413,7 @@ void Maxwell3D::ProcessQueryGet() { switch (regs.report_semaphore.query.operation) { case Regs::ReportSemaphore::Operation::Release: - if (regs.report_semaphore.query.release == - Regs::ReportSemaphore::Release::AfterAllPreceedingWrites || - regs.report_semaphore.query.short_query != 0) { + if (regs.report_semaphore.query.short_query != 0) { const GPUVAddr sequence_address{regs.report_semaphore.Address()}; const u32 payload = regs.report_semaphore.payload; std::function<void()> operation([this, sequence_address, payload] { @@ -489,11 +427,10 @@ void Maxwell3D::ProcessQueryGet() { }; const GPUVAddr sequence_address{regs.report_semaphore.Address()}; const u32 payload = regs.report_semaphore.payload; - std::function<void()> operation([this, sequence_address, payload] { + [this, sequence_address, payload] { memory_manager.Write<u64>(sequence_address + sizeof(u64), system.GPU().GetTicks()); memory_manager.Write<u64>(sequence_address, payload); - }); - rasterizer->SyncOperation(std::move(operation)); + }(); } break; case Regs::ReportSemaphore::Operation::Acquire: @@ -569,47 +506,11 @@ void Maxwell3D::ProcessCounterReset() { void Maxwell3D::ProcessSyncPoint() { const u32 sync_point = regs.sync_info.sync_point.Value(); - const auto condition = regs.sync_info.condition.Value(); - [[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value(); - if (condition == Regs::SyncInfo::Condition::RopWritesDone) { - rasterizer->SignalSyncPoint(sync_point); - } -} - -void Maxwell3D::DrawArrays() { - LOG_TRACE(HW_GPU, "called, topology={}, count={}", regs.draw.topology.Value(), - regs.vertex_buffer.count); - ASSERT_MSG(!(regs.index_buffer.count && regs.vertex_buffer.count), "Both indexed and direct?"); - - // Both instance configuration registers can not be set at the same time. - ASSERT_MSG(regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::First || - regs.draw.instance_id != Maxwell3D::Regs::Draw::InstanceId::Unchanged, - "Illegal combination of instancing parameters"); - - ProcessTopologyOverride(); - - if (regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::Subsequent) { - // Increment the current instance *before* drawing. - state.current_instance++; - } else if (regs.draw.instance_id != Maxwell3D::Regs::Draw::InstanceId::Unchanged) { - // Reset the current instance to 0. - state.current_instance = 0; - } - - const bool is_indexed{regs.index_buffer.count && !regs.vertex_buffer.count}; - if (ShouldExecute()) { - rasterizer->Draw(is_indexed, false); - } - - // TODO(bunnei): Below, we reset vertex count so that we can use these registers to determine if - // the game is trying to draw indexed or direct mode. This needs to be verified on HW still - - // it's possible that it is incorrect and that there is some other register used to specify the - // drawing mode. - if (is_indexed) { - regs.index_buffer.count = 0; - } else { - regs.vertex_buffer.count = 0; + const u32 cache_flush = regs.sync_info.clean_l2.Value(); + if (cache_flush != 0) { + rasterizer->InvalidateGPUCache(); } + rasterizer->SignalSyncPoint(sync_point); } std::optional<u64> Maxwell3D::GetQueryResult() { @@ -694,4 +595,90 @@ void Maxwell3D::ProcessClearBuffers() { rasterizer->Clear(); } +void Maxwell3D::ProcessDraw(u32 instance_count) { + LOG_TRACE(HW_GPU, "called, topology={}, count={}", regs.draw.topology.Value(), + regs.vertex_buffer.count); + + ASSERT_MSG(!(regs.index_buffer.count && regs.vertex_buffer.count), "Both indexed and direct?"); + + // Both instance configuration registers can not be set at the same time. + ASSERT_MSG(regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::First || + regs.draw.instance_id != Maxwell3D::Regs::Draw::InstanceId::Unchanged, + "Illegal combination of instancing parameters"); + + ProcessTopologyOverride(); + + const bool is_indexed = regs.index_buffer.count && !regs.vertex_buffer.count; + if (ShouldExecute()) { + rasterizer->Draw(is_indexed, instance_count); + } + + if (is_indexed) { + regs.index_buffer.count = 0; + } else { + regs.vertex_buffer.count = 0; + } +} + +void Maxwell3D::ProcessDeferredDraw() { + if (deferred_draw_method.empty()) { + return; + } + + enum class DrawMode { + Undefined, + General, + Instance, + }; + DrawMode draw_mode{DrawMode::Undefined}; + u32 instance_count = 1; + + u32 index = 0; + u32 method = 0; + u32 method_count = static_cast<u32>(deferred_draw_method.size()); + for (; index < method_count && + (method = deferred_draw_method[index]) != MAXWELL3D_REG_INDEX(draw.begin); + ++index) + ; + + if (MAXWELL3D_REG_INDEX(draw.begin) != method) { + return; + } + + // The minimum number of methods for drawing must be greater than or equal to + // 3[draw.begin->vertex(index)count(first)->draw.end] to avoid errors in index mode drawing + if ((method_count - index) < 3) { + return; + } + draw_mode = (regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::Subsequent) || + (regs.draw.instance_id == Maxwell3D::Regs::Draw::InstanceId::Unchanged) + ? DrawMode::Instance + : DrawMode::General; + + // Drawing will only begin with draw.begin or index_buffer method, other methods directly + // clear + if (draw_mode == DrawMode::Undefined) { + deferred_draw_method.clear(); + return; + } + + if (draw_mode == DrawMode::Instance) { + ASSERT_MSG(deferred_draw_method.size() % 4 == 0, "Instance mode method size error"); + instance_count = static_cast<u32>(method_count - index) / 4; + } else { + method = deferred_draw_method[index + 1]; + if (MAXWELL3D_REG_INDEX(draw_inline_index) == method || + MAXWELL3D_REG_INDEX(inline_index_2x16.even) == method || + MAXWELL3D_REG_INDEX(inline_index_4x8.index0) == method) { + regs.index_buffer.count = static_cast<u32>(inline_index_draw_indexes.size() / 4); + regs.index_buffer.format = Regs::IndexFormat::UnsignedInt; + } + } + + ProcessDraw(instance_count); + + deferred_draw_method.clear(); + inline_index_draw_indexes.clear(); +} + } // namespace Tegra::Engines diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 75e3b868d..a948fcb14 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -1739,14 +1739,11 @@ public: Footprint_1x1_Virtual = 2, }; - struct InlineIndex4x8Align { + struct InlineIndex4x8 { union { BitField<0, 30, u32> count; BitField<30, 2, u32> start; }; - }; - - struct InlineIndex4x8Index { union { BitField<0, 8, u32> index0; BitField<8, 8, u32> index1; @@ -2836,8 +2833,7 @@ public: u32 depth_write_enabled; ///< 0x12E8 u32 alpha_test_enabled; ///< 0x12EC INSERT_PADDING_BYTES_NOINIT(0x10); - InlineIndex4x8Align inline_index_4x8_align; ///< 0x1300 - InlineIndex4x8Index inline_index_4x8_index; ///< 0x1304 + InlineIndex4x8 inline_index_4x8; ///< 0x1300 D3DCullMode d3d_cull_mode; ///< 0x1308 ComparisonOp depth_test_func; ///< 0x130C f32 alpha_test_ref; ///< 0x1310 @@ -3048,8 +3044,6 @@ public: }; std::array<ShaderStageInfo, Regs::MaxShaderStage> shader_stages; - - u32 current_instance = 0; ///< Current instance to be used to simulate instanced rendering. }; State state{}; @@ -3064,11 +3058,6 @@ public: void CallMultiMethod(u32 method, const u32* base_start, u32 amount, u32 methods_pending) override; - /// Write the value to the register identified by method. - void CallMethodFromMME(u32 method, u32 method_argument); - - void FlushMMEInlineDraw(); - bool ShouldExecute() const { return execute_on; } @@ -3081,21 +3070,6 @@ public: return *rasterizer; } - enum class MMEDrawMode : u32 { - Undefined, - Array, - Indexed, - }; - - struct MMEDrawState { - MMEDrawMode current_mode{MMEDrawMode::Undefined}; - u32 current_count{}; - u32 instance_count{}; - bool instance_mode{}; - bool gl_begin_consume{}; - u32 gl_end_count{}; - } mme_draw; - struct DirtyState { using Flags = std::bitset<std::numeric_limits<u8>::max()>; using Table = std::array<u8, Regs::NUM_REGS>; @@ -3105,6 +3079,8 @@ public: Tables tables{}; } dirty; + std::vector<u8> inline_index_draw_indexes; + private: void InitializeRegisterDefaults(); @@ -3164,14 +3140,12 @@ private: /// Handles a write to the CB_BIND register. void ProcessCBBind(size_t stage_index); - /// Handles a write to the VERTEX_END_GL register, triggering a draw. - void DrawArrays(); - /// Handles use of topology overrides (e.g., to avoid using a topology assigned from a macro) void ProcessTopologyOverride(); - // Handles a instance drawcall from MME - void StepInstance(MMEDrawMode expected_mode, u32 count); + void ProcessDraw(u32 instance_count = 1); + + void ProcessDeferredDraw(); /// Returns a query's value or an empty object if the value will be deferred through a cache. std::optional<u64> GetQueryResult(); @@ -3184,8 +3158,6 @@ private: /// Start offsets of each macro in macro_memory std::array<u32, 0x80> macro_positions{}; - std::array<bool, Regs::NUM_REGS> mme_inline{}; - /// Macro method that is currently being executed / being fed parameters. u32 executing_macro = 0; /// Parameters that have been submitted to the macro call so far. @@ -3198,6 +3170,9 @@ private: bool execute_on{true}; bool use_topology_override{false}; + + std::array<bool, Regs::NUM_REGS> draw_command{}; + std::vector<u32> deferred_draw_method; }; #define ASSERT_REG_POSITION(field_name, position) \ @@ -3402,8 +3377,7 @@ ASSERT_REG_POSITION(alpha_to_coverage_dither, 0x12E0); ASSERT_REG_POSITION(blend_per_target_enabled, 0x12E4); ASSERT_REG_POSITION(depth_write_enabled, 0x12E8); ASSERT_REG_POSITION(alpha_test_enabled, 0x12EC); -ASSERT_REG_POSITION(inline_index_4x8_align, 0x1300); -ASSERT_REG_POSITION(inline_index_4x8_index, 0x1304); +ASSERT_REG_POSITION(inline_index_4x8, 0x1300); ASSERT_REG_POSITION(d3d_cull_mode, 0x1308); ASSERT_REG_POSITION(depth_test_func, 0x130C); ASSERT_REG_POSITION(alpha_test_ref, 0x1310); diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index 3909d36c1..4eb7a100d 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -56,66 +56,85 @@ void MaxwellDMA::Launch() { ASSERT(launch.interrupt_type == LaunchDMA::InterruptType::NONE); ASSERT(launch.data_transfer_type == LaunchDMA::DataTransferType::NON_PIPELINED); - const bool is_src_pitch = launch.src_memory_layout == LaunchDMA::MemoryLayout::PITCH; - const bool is_dst_pitch = launch.dst_memory_layout == LaunchDMA::MemoryLayout::PITCH; - - if (!is_src_pitch && !is_dst_pitch) { - // If both the source and the destination are in block layout, assert. - UNIMPLEMENTED_MSG("Tiled->Tiled DMA transfers are not yet implemented"); - return; - } + if (launch.multi_line_enable) { + const bool is_src_pitch = launch.src_memory_layout == LaunchDMA::MemoryLayout::PITCH; + const bool is_dst_pitch = launch.dst_memory_layout == LaunchDMA::MemoryLayout::PITCH; + + if (!is_src_pitch && !is_dst_pitch) { + // If both the source and the destination are in block layout, assert. + UNIMPLEMENTED_MSG("Tiled->Tiled DMA transfers are not yet implemented"); + return; + } - if (is_src_pitch && is_dst_pitch) { - CopyPitchToPitch(); + if (is_src_pitch && is_dst_pitch) { + for (u32 line = 0; line < regs.line_count; ++line) { + const GPUVAddr source_line = + regs.offset_in + static_cast<size_t>(line) * regs.pitch_in; + const GPUVAddr dest_line = + regs.offset_out + static_cast<size_t>(line) * regs.pitch_out; + memory_manager.CopyBlock(dest_line, source_line, regs.line_length_in); + } + } else { + if (!is_src_pitch && is_dst_pitch) { + CopyBlockLinearToPitch(); + } else { + CopyPitchToBlockLinear(); + } + } } else { - ASSERT(launch.multi_line_enable == 1); - - if (!is_src_pitch && is_dst_pitch) { - CopyBlockLinearToPitch(); + // TODO: allow multisized components. + auto& accelerate = rasterizer->AccessAccelerateDMA(); + const bool is_const_a_dst = regs.remap_const.dst_x == RemapConst::Swizzle::CONST_A; + if (regs.launch_dma.remap_enable != 0 && is_const_a_dst) { + ASSERT(regs.remap_const.component_size_minus_one == 3); + accelerate.BufferClear(regs.offset_out, regs.line_length_in, regs.remap_consta_value); + std::vector<u32> tmp_buffer(regs.line_length_in, regs.remap_consta_value); + memory_manager.WriteBlockUnsafe(regs.offset_out, + reinterpret_cast<u8*>(tmp_buffer.data()), + regs.line_length_in * sizeof(u32)); } else { - CopyPitchToBlockLinear(); + auto convert_linear_2_blocklinear_addr = [](u64 address) { + return (address & ~0x1f0ULL) | ((address & 0x40) >> 2) | ((address & 0x10) << 1) | + ((address & 0x180) >> 1) | ((address & 0x20) << 3); + }; + auto src_kind = memory_manager.GetPageKind(regs.offset_in); + auto dst_kind = memory_manager.GetPageKind(regs.offset_out); + const bool is_src_pitch = IsPitchKind(static_cast<PTEKind>(src_kind)); + const bool is_dst_pitch = IsPitchKind(static_cast<PTEKind>(dst_kind)); + if (!is_src_pitch && is_dst_pitch) { + std::vector<u8> tmp_buffer(regs.line_length_in); + std::vector<u8> dst_buffer(regs.line_length_in); + memory_manager.ReadBlockUnsafe(regs.offset_in, tmp_buffer.data(), + regs.line_length_in); + for (u32 offset = 0; offset < regs.line_length_in; ++offset) { + dst_buffer[offset] = + tmp_buffer[convert_linear_2_blocklinear_addr(regs.offset_in + offset) - + regs.offset_in]; + } + memory_manager.WriteBlock(regs.offset_out, dst_buffer.data(), regs.line_length_in); + } else if (is_src_pitch && !is_dst_pitch) { + std::vector<u8> tmp_buffer(regs.line_length_in); + std::vector<u8> dst_buffer(regs.line_length_in); + memory_manager.ReadBlockUnsafe(regs.offset_in, tmp_buffer.data(), + regs.line_length_in); + for (u32 offset = 0; offset < regs.line_length_in; ++offset) { + dst_buffer[convert_linear_2_blocklinear_addr(regs.offset_out + offset) - + regs.offset_out] = tmp_buffer[offset]; + } + memory_manager.WriteBlock(regs.offset_out, dst_buffer.data(), regs.line_length_in); + } else { + if (!accelerate.BufferCopy(regs.offset_in, regs.offset_out, regs.line_length_in)) { + std::vector<u8> tmp_buffer(regs.line_length_in); + memory_manager.ReadBlockUnsafe(regs.offset_in, tmp_buffer.data(), + regs.line_length_in); + memory_manager.WriteBlock(regs.offset_out, tmp_buffer.data(), + regs.line_length_in); + } + } } } - ReleaseSemaphore(); -} -void MaxwellDMA::CopyPitchToPitch() { - // When `multi_line_enable` bit is enabled we copy a 2D image of dimensions - // (line_length_in, line_count). - // Otherwise the copy is performed as if we were copying a 1D buffer of length line_length_in. - const bool remap_enabled = regs.launch_dma.remap_enable != 0; - if (regs.launch_dma.multi_line_enable) { - UNIMPLEMENTED_IF(remap_enabled); - - // Perform a line-by-line copy. - // We're going to take a subrect of size (line_length_in, line_count) from the source - // rectangle. There is no need to manually flush/invalidate the regions because CopyBlock - // does that for us. - for (u32 line = 0; line < regs.line_count; ++line) { - const GPUVAddr source_line = regs.offset_in + static_cast<size_t>(line) * regs.pitch_in; - const GPUVAddr dest_line = regs.offset_out + static_cast<size_t>(line) * regs.pitch_out; - memory_manager.CopyBlock(dest_line, source_line, regs.line_length_in); - } - return; - } - // TODO: allow multisized components. - auto& accelerate = rasterizer->AccessAccelerateDMA(); - const bool is_const_a_dst = regs.remap_const.dst_x == RemapConst::Swizzle::CONST_A; - const bool is_buffer_clear = remap_enabled && is_const_a_dst; - if (is_buffer_clear) { - ASSERT(regs.remap_const.component_size_minus_one == 3); - accelerate.BufferClear(regs.offset_out, regs.line_length_in, regs.remap_consta_value); - std::vector<u32> tmp_buffer(regs.line_length_in, regs.remap_consta_value); - memory_manager.WriteBlockUnsafe(regs.offset_out, reinterpret_cast<u8*>(tmp_buffer.data()), - regs.line_length_in * sizeof(u32)); - return; - } - UNIMPLEMENTED_IF(remap_enabled); - if (!accelerate.BufferCopy(regs.offset_in, regs.offset_out, regs.line_length_in)) { - std::vector<u8> tmp_buffer(regs.line_length_in); - memory_manager.ReadBlockUnsafe(regs.offset_in, tmp_buffer.data(), regs.line_length_in); - memory_manager.WriteBlock(regs.offset_out, tmp_buffer.data(), regs.line_length_in); - } + ReleaseSemaphore(); } void MaxwellDMA::CopyBlockLinearToPitch() { diff --git a/src/video_core/engines/maxwell_dma.h b/src/video_core/engines/maxwell_dma.h index bc48320ce..953e34adc 100644 --- a/src/video_core/engines/maxwell_dma.h +++ b/src/video_core/engines/maxwell_dma.h @@ -219,8 +219,6 @@ private: /// registers. void Launch(); - void CopyPitchToPitch(); - void CopyBlockLinearToPitch(); void CopyPitchToBlockLinear(); diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index cca890792..3977bb0fb 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -75,11 +75,10 @@ void Puller::ProcessSemaphoreTriggerMethod() { if (op == GpuSemaphoreOperation::WriteLong) { const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()}; const u32 payload = regs.semaphore_sequence; - std::function<void()> operation([this, sequence_address, payload] { + [this, sequence_address, payload] { memory_manager.Write<u64>(sequence_address + sizeof(u64), gpu.GetTicks()); memory_manager.Write<u64>(sequence_address, payload); - }); - rasterizer->SignalFence(std::move(operation)); + }(); } else { do { const u32 word{memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress())}; diff --git a/src/video_core/macro/macro_hle.cpp b/src/video_core/macro/macro_hle.cpp index 8a8adbb42..f896591bf 100644 --- a/src/video_core/macro/macro_hle.cpp +++ b/src/video_core/macro/macro_hle.cpp @@ -22,35 +22,29 @@ void HLE_771BB18C62444DA0(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& maxwell3d.regs.draw.topology.Assign( static_cast<Tegra::Engines::Maxwell3D::Regs::PrimitiveTopology>(parameters[0] & 0x3ffffff)); maxwell3d.regs.global_base_instance_index = parameters[5]; - maxwell3d.mme_draw.instance_count = instance_count; maxwell3d.regs.global_base_vertex_index = parameters[3]; maxwell3d.regs.index_buffer.count = parameters[1]; maxwell3d.regs.index_buffer.first = parameters[4]; if (maxwell3d.ShouldExecute()) { - maxwell3d.Rasterizer().Draw(true, true); + maxwell3d.Rasterizer().Draw(true, instance_count); } maxwell3d.regs.index_buffer.count = 0; - maxwell3d.mme_draw.instance_count = 0; - maxwell3d.mme_draw.current_mode = Engines::Maxwell3D::MMEDrawMode::Undefined; } void HLE_0D61FC9FAAC9FCAD(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& parameters) { - const u32 count = (maxwell3d.GetRegisterValue(0xD1B) & parameters[2]); + const u32 instance_count = (maxwell3d.GetRegisterValue(0xD1B) & parameters[2]); maxwell3d.regs.vertex_buffer.first = parameters[3]; maxwell3d.regs.vertex_buffer.count = parameters[1]; maxwell3d.regs.global_base_instance_index = parameters[4]; maxwell3d.regs.draw.topology.Assign( static_cast<Tegra::Engines::Maxwell3D::Regs::PrimitiveTopology>(parameters[0])); - maxwell3d.mme_draw.instance_count = count; if (maxwell3d.ShouldExecute()) { - maxwell3d.Rasterizer().Draw(false, true); + maxwell3d.Rasterizer().Draw(false, instance_count); } maxwell3d.regs.vertex_buffer.count = 0; - maxwell3d.mme_draw.instance_count = 0; - maxwell3d.mme_draw.current_mode = Engines::Maxwell3D::MMEDrawMode::Undefined; } void HLE_0217920100488FF7(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& parameters) { @@ -63,24 +57,21 @@ void HLE_0217920100488FF7(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; maxwell3d.regs.global_base_vertex_index = element_base; maxwell3d.regs.global_base_instance_index = base_instance; - maxwell3d.mme_draw.instance_count = instance_count; - maxwell3d.CallMethodFromMME(0x8e3, 0x640); - maxwell3d.CallMethodFromMME(0x8e4, element_base); - maxwell3d.CallMethodFromMME(0x8e5, base_instance); + maxwell3d.CallMethod(0x8e3, 0x640, true); + maxwell3d.CallMethod(0x8e4, element_base, true); + maxwell3d.CallMethod(0x8e5, base_instance, true); maxwell3d.regs.draw.topology.Assign( static_cast<Tegra::Engines::Maxwell3D::Regs::PrimitiveTopology>(parameters[0])); if (maxwell3d.ShouldExecute()) { - maxwell3d.Rasterizer().Draw(true, true); + maxwell3d.Rasterizer().Draw(true, instance_count); } maxwell3d.regs.vertex_id_base = 0x0; maxwell3d.regs.index_buffer.count = 0; maxwell3d.regs.global_base_vertex_index = 0x0; maxwell3d.regs.global_base_instance_index = 0x0; - maxwell3d.mme_draw.instance_count = 0; - maxwell3d.CallMethodFromMME(0x8e3, 0x640); - maxwell3d.CallMethodFromMME(0x8e4, 0x0); - maxwell3d.CallMethodFromMME(0x8e5, 0x0); - maxwell3d.mme_draw.current_mode = Engines::Maxwell3D::MMEDrawMode::Undefined; + maxwell3d.CallMethod(0x8e3, 0x640, true); + maxwell3d.CallMethod(0x8e4, 0x0, true); + maxwell3d.CallMethod(0x8e5, 0x0, true); } // Multidraw Indirect @@ -91,11 +82,9 @@ void HLE_3F5E74B9C9A50164(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& maxwell3d.regs.index_buffer.count = 0; maxwell3d.regs.global_base_vertex_index = 0x0; maxwell3d.regs.global_base_instance_index = 0x0; - maxwell3d.mme_draw.instance_count = 0; - maxwell3d.CallMethodFromMME(0x8e3, 0x640); - maxwell3d.CallMethodFromMME(0x8e4, 0x0); - maxwell3d.CallMethodFromMME(0x8e5, 0x0); - maxwell3d.mme_draw.current_mode = Engines::Maxwell3D::MMEDrawMode::Undefined; + maxwell3d.CallMethod(0x8e3, 0x640, true); + maxwell3d.CallMethod(0x8e4, 0x0, true); + maxwell3d.CallMethod(0x8e5, 0x0, true); maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; }); const u32 start_indirect = parameters[0]; @@ -127,15 +116,13 @@ void HLE_3F5E74B9C9A50164(Engines::Maxwell3D& maxwell3d, const std::vector<u32>& maxwell3d.regs.index_buffer.count = num_vertices; maxwell3d.regs.global_base_vertex_index = base_vertex; maxwell3d.regs.global_base_instance_index = base_instance; - maxwell3d.mme_draw.instance_count = instance_count; - maxwell3d.CallMethodFromMME(0x8e3, 0x640); - maxwell3d.CallMethodFromMME(0x8e4, base_vertex); - maxwell3d.CallMethodFromMME(0x8e5, base_instance); + maxwell3d.CallMethod(0x8e3, 0x640, true); + maxwell3d.CallMethod(0x8e4, base_vertex, true); + maxwell3d.CallMethod(0x8e5, base_instance, true); maxwell3d.dirty.flags[VideoCommon::Dirty::IndexBuffer] = true; if (maxwell3d.ShouldExecute()) { - maxwell3d.Rasterizer().Draw(true, true); + maxwell3d.Rasterizer().Draw(true, instance_count); } - maxwell3d.mme_draw.current_mode = Engines::Maxwell3D::MMEDrawMode::Undefined; } } diff --git a/src/video_core/macro/macro_interpreter.cpp b/src/video_core/macro/macro_interpreter.cpp index f670b1bca..c0d32c112 100644 --- a/src/video_core/macro/macro_interpreter.cpp +++ b/src/video_core/macro/macro_interpreter.cpp @@ -335,7 +335,7 @@ void MacroInterpreterImpl::SetMethodAddress(u32 address) { } void MacroInterpreterImpl::Send(u32 value) { - maxwell3d.CallMethodFromMME(method_address.address, value); + maxwell3d.CallMethod(method_address.address, value, true); // Increment the method address by the method increment. method_address.address.Assign(method_address.address.Value() + method_address.increment.Value()); diff --git a/src/video_core/macro/macro_jit_x64.cpp b/src/video_core/macro/macro_jit_x64.cpp index a302a9603..25c1ce798 100644 --- a/src/video_core/macro/macro_jit_x64.cpp +++ b/src/video_core/macro/macro_jit_x64.cpp @@ -346,7 +346,7 @@ void MacroJITx64Impl::Compile_Read(Macro::Opcode opcode) { } void Send(Engines::Maxwell3D* maxwell3d, Macro::MethodAddress method_address, u32 value) { - maxwell3d->CallMethodFromMME(method_address.address, value); + maxwell3d->CallMethod(method_address.address, value, true); } void MacroJITx64Impl::Compile_Send(Xbyak::Reg32 value) { diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index cca401c74..384350dbd 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -41,7 +41,11 @@ MemoryManager::MemoryManager(Core::System& system_, u64 address_space_bits_, u64 big_entries.resize(big_page_table_size / 32, 0); big_page_table_cpu.resize(big_page_table_size); big_page_continous.resize(big_page_table_size / continous_bits, 0); + std::array<PTEKind, 32> kind_valus; + kind_valus.fill(PTEKind::INVALID); + big_kinds.resize(big_page_table_size / 32, kind_valus); entries.resize(page_table_size / 32, 0); + kinds.resize(big_page_table_size / 32, kind_valus); } MemoryManager::~MemoryManager() = default; @@ -78,6 +82,41 @@ void MemoryManager::SetEntry(size_t position, MemoryManager::EntryType entry) { } } +PTEKind MemoryManager::GetPageKind(GPUVAddr gpu_addr) const { + auto entry = GetEntry<true>(gpu_addr); + if (entry == EntryType::Mapped || entry == EntryType::Reserved) [[likely]] { + return GetKind<true>(gpu_addr); + } else { + return GetKind<false>(gpu_addr); + } +} + +template <bool is_big_page> +PTEKind MemoryManager::GetKind(size_t position) const { + if constexpr (is_big_page) { + position = position >> big_page_bits; + const size_t sub_index = position % 32; + return big_kinds[position / 32][sub_index]; + } else { + position = position >> page_bits; + const size_t sub_index = position % 32; + return kinds[position / 32][sub_index]; + } +} + +template <bool is_big_page> +void MemoryManager::SetKind(size_t position, PTEKind kind) { + if constexpr (is_big_page) { + position = position >> big_page_bits; + const size_t sub_index = position % 32; + big_kinds[position / 32][sub_index] = kind; + } else { + position = position >> page_bits; + const size_t sub_index = position % 32; + kinds[position / 32][sub_index] = kind; + } +} + inline bool MemoryManager::IsBigPageContinous(size_t big_page_index) const { const u64 entry_mask = big_page_continous[big_page_index / continous_bits]; const size_t sub_index = big_page_index % continous_bits; @@ -92,9 +131,9 @@ inline void MemoryManager::SetBigPageContinous(size_t big_page_index, bool value } template <MemoryManager::EntryType entry_type> -GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, - size_t size) { - u64 remaining_size{size}; +GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size, + PTEKind kind) { + [[maybe_unused]] u64 remaining_size{size}; if constexpr (entry_type == EntryType::Mapped) { page_table.ReserveRange(gpu_addr, size); } @@ -102,6 +141,7 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cp const GPUVAddr current_gpu_addr = gpu_addr + offset; [[maybe_unused]] const auto current_entry_type = GetEntry<false>(current_gpu_addr); SetEntry<false>(current_gpu_addr, entry_type); + SetKind<false>(current_gpu_addr, kind); if (current_entry_type != entry_type) { rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, page_size); } @@ -118,12 +158,13 @@ GPUVAddr MemoryManager::PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cp template <MemoryManager::EntryType entry_type> GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, - size_t size) { - u64 remaining_size{size}; + size_t size, PTEKind kind) { + [[maybe_unused]] u64 remaining_size{size}; for (u64 offset{}; offset < size; offset += big_page_size) { const GPUVAddr current_gpu_addr = gpu_addr + offset; [[maybe_unused]] const auto current_entry_type = GetEntry<true>(current_gpu_addr); SetEntry<true>(current_gpu_addr, entry_type); + SetKind<true>(current_gpu_addr, kind); if (current_entry_type != entry_type) { rasterizer->ModifyGPUMemory(unique_identifier, gpu_addr, big_page_size); } @@ -159,19 +200,19 @@ void MemoryManager::BindRasterizer(VideoCore::RasterizerInterface* rasterizer_) rasterizer = rasterizer_; } -GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size, +GPUVAddr MemoryManager::Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size, PTEKind kind, bool is_big_pages) { if (is_big_pages) [[likely]] { - return BigPageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size); + return BigPageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size, kind); } - return PageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size); + return PageTableOp<EntryType::Mapped>(gpu_addr, cpu_addr, size, kind); } GPUVAddr MemoryManager::MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages) { if (is_big_pages) [[likely]] { - return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size); + return BigPageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID); } - return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size); + return PageTableOp<EntryType::Reserved>(gpu_addr, 0, size, PTEKind::INVALID); } void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { @@ -188,8 +229,8 @@ void MemoryManager::Unmap(GPUVAddr gpu_addr, std::size_t size) { rasterizer->UnmapMemory(*cpu_addr, map_size); } - BigPageTableOp<EntryType::Free>(gpu_addr, 0, size); - PageTableOp<EntryType::Free>(gpu_addr, 0, size); + BigPageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID); + PageTableOp<EntryType::Free>(gpu_addr, 0, size, PTEKind::INVALID); } std::optional<VAddr> MemoryManager::GpuToCpuAddress(GPUVAddr gpu_addr) const { diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index f992e29f3..ab4bc9ec6 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -11,6 +11,7 @@ #include "common/common_types.h" #include "common/multi_level_page_table.h" #include "common/virtual_buffer.h" +#include "video_core/pte_kind.h" namespace VideoCore { class RasterizerInterface; @@ -98,7 +99,8 @@ public: std::vector<std::pair<GPUVAddr, std::size_t>> GetSubmappedRange(GPUVAddr gpu_addr, std::size_t size) const; - GPUVAddr Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size, bool is_big_pages = true); + GPUVAddr Map(GPUVAddr gpu_addr, VAddr cpu_addr, std::size_t size, + PTEKind kind = PTEKind::INVALID, bool is_big_pages = true); GPUVAddr MapSparse(GPUVAddr gpu_addr, std::size_t size, bool is_big_pages = true); void Unmap(GPUVAddr gpu_addr, std::size_t size); @@ -114,6 +116,8 @@ public: return gpu_addr < address_space_size; } + PTEKind GetPageKind(GPUVAddr gpu_addr) const; + private: template <bool is_big_pages, typename FuncMapped, typename FuncReserved, typename FuncUnmapped> inline void MemoryOperation(GPUVAddr gpu_src_addr, std::size_t size, FuncMapped&& func_mapped, @@ -166,10 +170,12 @@ private: std::vector<u64> big_entries; template <EntryType entry_type> - GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size); + GPUVAddr PageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size, + PTEKind kind); template <EntryType entry_type> - GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size); + GPUVAddr BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] VAddr cpu_addr, size_t size, + PTEKind kind); template <bool is_big_page> inline EntryType GetEntry(size_t position) const; @@ -177,6 +183,15 @@ private: template <bool is_big_page> inline void SetEntry(size_t position, EntryType entry); + std::vector<std::array<PTEKind, 32>> kinds; + std::vector<std::array<PTEKind, 32>> big_kinds; + + template <bool is_big_page> + inline PTEKind GetKind(size_t position) const; + + template <bool is_big_page> + inline void SetKind(size_t position, PTEKind kind); + Common::MultiLevelPageTable<u32> page_table; Common::VirtualBuffer<u32> big_page_table_cpu; diff --git a/src/video_core/pte_kind.h b/src/video_core/pte_kind.h new file mode 100644 index 000000000..591d7214b --- /dev/null +++ b/src/video_core/pte_kind.h @@ -0,0 +1,264 @@ +// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include "common/common_types.h" + +namespace Tegra { + +// https://github.com/NVIDIA/open-gpu-doc/blob/master/manuals/volta/gv100/dev_mmu.ref.txt +enum class PTEKind : u8 { + INVALID = 0xff, + PITCH = 0x00, + Z16 = 0x01, + Z16_2C = 0x02, + Z16_MS2_2C = 0x03, + Z16_MS4_2C = 0x04, + Z16_MS8_2C = 0x05, + Z16_MS16_2C = 0x06, + Z16_2Z = 0x07, + Z16_MS2_2Z = 0x08, + Z16_MS4_2Z = 0x09, + Z16_MS8_2Z = 0x0a, + Z16_MS16_2Z = 0x0b, + Z16_2CZ = 0x36, + Z16_MS2_2CZ = 0x37, + Z16_MS4_2CZ = 0x38, + Z16_MS8_2CZ = 0x39, + Z16_MS16_2CZ = 0x5f, + Z16_4CZ = 0x0c, + Z16_MS2_4CZ = 0x0d, + Z16_MS4_4CZ = 0x0e, + Z16_MS8_4CZ = 0x0f, + Z16_MS16_4CZ = 0x10, + S8Z24 = 0x11, + S8Z24_1Z = 0x12, + S8Z24_MS2_1Z = 0x13, + S8Z24_MS4_1Z = 0x14, + S8Z24_MS8_1Z = 0x15, + S8Z24_MS16_1Z = 0x16, + S8Z24_2CZ = 0x17, + S8Z24_MS2_2CZ = 0x18, + S8Z24_MS4_2CZ = 0x19, + S8Z24_MS8_2CZ = 0x1a, + S8Z24_MS16_2CZ = 0x1b, + S8Z24_2CS = 0x1c, + S8Z24_MS2_2CS = 0x1d, + S8Z24_MS4_2CS = 0x1e, + S8Z24_MS8_2CS = 0x1f, + S8Z24_MS16_2CS = 0x20, + S8Z24_4CSZV = 0x21, + S8Z24_MS2_4CSZV = 0x22, + S8Z24_MS4_4CSZV = 0x23, + S8Z24_MS8_4CSZV = 0x24, + S8Z24_MS16_4CSZV = 0x25, + V8Z24_MS4_VC12 = 0x26, + V8Z24_MS4_VC4 = 0x27, + V8Z24_MS8_VC8 = 0x28, + V8Z24_MS8_VC24 = 0x29, + V8Z24_MS4_VC12_1ZV = 0x2e, + V8Z24_MS4_VC4_1ZV = 0x2f, + V8Z24_MS8_VC8_1ZV = 0x30, + V8Z24_MS8_VC24_1ZV = 0x31, + V8Z24_MS4_VC12_2CS = 0x32, + V8Z24_MS4_VC4_2CS = 0x33, + V8Z24_MS8_VC8_2CS = 0x34, + V8Z24_MS8_VC24_2CS = 0x35, + V8Z24_MS4_VC12_2CZV = 0x3a, + V8Z24_MS4_VC4_2CZV = 0x3b, + V8Z24_MS8_VC8_2CZV = 0x3c, + V8Z24_MS8_VC24_2CZV = 0x3d, + V8Z24_MS4_VC12_2ZV = 0x3e, + V8Z24_MS4_VC4_2ZV = 0x3f, + V8Z24_MS8_VC8_2ZV = 0x40, + V8Z24_MS8_VC24_2ZV = 0x41, + V8Z24_MS4_VC12_4CSZV = 0x42, + V8Z24_MS4_VC4_4CSZV = 0x43, + V8Z24_MS8_VC8_4CSZV = 0x44, + V8Z24_MS8_VC24_4CSZV = 0x45, + Z24S8 = 0x46, + Z24S8_1Z = 0x47, + Z24S8_MS2_1Z = 0x48, + Z24S8_MS4_1Z = 0x49, + Z24S8_MS8_1Z = 0x4a, + Z24S8_MS16_1Z = 0x4b, + Z24S8_2CS = 0x4c, + Z24S8_MS2_2CS = 0x4d, + Z24S8_MS4_2CS = 0x4e, + Z24S8_MS8_2CS = 0x4f, + Z24S8_MS16_2CS = 0x50, + Z24S8_2CZ = 0x51, + Z24S8_MS2_2CZ = 0x52, + Z24S8_MS4_2CZ = 0x53, + Z24S8_MS8_2CZ = 0x54, + Z24S8_MS16_2CZ = 0x55, + Z24S8_4CSZV = 0x56, + Z24S8_MS2_4CSZV = 0x57, + Z24S8_MS4_4CSZV = 0x58, + Z24S8_MS8_4CSZV = 0x59, + Z24S8_MS16_4CSZV = 0x5a, + Z24V8_MS4_VC12 = 0x5b, + Z24V8_MS4_VC4 = 0x5c, + Z24V8_MS8_VC8 = 0x5d, + Z24V8_MS8_VC24 = 0x5e, + YUV_B8C1_2Y = 0x60, + YUV_B8C2_2Y = 0x61, + YUV_B10C1_2Y = 0x62, + YUV_B10C2_2Y = 0x6b, + YUV_B12C1_2Y = 0x6c, + YUV_B12C2_2Y = 0x6d, + Z24V8_MS4_VC12_1ZV = 0x63, + Z24V8_MS4_VC4_1ZV = 0x64, + Z24V8_MS8_VC8_1ZV = 0x65, + Z24V8_MS8_VC24_1ZV = 0x66, + Z24V8_MS4_VC12_2CS = 0x67, + Z24V8_MS4_VC4_2CS = 0x68, + Z24V8_MS8_VC8_2CS = 0x69, + Z24V8_MS8_VC24_2CS = 0x6a, + Z24V8_MS4_VC12_2CZV = 0x6f, + Z24V8_MS4_VC4_2CZV = 0x70, + Z24V8_MS8_VC8_2CZV = 0x71, + Z24V8_MS8_VC24_2CZV = 0x72, + Z24V8_MS4_VC12_2ZV = 0x73, + Z24V8_MS4_VC4_2ZV = 0x74, + Z24V8_MS8_VC8_2ZV = 0x75, + Z24V8_MS8_VC24_2ZV = 0x76, + Z24V8_MS4_VC12_4CSZV = 0x77, + Z24V8_MS4_VC4_4CSZV = 0x78, + Z24V8_MS8_VC8_4CSZV = 0x79, + Z24V8_MS8_VC24_4CSZV = 0x7a, + ZF32 = 0x7b, + ZF32_1Z = 0x7c, + ZF32_MS2_1Z = 0x7d, + ZF32_MS4_1Z = 0x7e, + ZF32_MS8_1Z = 0x7f, + ZF32_MS16_1Z = 0x80, + ZF32_2CS = 0x81, + ZF32_MS2_2CS = 0x82, + ZF32_MS4_2CS = 0x83, + ZF32_MS8_2CS = 0x84, + ZF32_MS16_2CS = 0x85, + ZF32_2CZ = 0x86, + ZF32_MS2_2CZ = 0x87, + ZF32_MS4_2CZ = 0x88, + ZF32_MS8_2CZ = 0x89, + ZF32_MS16_2CZ = 0x8a, + X8Z24_X16V8S8_MS4_VC12 = 0x8b, + X8Z24_X16V8S8_MS4_VC4 = 0x8c, + X8Z24_X16V8S8_MS8_VC8 = 0x8d, + X8Z24_X16V8S8_MS8_VC24 = 0x8e, + X8Z24_X16V8S8_MS4_VC12_1CS = 0x8f, + X8Z24_X16V8S8_MS4_VC4_1CS = 0x90, + X8Z24_X16V8S8_MS8_VC8_1CS = 0x91, + X8Z24_X16V8S8_MS8_VC24_1CS = 0x92, + X8Z24_X16V8S8_MS4_VC12_1ZV = 0x97, + X8Z24_X16V8S8_MS4_VC4_1ZV = 0x98, + X8Z24_X16V8S8_MS8_VC8_1ZV = 0x99, + X8Z24_X16V8S8_MS8_VC24_1ZV = 0x9a, + X8Z24_X16V8S8_MS4_VC12_1CZV = 0x9b, + X8Z24_X16V8S8_MS4_VC4_1CZV = 0x9c, + X8Z24_X16V8S8_MS8_VC8_1CZV = 0x9d, + X8Z24_X16V8S8_MS8_VC24_1CZV = 0x9e, + X8Z24_X16V8S8_MS4_VC12_2CS = 0x9f, + X8Z24_X16V8S8_MS4_VC4_2CS = 0xa0, + X8Z24_X16V8S8_MS8_VC8_2CS = 0xa1, + X8Z24_X16V8S8_MS8_VC24_2CS = 0xa2, + X8Z24_X16V8S8_MS4_VC12_2CSZV = 0xa3, + X8Z24_X16V8S8_MS4_VC4_2CSZV = 0xa4, + X8Z24_X16V8S8_MS8_VC8_2CSZV = 0xa5, + X8Z24_X16V8S8_MS8_VC24_2CSZV = 0xa6, + ZF32_X16V8S8_MS4_VC12 = 0xa7, + ZF32_X16V8S8_MS4_VC4 = 0xa8, + ZF32_X16V8S8_MS8_VC8 = 0xa9, + ZF32_X16V8S8_MS8_VC24 = 0xaa, + ZF32_X16V8S8_MS4_VC12_1CS = 0xab, + ZF32_X16V8S8_MS4_VC4_1CS = 0xac, + ZF32_X16V8S8_MS8_VC8_1CS = 0xad, + ZF32_X16V8S8_MS8_VC24_1CS = 0xae, + ZF32_X16V8S8_MS4_VC12_1ZV = 0xb3, + ZF32_X16V8S8_MS4_VC4_1ZV = 0xb4, + ZF32_X16V8S8_MS8_VC8_1ZV = 0xb5, + ZF32_X16V8S8_MS8_VC24_1ZV = 0xb6, + ZF32_X16V8S8_MS4_VC12_1CZV = 0xb7, + ZF32_X16V8S8_MS4_VC4_1CZV = 0xb8, + ZF32_X16V8S8_MS8_VC8_1CZV = 0xb9, + ZF32_X16V8S8_MS8_VC24_1CZV = 0xba, + ZF32_X16V8S8_MS4_VC12_2CS = 0xbb, + ZF32_X16V8S8_MS4_VC4_2CS = 0xbc, + ZF32_X16V8S8_MS8_VC8_2CS = 0xbd, + ZF32_X16V8S8_MS8_VC24_2CS = 0xbe, + ZF32_X16V8S8_MS4_VC12_2CSZV = 0xbf, + ZF32_X16V8S8_MS4_VC4_2CSZV = 0xc0, + ZF32_X16V8S8_MS8_VC8_2CSZV = 0xc1, + ZF32_X16V8S8_MS8_VC24_2CSZV = 0xc2, + ZF32_X24S8 = 0xc3, + ZF32_X24S8_1CS = 0xc4, + ZF32_X24S8_MS2_1CS = 0xc5, + ZF32_X24S8_MS4_1CS = 0xc6, + ZF32_X24S8_MS8_1CS = 0xc7, + ZF32_X24S8_MS16_1CS = 0xc8, + ZF32_X24S8_2CSZV = 0xce, + ZF32_X24S8_MS2_2CSZV = 0xcf, + ZF32_X24S8_MS4_2CSZV = 0xd0, + ZF32_X24S8_MS8_2CSZV = 0xd1, + ZF32_X24S8_MS16_2CSZV = 0xd2, + ZF32_X24S8_2CS = 0xd3, + ZF32_X24S8_MS2_2CS = 0xd4, + ZF32_X24S8_MS4_2CS = 0xd5, + ZF32_X24S8_MS8_2CS = 0xd6, + ZF32_X24S8_MS16_2CS = 0xd7, + S8 = 0x2a, + S8_2S = 0x2b, + GENERIC_16BX2 = 0xfe, + C32_2C = 0xd8, + C32_2CBR = 0xd9, + C32_2CBA = 0xda, + C32_2CRA = 0xdb, + C32_2BRA = 0xdc, + C32_MS2_2C = 0xdd, + C32_MS2_2CBR = 0xde, + C32_MS2_4CBRA = 0xcc, + C32_MS4_2C = 0xdf, + C32_MS4_2CBR = 0xe0, + C32_MS4_2CBA = 0xe1, + C32_MS4_2CRA = 0xe2, + C32_MS4_2BRA = 0xe3, + C32_MS4_4CBRA = 0x2c, + C32_MS8_MS16_2C = 0xe4, + C32_MS8_MS16_2CRA = 0xe5, + C64_2C = 0xe6, + C64_2CBR = 0xe7, + C64_2CBA = 0xe8, + C64_2CRA = 0xe9, + C64_2BRA = 0xea, + C64_MS2_2C = 0xeb, + C64_MS2_2CBR = 0xec, + C64_MS2_4CBRA = 0xcd, + C64_MS4_2C = 0xed, + C64_MS4_2CBR = 0xee, + C64_MS4_2CBA = 0xef, + C64_MS4_2CRA = 0xf0, + C64_MS4_2BRA = 0xf1, + C64_MS4_4CBRA = 0x2d, + C64_MS8_MS16_2C = 0xf2, + C64_MS8_MS16_2CRA = 0xf3, + C128_2C = 0xf4, + C128_2CR = 0xf5, + C128_MS2_2C = 0xf6, + C128_MS2_2CR = 0xf7, + C128_MS4_2C = 0xf8, + C128_MS4_2CR = 0xf9, + C128_MS8_MS16_2C = 0xfa, + C128_MS8_MS16_2CR = 0xfb, + X8C24 = 0xfc, + PITCH_NO_SWIZZLE = 0xfd, + SMSKED_MESSAGE = 0xca, + SMHOST_MESSAGE = 0xcb, +}; + +constexpr bool IsPitchKind(PTEKind kind) { + return kind == PTEKind::PITCH || kind == PTEKind::PITCH_NO_SWIZZLE; +} + +} // namespace Tegra diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index d2d40884c..1cbfef090 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -40,7 +40,7 @@ public: virtual ~RasterizerInterface() = default; /// Dispatches a draw invocation - virtual void Draw(bool is_indexed, bool is_instanced) = 0; + virtual void Draw(bool is_indexed, u32 instance_count) = 0; /// Clear the current framebuffer virtual void Clear() = 0; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index e5c09a969..1590b21de 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -205,7 +205,7 @@ void RasterizerOpenGL::Clear() { ++num_queued_commands; } -void RasterizerOpenGL::Draw(bool is_indexed, bool is_instanced) { +void RasterizerOpenGL::Draw(bool is_indexed, u32 instance_count) { MICROPROFILE_SCOPE(OpenGL_Drawing); SCOPE_EXIT({ gpu.TickWork(); }); @@ -222,14 +222,15 @@ void RasterizerOpenGL::Draw(bool is_indexed, bool is_instanced) { pipeline->SetEngine(maxwell3d, gpu_memory); pipeline->Configure(is_indexed); + BindInlineIndexBuffer(); + SyncState(); const GLenum primitive_mode = MaxwellToGL::PrimitiveTopology(maxwell3d->regs.draw.topology); BeginTransformFeedback(pipeline, primitive_mode); const GLuint base_instance = static_cast<GLuint>(maxwell3d->regs.global_base_instance_index); - const GLsizei num_instances = - static_cast<GLsizei>(is_instanced ? maxwell3d->mme_draw.instance_count : 1); + const GLsizei num_instances = static_cast<GLsizei>(instance_count); if (is_indexed) { const GLint base_vertex = static_cast<GLint>(maxwell3d->regs.global_base_vertex_index); const GLsizei num_vertices = static_cast<GLsizei>(maxwell3d->regs.index_buffer.count); @@ -1129,6 +1130,16 @@ void RasterizerOpenGL::ReleaseChannel(s32 channel_id) { query_cache.EraseChannel(channel_id); } +void RasterizerOpenGL::BindInlineIndexBuffer() { + if (maxwell3d->inline_index_draw_indexes.empty()) { + return; + } + const auto data_count = static_cast<u32>(maxwell3d->inline_index_draw_indexes.size()); + auto buffer = Buffer(buffer_cache_runtime, *this, 0, data_count); + buffer.ImmediateUpload(0, maxwell3d->inline_index_draw_indexes); + buffer_cache_runtime.BindIndexBuffer(buffer, 0, data_count); +} + AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_) : buffer_cache{buffer_cache_} {} bool AccelerateDMA::BufferCopy(GPUVAddr src_address, GPUVAddr dest_address, u64 amount) { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 45131b785..793e0d608 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -68,7 +68,7 @@ public: StateTracker& state_tracker_); ~RasterizerOpenGL() override; - void Draw(bool is_indexed, bool is_instanced) override; + void Draw(bool is_indexed, u32 instance_count) override; void Clear() override; void DispatchCompute() override; void ResetCounter(VideoCore::QueryType type) override; @@ -199,6 +199,8 @@ private: /// End a transform feedback void EndTransformFeedback(); + void BindInlineIndexBuffer(); + Tegra::GPU& gpu; const Device& device; diff --git a/src/video_core/renderer_opengl/gl_shader_cache.cpp b/src/video_core/renderer_opengl/gl_shader_cache.cpp index 609f0a772..e94cfdb1a 100644 --- a/src/video_core/renderer_opengl/gl_shader_cache.cpp +++ b/src/video_core/renderer_opengl/gl_shader_cache.cpp @@ -63,6 +63,7 @@ Shader::RuntimeInfo MakeRuntimeInfo(const GraphicsPipelineKey& key, Shader::RuntimeInfo info; if (previous_program) { info.previous_stage_stores = previous_program->info.stores; + info.previous_stage_legacy_stores_mapping = previous_program->info.legacy_stores_mapping; } else { // Mark all stores as available for vertex shaders info.previous_stage_stores.mask.set(); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 20f1d6584..13d5a1f67 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -134,6 +134,7 @@ Shader::RuntimeInfo MakeRuntimeInfo(std::span<const Shader::IR::Program> program Shader::RuntimeInfo info; if (previous_program) { info.previous_stage_stores = previous_program->info.stores; + info.previous_stage_legacy_stores_mapping = previous_program->info.legacy_stores_mapping; if (previous_program->is_geometry_passthrough) { info.previous_stage_stores.mask |= previous_program->info.passthrough.mask; } diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 7cb02631c..4b15c0f85 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -59,10 +59,11 @@ void QueryPool::Reserve(std::pair<VkQueryPool, u32> query) { std::find_if(pools.begin(), pools.end(), [query_pool = query.first](vk::QueryPool& pool) { return query_pool == *pool; }); - ASSERT(it != std::end(pools)); - const std::ptrdiff_t pool_index = std::distance(std::begin(pools), it); - usage[pool_index * GROW_STEP + static_cast<std::ptrdiff_t>(query.second)] = false; + if (it != std::end(pools)) { + const std::ptrdiff_t pool_index = std::distance(std::begin(pools), it); + usage[pool_index * GROW_STEP + static_cast<std::ptrdiff_t>(query.second)] = false; + } } QueryCache::QueryCache(VideoCore::RasterizerInterface& rasterizer_, const Device& device_, diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 47dfb45a1..6ab68892c 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -127,11 +127,10 @@ VkRect2D GetScissorState(const Maxwell& regs, size_t index, u32 up_scale = 1, u3 return scissor; } -DrawParams MakeDrawParams(const Maxwell& regs, u32 num_instances, bool is_instanced, - bool is_indexed) { +DrawParams MakeDrawParams(const Maxwell& regs, u32 num_instances, bool is_indexed) { DrawParams params{ .base_instance = regs.global_base_instance_index, - .num_instances = is_instanced ? num_instances : 1, + .num_instances = num_instances, .base_vertex = is_indexed ? regs.global_base_vertex_index : regs.vertex_buffer.first, .num_vertices = is_indexed ? regs.index_buffer.count : regs.vertex_buffer.count, .first_index = is_indexed ? regs.index_buffer.first : 0, @@ -177,7 +176,7 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra RasterizerVulkan::~RasterizerVulkan() = default; -void RasterizerVulkan::Draw(bool is_indexed, bool is_instanced) { +void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { MICROPROFILE_SCOPE(Vulkan_Drawing); SCOPE_EXIT({ gpu.TickWork(); }); @@ -194,13 +193,15 @@ void RasterizerVulkan::Draw(bool is_indexed, bool is_instanced) { pipeline->SetEngine(maxwell3d, gpu_memory); pipeline->Configure(is_indexed); + BindInlineIndexBuffer(); + BeginTransformFeedback(); UpdateDynamicStates(); const auto& regs{maxwell3d->regs}; - const u32 num_instances{maxwell3d->mme_draw.instance_count}; - const DrawParams draw_params{MakeDrawParams(regs, num_instances, is_instanced, is_indexed)}; + const u32 num_instances{instance_count}; + const DrawParams draw_params{MakeDrawParams(regs, num_instances, is_indexed)}; scheduler.Record([draw_params](vk::CommandBuffer cmdbuf) { if (draw_params.is_indexed) { cmdbuf.DrawIndexed(draw_params.num_vertices, draw_params.num_instances, @@ -304,14 +305,19 @@ void RasterizerVulkan::Clear() { } } - scheduler.Record([color_attachment, clear_value, clear_rect](vk::CommandBuffer cmdbuf) { - const VkClearAttachment attachment{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .colorAttachment = color_attachment, - .clearValue = clear_value, - }; - cmdbuf.ClearAttachments(attachment, clear_rect); - }); + if (regs.clear_surface.R && regs.clear_surface.G && regs.clear_surface.B && + regs.clear_surface.A) { + scheduler.Record([color_attachment, clear_value, clear_rect](vk::CommandBuffer cmdbuf) { + const VkClearAttachment attachment{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .colorAttachment = color_attachment, + .clearValue = clear_value, + }; + cmdbuf.ClearAttachments(attachment, clear_rect); + }); + } else { + UNIMPLEMENTED_MSG("Unimplemented Clear only the specified channel"); + } } if (!use_depth && !use_stencil) { @@ -1009,4 +1015,17 @@ void RasterizerVulkan::ReleaseChannel(s32 channel_id) { query_cache.EraseChannel(channel_id); } +void RasterizerVulkan::BindInlineIndexBuffer() { + if (maxwell3d->inline_index_draw_indexes.empty()) { + return; + } + const auto data_count = static_cast<u32>(maxwell3d->inline_index_draw_indexes.size()); + auto buffer = buffer_cache_runtime.UploadStagingBuffer(data_count); + std::memcpy(buffer.mapped_span.data(), maxwell3d->inline_index_draw_indexes.data(), data_count); + buffer_cache_runtime.BindIndexBuffer( + maxwell3d->regs.draw.topology, maxwell3d->regs.index_buffer.format, + maxwell3d->regs.index_buffer.first, maxwell3d->regs.index_buffer.count, buffer.buffer, + static_cast<u32>(buffer.offset), data_count); +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 4cde3c983..e2fdc7611 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -64,7 +64,7 @@ public: StateTracker& state_tracker_, Scheduler& scheduler_); ~RasterizerVulkan() override; - void Draw(bool is_indexed, bool is_instanced) override; + void Draw(bool is_indexed, u32 instance_count) override; void Clear() override; void DispatchCompute() override; void ResetCounter(VideoCore::QueryType type) override; @@ -141,6 +141,8 @@ private: void UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs); + void BindInlineIndexBuffer(); + Tegra::GPU& gpu; ScreenInfo& screen_info; diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index ad935d386..08aa8ca33 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -150,6 +150,8 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, return PixelFormat::D24_UNORM_S8_UINT; case Hash(TextureFormat::D32S8, FLOAT, UINT, UNORM, UNORM, LINEAR): return PixelFormat::D32_FLOAT_S8_UINT; + case Hash(TextureFormat::R32_B24G8, FLOAT, UINT, UNORM, UNORM, LINEAR): + return PixelFormat::D32_FLOAT_S8_UINT; case Hash(TextureFormat::BC1_RGBA, UNORM, LINEAR): return PixelFormat::BC1_RGBA_UNORM; case Hash(TextureFormat::BC1_RGBA, UNORM, SRGB): diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 0e0fd410f..8ef75fe73 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -442,7 +442,7 @@ void TextureCache<P>::WriteMemory(VAddr cpu_addr, size_t size) { template <class P> void TextureCache<P>::DownloadMemory(VAddr cpu_addr, size_t size) { std::vector<ImageId> images; - ForEachImageInRegion(cpu_addr, size, [this, &images](ImageId image_id, ImageBase& image) { + ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) { if (!image.IsSafeDownload()) { return; } @@ -1502,9 +1502,9 @@ void TextureCache<P>::UnregisterImage(ImageId image_id) { image.flags &= ~ImageFlagBits::BadOverlap; lru_cache.Free(image.lru_index); const auto& clear_page_table = - [this, image_id](u64 page, - std::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>& - selected_page_table) { + [image_id](u64 page, + std::unordered_map<u64, std::vector<ImageId>, Common::IdentityHash<u64>>& + selected_page_table) { const auto page_it = selected_page_table.find(page); if (page_it == selected_page_table.end()) { ASSERT_MSG(false, "Unregistering unregistered page=0x{:x}", page << YUZU_PAGEBITS); diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 15b9d4182..69a32819a 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -1661,8 +1661,8 @@ void Decompress(std::span<const uint8_t> data, uint32_t width, uint32_t height, for (u32 z = 0; z < depth; ++z) { const u32 depth_offset = z * height * width * 4; for (u32 y_index = 0; y_index < rows; ++y_index) { - auto decompress_stride = [data, width, height, depth, block_width, block_height, output, - rows, cols, z, depth_offset, y_index] { + auto decompress_stride = [data, width, height, block_width, block_height, output, rows, + cols, z, depth_offset, y_index] { const u32 y = y_index * block_height; for (u32 x_index = 0; x_index < cols; ++x_index) { const u32 block_index = (z * rows * cols) + (y_index * cols) + x_index; diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 52d067a2d..fd1a4b987 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -21,7 +21,7 @@ constexpr u32 pdep(u32 value) { u32 m = mask; for (u32 bit = 1; m; bit += bit) { if (value & bit) - result |= m & -m; + result |= m & (~m + 1); m &= m - 1; } return result; diff --git a/src/yuzu/applets/qt_controller.ui b/src/yuzu/applets/qt_controller.ui index c8cb6bcf3..f5eccba70 100644 --- a/src/yuzu/applets/qt_controller.ui +++ b/src/yuzu/applets/qt_controller.ui @@ -2300,7 +2300,7 @@ <item> <widget class="QRadioButton" name="radioUndocked"> <property name="text"> - <string>Undocked</string> + <string>Handheld</string> </property> </widget> </item> diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp index 24251247d..6acfb7b06 100644 --- a/src/yuzu/bootmanager.cpp +++ b/src/yuzu/bootmanager.cpp @@ -120,8 +120,8 @@ void EmuThread::run() { } } - // Shutdown the core emulation - system.Shutdown(); + // Shutdown the main emulated process + system.ShutdownMainProcess(); #if MICROPROFILE_ENABLED MicroProfileOnThreadExit(); diff --git a/src/yuzu/main.cpp b/src/yuzu/main.cpp index a94624be6..7b16d7f7e 100644 --- a/src/yuzu/main.cpp +++ b/src/yuzu/main.cpp @@ -294,6 +294,7 @@ GMainWindow::GMainWindow(std::unique_ptr<Config> config_, bool has_broken_vulkan #ifdef __linux__ SetupSigInterrupts(); #endif + system->Initialize(); Common::Log::Initialize(); LoadTranslation(); @@ -1895,6 +1896,8 @@ void GMainWindow::OnGameListOpenFolder(u64 program_id, GameListOpenTarget target case GameListOpenTarget::SaveData: { open_target = tr("Save Data"); const auto nand_dir = Common::FS::GetYuzuPath(Common::FS::YuzuPath::NANDDir); + auto vfs_nand_dir = + vfs->OpenDirectory(Common::FS::PathToUTF8String(nand_dir), FileSys::Mode::Read); if (has_user_save) { // User save data @@ -1921,15 +1924,15 @@ void GMainWindow::OnGameListOpenFolder(u64 program_id, GameListOpenTarget target ASSERT(user_id); const auto user_save_data_path = FileSys::SaveDataFactory::GetFullPath( - *system, FileSys::SaveDataSpaceId::NandUser, FileSys::SaveDataType::SaveData, - program_id, user_id->AsU128(), 0); + *system, vfs_nand_dir, FileSys::SaveDataSpaceId::NandUser, + FileSys::SaveDataType::SaveData, program_id, user_id->AsU128(), 0); path = Common::FS::ConcatPathSafe(nand_dir, user_save_data_path); } else { // Device save data const auto device_save_data_path = FileSys::SaveDataFactory::GetFullPath( - *system, FileSys::SaveDataSpaceId::NandUser, FileSys::SaveDataType::SaveData, - program_id, {}, 0); + *system, vfs_nand_dir, FileSys::SaveDataSpaceId::NandUser, + FileSys::SaveDataType::SaveData, program_id, {}, 0); path = Common::FS::ConcatPathSafe(nand_dir, device_save_data_path); } diff --git a/src/yuzu/multiplayer/state.cpp b/src/yuzu/multiplayer/state.cpp index ae2738ad4..285bb150d 100644 --- a/src/yuzu/multiplayer/state.cpp +++ b/src/yuzu/multiplayer/state.cpp @@ -268,7 +268,7 @@ bool MultiplayerState::OnCloseRoom() { return true; } // Save ban list - UISettings::values.multiplayer_ban_list = std::move(room->GetBanList()); + UISettings::values.multiplayer_ban_list = room->GetBanList(); room->Destroy(); announce_multiplayer_session->Stop(); diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp index fc2693f9d..6a91212e2 100644 --- a/src/yuzu/startup_checks.cpp +++ b/src/yuzu/startup_checks.cpp @@ -49,7 +49,7 @@ bool CheckEnvVars(bool* is_child) { *is_child = true; return false; } else if (!SetEnvironmentVariableA(IS_CHILD_ENV_VAR, ENV_VAR_ENABLED_TEXT)) { - std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %d\n", + std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %lu\n", IS_CHILD_ENV_VAR, GetLastError()); return true; } @@ -62,7 +62,7 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka // Set the startup variable for child processes const bool env_var_set = SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, ENV_VAR_ENABLED_TEXT); if (!env_var_set) { - std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %d\n", + std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %lu\n", STARTUP_CHECK_ENV_VAR, GetLastError()); return false; } @@ -81,22 +81,22 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka DWORD exit_code = STILL_ACTIVE; const int err = GetExitCodeProcess(process_info.hProcess, &exit_code); if (err == 0) { - std::fprintf(stderr, "GetExitCodeProcess failed with error %d\n", GetLastError()); + std::fprintf(stderr, "GetExitCodeProcess failed with error %lu\n", GetLastError()); } // Vulkan is broken if the child crashed (return value is not zero) *has_broken_vulkan = (exit_code != 0); if (CloseHandle(process_info.hProcess) == 0) { - std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError()); + std::fprintf(stderr, "CloseHandle failed with error %lu\n", GetLastError()); } if (CloseHandle(process_info.hThread) == 0) { - std::fprintf(stderr, "CloseHandle failed with error %d\n", GetLastError()); + std::fprintf(stderr, "CloseHandle failed with error %lu\n", GetLastError()); } } if (!SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, nullptr)) { - std::fprintf(stderr, "SetEnvironmentVariableA failed to clear %s with error %d\n", + std::fprintf(stderr, "SetEnvironmentVariableA failed to clear %s with error %lu\n", STARTUP_CHECK_ENV_VAR, GetLastError()); } @@ -135,7 +135,8 @@ bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags) { startup_info.cb = sizeof(startup_info); char p_name[255]; - std::strncpy(p_name, arg0, 255); + std::strncpy(p_name, arg0, 254); + p_name[254] = '\0'; const bool process_created = CreateProcessA(nullptr, // lpApplicationName p_name, // lpCommandLine @@ -149,7 +150,7 @@ bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags) { pi // lpProcessInformation ); if (!process_created) { - std::fprintf(stderr, "CreateProcessA failed with error %d\n", GetLastError()); + std::fprintf(stderr, "CreateProcessA failed with error %lu\n", GetLastError()); return false; } diff --git a/src/yuzu_cmd/yuzu.cpp b/src/yuzu_cmd/yuzu.cpp index 3a0f33cba..e16f79eb4 100644 --- a/src/yuzu_cmd/yuzu.cpp +++ b/src/yuzu_cmd/yuzu.cpp @@ -302,6 +302,8 @@ int main(int argc, char** argv) { } Core::System system{}; + system.Initialize(); + InputCommon::InputSubsystem input_subsystem{}; // Apply the command line arguments @@ -392,7 +394,7 @@ int main(int argc, char** argv) { } system.DetachDebugger(); void(system.Pause()); - system.Shutdown(); + system.ShutdownMainProcess(); detached_tasks.WaitForAllTasks(); return 0; |