Improve call_once performance by using Init­Once­Begin­Initialize / InitOnceComplete

stl/CMakeLists.txt

@@ -244,6 +244,7 @@ set(IMPLIB_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/src/sharedmutex.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/syserror_import_lib.cpp
     ${CMAKE_CURRENT_LIST_DIR}/src/vector_algorithms.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/src/xonce2.cpp
 )
 
 # The following files are linked in msvcp140[d][_clr].dll.

stl/inc/mutex

@@ -14,6 +14,7 @@
 #endif // _M_CEE_PURE
 
 #include <chrono>
+#include <exception>
 #include <system_error>
 #include <thread>
 #include <tuple>
@@ -543,24 +544,49 @@ int __stdcall _Callback_once(void*, void* _Pv, void**) { // adapt call_once() to
 }
 
 template <class _Fn, class... _Args>
-void(call_once)(once_flag& _Flag, _Fn&& _Fx, _Args&&... _Ax) { // call _Fx(_Ax...) once
-    using _Tuple = tuple<_Fn&&, _Args&&..., exception_ptr&>;
-    using _Seq   = make_index_sequence<1 + sizeof...(_Args)>;
+void(call_once)(once_flag& _Once, _Fn&& _Fx, _Args&&... _Ax) { // call _Fx(_Ax...) once
+    int _Pending;
+    bool _Fallback = false;
+    if (_Execute_once_begin(_Once, _Pending, _Fallback) == 0) {
+        if (_Fallback) {
+            using _Tuple = tuple<_Fn&&, _Args&&..., exception_ptr&>;
+            using _Seq   = make_index_sequence<1 + sizeof...(_Args)>;
 
-    exception_ptr _Exc;
-    _Tuple _Tup(_STD forward<_Fn>(_Fx), _STD forward<_Args>(_Ax)..., _Exc);
+            exception_ptr _Exc;
+            _Tuple _Tup(_STD forward<_Fn>(_Fx), _STD forward<_Args>(_Ax)..., _Exc);
 
-    _Execute_once_fp_t _Fp = &_Callback_once<_Tuple, _Seq, 1 + sizeof...(_Args)>;
+            _Execute_once_fp_t _Fp = &_Callback_once<_Tuple, _Seq, 1 + sizeof...(_Args)>;
 
-    if (_Execute_once(_Flag, _Fp, _STD addressof(_Tup)) != 0) {
-        return;
+            if (_Execute_once(_Once, _Fp, _STD addressof(_Tup)) != 0) {
+                return;
+            }
+
+            if (_Exc) {
+                _STD rethrow_exception(_Exc);
+            }
+        }
+        _XGetLastError();
     }
 
-    if (_Exc) {
-        _STD rethrow_exception(_Exc);
+    if (_Pending == 0) {
+        return; // already initialized (won the race otherwise)
     }
 
-    _XGetLastError();
+    struct _Execute_once_scope_t {
+        ~_Execute_once_scope_t() {
+            if (!_Execute_once_complete(_Once_flag, _Succeeded)) {
+                _STD terminate(); // Cannot throw, even if moved out of destructor.
+                // call_once is permited to throw if some condition prevented from calling the callback.
+            }
+        }
+
+        once_flag& _Once_flag;
+        bool _Succeeded;
+    } _Execute_once_scope = {_Once, false};
+
+    _STD invoke(_STD forward<_Fn>(_Fx), _STD forward<_Args>(_Ax)...);
+
+    _Execute_once_scope._Succeeded = true;
 }
 
 // condition_variable, timed_mutex, and recursive_timed_mutex are not supported under /clr

stl/inc/xcall_once.h

@@ -31,6 +31,8 @@ using _Execute_once_fp_t = int(__stdcall*)(void*, void*, void**);
 
 _CRTIMP2_PURE int __CLRCALL_PURE_OR_CDECL _Execute_once(
     once_flag& _Flag, _Execute_once_fp_t _Callback, void* _Pv) noexcept;
+int __CLRCALL_PURE_OR_CDECL _Execute_once_begin(once_flag& _Once_flag, int& _Pending, bool& _Fallback) noexcept;
+int __CLRCALL_PURE_OR_CDECL _Execute_once_complete(once_flag& _Once_flag, bool _Succeeded) noexcept;
 
 template <class _Ty>
 union _Immortalizer_impl { // constructs _Ty, never destroys

stl/msbuild/stl_base/stl.files.settings.targets

@@ -174,6 +174,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
             $(CrtRoot)\github\stl\src\sharedmutex.cpp;
             $(CrtRoot)\github\stl\src\syserror_import_lib.cpp;
             $(CrtRoot)\github\stl\src\vector_algorithms.cpp;
+            $(CrtRoot)\github\stl\src\xonce2.cpp;
             ">
             <BuildAs>nativecpp</BuildAs>
             <IncludeInLink>false</IncludeInLink>

stl/src/xonce2.cpp

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// __std_execute_once_begin and __std_execute_once_complete functions
+
+#include <atomic>
+#include <libloaderapi.h>
+#include <synchapi.h>
+#include <xcall_once.h>
+
+// This must be as small as possible, because its contents are
+// injected into the msvcprt.lib and msvcprtd.lib import libraries.
+// Do not include or define anything else here.
+// In particular, basic_string must not be included here.
+
+static_assert(sizeof(std::once_flag::_Opaque) == sizeof(INIT_ONCE), "invalid size");
+
+// these declarations must be in sync with those in xcall_once.h
+
+#if _STL_WIN32_WINNT >= _WIN32_WINNT_VISTA
+
+_STD_BEGIN
+int __CLRCALL_PURE_OR_CDECL __std_execute_once_begin(
+    once_flag& _Once_flag, int& _Pending, bool& _Fallback) noexcept { // wrap Win32 InitOnceBeginInitialize()
+    (void) _Fallback;
+    return ::InitOnceBeginInitialize(reinterpret_cast<PINIT_ONCE>(&_Once_flag._Opaque), 0, &_Pending, nullptr);
+}
+
+int __CLRCALL_PURE_OR_CDECL __std_execute_once_complete(
+    once_flag& _Once_flag, const unsigned long _Completion_flags) noexcept { // wrap Win32 InitOnceComplete()
+    return ::InitOnceComplete(reinterpret_cast<PINIT_ONCE>(&_Once_flag._Opaque), _Completion_flags, nullptr);
+}
+_STD_END
+
+#else // ^^^ _STL_WIN32_WINNT >= _WIN32_WINNT_VISTA / _STL_WIN32_WINNT < _WIN32_WINNT_VISTA vvv
+
+namespace {
+    struct _Init_once_vista_functions_t {
+        std::atomic<decltype(&::InitOnceBeginInitialize)> _Pfn_InitOnceBeginInitialize{};
+        std::atomic<decltype(&::InitOnceComplete)> _Pfn_InitOnceComplete{};
+        std::atomic<bool> _Initialized{};
+    };
+
+    const _Init_once_vista_functions_t& _Get_init_once_vista_functions() noexcept {
+        static _Init_once_vista_functions_t functions;
+        if (!functions._Initialized.load(std::memory_order_acquire)) {
+            HMODULE kernel_module              = ::GetModuleHandleW(L"Kernel32.dll");
+            FARPROC init_once_begin_initialize = ::GetProcAddress(kernel_module, "InitOnceBeginInitialize");
+            FARPROC init_once_complete         = ::GetProcAddress(kernel_module, "InitOnceComplete");
+            if (init_once_begin_initialize != nullptr && init_once_complete != nullptr) {
+                functions._Pfn_InitOnceBeginInitialize.store(
+                    reinterpret_cast<decltype(&::InitOnceBeginInitialize)>(init_once_begin_initialize),
+                    std::memory_order_relaxed);
+                functions._Pfn_InitOnceComplete.store(
+                    reinterpret_cast<decltype(&::InitOnceComplete)>(init_once_complete), std::memory_order_relaxed);
+            }
+            functions._Initialized.store(true, std::memory_order_release);
+        }
+        return functions;
+    }
+} // unnamed namespace
+
+_STD_BEGIN
+int __CLRCALL_PURE_OR_CDECL _Execute_once_begin(
+    once_flag& _Once_flag, int& _Pending, bool& _Fallback) noexcept { // wrap Win32 InitOnceBeginInitialize()
+    const auto init_once_begin_initialize =
+        _Get_init_once_vista_functions()._Pfn_InitOnceBeginInitialize.load(std::memory_order_relaxed);
+    if (init_once_begin_initialize == nullptr) {
+        _Fallback = true;
+        return false;
+    }
+    return init_once_begin_initialize(reinterpret_cast<PINIT_ONCE>(&_Once_flag._Opaque), 0, &_Pending, nullptr);
+}
+
+int __CLRCALL_PURE_OR_CDECL _Execute_once_complete(
+    once_flag& _Once_flag, const bool _Succeeded) noexcept { // wrap Win32 InitOnceComplete()
+    const auto init_once_complete =
+        _Get_init_once_vista_functions()._Pfn_InitOnceComplete.load(std::memory_order_relaxed);
+    return init_once_complete(
+        reinterpret_cast<PINIT_ONCE>(&_Once_flag._Opaque), _Succeeded ? 0 : INIT_ONCE_INIT_FAILED, nullptr);
+}
+_STD_END
+
+#endif //  _STL_WIN32_WINNT >= _WIN32_WINNT_VISTA