diff --git a/CMakeLists.txt b/CMakeLists.txt index b10635e..a3c1388 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -295,7 +295,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) if (NOT ${SNMALLOC_CLEANUP} STREQUAL CXX11_DESTRUCTORS) check_linker_flag(CXX "-nostdlib++" SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) if (SNMALLOC_LINKER_SUPPORT_NOSTDLIBXX) - target_link_options(${name} PRIVATE -nostdlib++) + target_link_options(${name} PRIVATE -nostdlib++) else() set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C) endif() diff --git a/src/ds/flaglock.h b/src/ds/flaglock.h index 9b2458d..69ec307 100644 --- a/src/ds/flaglock.h +++ b/src/ds/flaglock.h @@ -2,6 +2,8 @@ #include "bits.h" +#include + namespace snmalloc { class FlagLock diff --git a/src/ds/helpers.h b/src/ds/helpers.h index f8ad92c..b8bc15b 100644 --- a/src/ds/helpers.h +++ b/src/ds/helpers.h @@ -4,6 +4,7 @@ #include "flaglock.h" #include +#include #include namespace snmalloc diff --git a/src/mem/corealloc.h b/src/mem/corealloc.h index 212d14f..ed97a6b 100644 --- a/src/mem/corealloc.h +++ b/src/mem/corealloc.h @@ -8,6 +8,7 @@ #include "pool.h" #include "remotecache.h" #include "sizeclasstable.h" +#include "ticker.h" namespace snmalloc { @@ -96,6 +97,11 @@ namespace snmalloc */ LocalCache* attached_cache; + /** + * Ticker to query the clock regularly at a lower cost. + */ + Ticker ticker; + /** * The message queue needs to be accessible from other threads * @@ -405,6 +411,7 @@ namespace snmalloc std::cout << "Slab is woken up" << std::endl; #endif + ticker.check_tick(); return; } @@ -419,6 +426,7 @@ namespace snmalloc { dealloc_local_slabs(sizeclass); } + ticker.check_tick(); } /** @@ -698,7 +706,8 @@ namespace snmalloc sl.insert(meta); } - return finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); + return ticker.check_tick(r); } return small_alloc_slow(sizeclass, fast_free_list, rsize); } @@ -766,7 +775,8 @@ namespace snmalloc alloc_classes[sizeclass].available.insert(meta); } - return finish_alloc(p, sizeclass); + auto r = finish_alloc(p, sizeclass); + return ticker.check_tick(r); } /** diff --git a/src/mem/ticker.h b/src/mem/ticker.h new file mode 100644 index 0000000..af397c6 --- /dev/null +++ b/src/mem/ticker.h @@ -0,0 +1,97 @@ +#pragma once + +#include "../ds/defines.h" + +#include + +namespace snmalloc +{ + /** + * This class will attempt to call the PAL every 50ms to check the time. + * If the caller of check_tick, does so more frequently, it will attempt + * to back-off to only query the time, every n calls to check_tick, where + * `n` adapts to the current frequency of calling. + * + * The aim is to reduce the time spent querying the time as this might be + * an expensive operation if time has been virtualised. + */ + template + class Ticker + { + /** + * Calls to check_tick required before the time is next queried + */ + uint64_t count_down = 1; + + /** + * Number of ticks next time we check the time. + * That is, + * counted - count_down + * Is how many ticks, since last_epoch_ms was updated. + */ + uint64_t counted = 1; + + /** + * Last time we queried the clock. + */ + uint64_t last_query_ms = 0; + + /** + * Slow path that actually queries clock and sets up + * how many calls for the next time we hit the slow path. + */ + template + SNMALLOC_SLOW_PATH T check_tick_slow(T p = nullptr) + { + uint64_t now_ms = PAL::time_in_ms(); + + // Set up clock. + if (last_query_ms == 0) + { + last_query_ms = now_ms; + count_down = 1; + counted = 1; + return p; + } + + uint64_t duration_ms = now_ms - last_query_ms; + last_query_ms = now_ms; + + // Check is below clock resolution + if (duration_ms == 0) + { + // Exponential back off + count_down = counted; + counted *= 2; + return p; + } + + constexpr size_t deadline_in_ms = 50; + + // Estimate number of ticks to get to the new deadline, based on the + // current interval + auto new_deadline_in_ticks = + ((1 + counted) * deadline_in_ms) / duration_ms; + + counted = new_deadline_in_ticks; + count_down = new_deadline_in_ticks; + + return p; + } + + public: + template + SNMALLOC_FAST_PATH T check_tick(T p = nullptr) + { + // Check before decrement, so that later calcations can use + // count_down == 0 for check on the next call. + // This is used if the ticks are way below the frequency of + // heart beat. + if (--count_down == 0) + { + return check_tick_slow(p); + } + return p; + } + }; +} // namespace snmalloc diff --git a/src/pal/pal_concept.h b/src/pal/pal_concept.h index aff9354..a476cce 100644 --- a/src/pal/pal_concept.h +++ b/src/pal/pal_concept.h @@ -3,7 +3,7 @@ #ifdef __cpp_concepts # include "../ds/concept.h" # include "pal_consts.h" - +# include "pal_ds.h" # include namespace snmalloc diff --git a/src/pal/pal_consts.h b/src/pal/pal_consts.h index 8497e33..537a7c8 100644 --- a/src/pal/pal_consts.h +++ b/src/pal/pal_consts.h @@ -3,6 +3,7 @@ #include "../ds/defines.h" #include +#include namespace snmalloc { @@ -85,70 +86,7 @@ namespace snmalloc /** * Default Tag ID for the Apple class */ - static const uint8_t PALAnonDefaultID = 241; - - /** - * This struct is used to represent callbacks for notification from the - * platform. It contains a next pointer as client is responsible for - * allocation as we cannot assume an allocator at this point. - */ - struct PalNotificationObject - { - std::atomic pal_next = nullptr; - - void (*pal_notify)(PalNotificationObject* self); - - PalNotificationObject(void (*pal_notify)(PalNotificationObject* self)) - : pal_notify(pal_notify) - {} - }; - - /*** - * Wrapper for managing notifications for PAL events - */ - class PalNotifier - { - /** - * List of callbacks to notify - */ - std::atomic callbacks{nullptr}; - - public: - /** - * Register a callback object to be notified - * - * The object should never be deallocated by the client after calling - * this. - */ - void register_notification(PalNotificationObject* callback) - { - callback->pal_next = nullptr; - - auto prev = &callbacks; - auto curr = prev->load(); - do - { - while (curr != nullptr) - { - prev = &(curr->pal_next); - curr = prev->load(); - } - } while (!prev->compare_exchange_weak(curr, callback)); - } - - /** - * Calls the pal_notify of all the registered objects. - */ - void notify_all() - { - PalNotificationObject* curr = callbacks; - while (curr != nullptr) - { - curr->pal_notify(curr); - curr = curr->pal_next; - } - } - }; + static const int PALAnonDefaultID = 241; /** * Query whether the PAL supports a specific feature. diff --git a/src/pal/pal_ds.h b/src/pal/pal_ds.h new file mode 100644 index 0000000..852d83c --- /dev/null +++ b/src/pal/pal_ds.h @@ -0,0 +1,163 @@ +#pragma once + +#include "../ds/defines.h" +#include "../ds/helpers.h" + +#include +#include + +namespace snmalloc +{ + template + class PalList + { + /** + * List of callbacks to notify + */ + std::atomic elements{nullptr}; + + static_assert( + std::is_same>::value, + "Required pal_next type."); + + public: + /** + * Add an element to the list + */ + void add(T* element) + { + auto prev = &elements; + auto curr = prev->load(); + do + { + while (curr != nullptr) + { + prev = &(curr->pal_next); + curr = prev->load(); + } + } while (!prev->compare_exchange_weak(curr, element)); + } + + /** + * Applies function to all the elements of the list + */ + void apply_all(function_ref func) + { + T* curr = elements; + while (curr != nullptr) + { + func(curr); + curr = curr->pal_next; + } + } + }; + + /** + * This struct is used to represent callbacks for notification from the + * platform. It contains a next pointer as client is responsible for + * allocation as we cannot assume an allocator at this point. + */ + struct PalNotificationObject + { + std::atomic pal_next = nullptr; + + void (*pal_notify)(PalNotificationObject* self); + + PalNotificationObject(void (*pal_notify)(PalNotificationObject* self)) + : pal_notify(pal_notify) + {} + }; + + /*** + * Wrapper for managing notifications for PAL events + */ + class PalNotifier + { + /** + * List of callbacks to notify + */ + PalList callbacks; + + public: + /** + * Register a callback object to be notified + * + * The object should never be deallocated by the client after calling + * this. + */ + void register_notification(PalNotificationObject* callback) + { + callbacks.add(callback); + } + + /** + * Calls the pal_notify of all the registered objects. + */ + void notify_all() + { + callbacks.apply_all([](auto curr) { curr->pal_notify(curr); }); + } + }; + + class PalTimerObject + { + friend class PalTimer; + template + friend class PalList; + + std::atomic pal_next; + + void (*pal_notify)(PalTimerObject* self); + + uint64_t last_run = 0; + uint64_t repeat; + + public: + PalTimerObject(void (*pal_notify)(PalTimerObject* self), uint64_t repeat) + : pal_notify(pal_notify), repeat(repeat) + {} + }; + + /** + * Simple mechanism for handling timers. + * + * Note: This is really designed for a very small number of timers, + * and this design should be changed if that is no longer the case. + */ + class PalTimer + { + /** + * List of callbacks to notify + */ + PalList timers; + + public: + /** + * Register a callback to be called every repeat milliseconds. + */ + void register_timer(PalTimerObject* timer) + { + timers.add(timer); + } + + void check(uint64_t time_ms) + { + static std::atomic_flag lock = ATOMIC_FLAG_INIT; + + // Depulicate calls into here, and make single threaded. + if (lock.test_and_set()) + { + timers.apply_all([time_ms](PalTimerObject* curr) { + if ( + (curr->last_run == 0) || + ((time_ms - curr->last_run) > curr->repeat)) + { + curr->last_run = time_ms; + curr->pal_notify(curr); + } + }); + lock.clear(); + } + } + }; +} // namespace snmalloc diff --git a/src/pal/pal_noalloc.h b/src/pal/pal_noalloc.h index a98add6..b6882c3 100644 --- a/src/pal/pal_noalloc.h +++ b/src/pal/pal_noalloc.h @@ -6,6 +6,7 @@ #include "../aal/aal.h" #include "pal_concept.h" #include "pal_consts.h" +#include "pal_timer_default.h" #include @@ -27,7 +28,7 @@ namespace snmalloc * ever use. */ template - struct PALNoAlloc + struct PALNoAlloc : public PalTimerDefaultImpl { /** * Bitmap of PalFeatures flags indicating the optional features that this diff --git a/src/pal/pal_plain.h b/src/pal/pal_plain.h index 9d8c8d9..57bfe18 100644 --- a/src/pal/pal_plain.h +++ b/src/pal/pal_plain.h @@ -1,13 +1,14 @@ #pragma once #include "../ds/bits.h" +#include "pal_timer_default.h" namespace snmalloc { // Can be extended // Will require a reserve method in subclasses. template - class PALPlainMixin : public State + class PALPlainMixin : public State, public PalTimerDefaultImpl { public: // Notify platform that we will not be using these pages diff --git a/src/pal/pal_posix.h b/src/pal/pal_posix.h index 3f3d163..e166bd2 100644 --- a/src/pal/pal_posix.h +++ b/src/pal/pal_posix.h @@ -4,6 +4,7 @@ # include #endif #include "../ds/address.h" +#include "pal_timer_default.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif @@ -39,7 +40,7 @@ namespace snmalloc * version. */ template - class PALPOSIX + class PALPOSIX : public PalTimerDefaultImpl> { /** * Helper class to access the `default_mmap_flags` field of `OS` if one @@ -348,5 +349,19 @@ namespace snmalloc } error("Entropy requested on platform that does not provide entropy"); } + + static uint64_t internal_time_in_ms() + { + auto hold = KeepErrno(); + + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) + { + error("Failed to get time"); + } + + return (static_cast(ts.tv_sec) * 1000) + + (static_cast(ts.tv_nsec) / 1000000); + } }; } // namespace snmalloc diff --git a/src/pal/pal_timer_default.h b/src/pal/pal_timer_default.h new file mode 100644 index 0000000..a38094a --- /dev/null +++ b/src/pal/pal_timer_default.h @@ -0,0 +1,32 @@ + +#pragma once + +#include "pal_consts.h" +#include "pal_ds.h" + +#include + +namespace snmalloc +{ + template + class PalTimerDefaultImpl + { + inline static PalTimer timers{}; + + public: + static uint64_t time_in_ms() + { + auto time = PalTime::internal_time_in_ms(); + + // Process timers + timers.check(time); + + return time; + } + + static void register_timer(PalTimerObject* timer) + { + timers.register_timer(timer); + } + }; +} \ No newline at end of file diff --git a/src/pal/pal_windows.h b/src/pal/pal_windows.h index 425810b..b7de864 100644 --- a/src/pal/pal_windows.h +++ b/src/pal/pal_windows.h @@ -2,6 +2,7 @@ #include "../ds/address.h" #include "../ds/bits.h" +#include "pal_timer_default.h" #ifdef _WIN32 # ifndef _MSC_VER @@ -22,9 +23,11 @@ # endif # endif +# include + namespace snmalloc { - class PALWindows + class PALWindows : public PalTimerDefaultImpl { /** * A flag indicating that we have tried to register for low-memory @@ -206,6 +209,14 @@ namespace snmalloc error("Failed to get entropy."); return result; } + + static uint64_t internal_time_in_ms() + { + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + } }; } #endif