From a31c5cfdefd0b97acaf64d2667529a21ba023e45 Mon Sep 17 00:00:00 2001 From: Braydn Date: Mon, 22 Aug 2022 19:56:26 -0400 Subject: [PATCH] feat(server): Implemented periodic snapshotting (#161) Code cleanup & CONTRIBUTORS.md modifcation Signed-off-by: Braydn --- CONTRIBUTORS.md | 1 + src/server/server_family.cc | 128 ++++++++++++++++-------------------- src/server/server_family.h | 7 +- src/server/snapshot_test.cc | 65 ++++++++++-------- 4 files changed, 103 insertions(+), 98 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 71e7e0bb6ab1..4f2f1fe5d4e0 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -2,6 +2,7 @@ * **[Philipp Born](https://github.com/tamcore)** * Helm Chart +* **[Braydn Moore](https://github.com/braydnm)** * **[Ryan Russell](https://github.com/ryanrussell)** * Docs & Code Readability * **[Ali-Akber Saifee](https://github.com/alisaifee)** diff --git a/src/server/server_family.cc b/src/server/server_family.cc index 09a6d1e1958c..04ba51735fe9 100644 --- a/src/server/server_family.cc +++ b/src/server/server_family.cc @@ -13,6 +13,7 @@ #include #include +#include extern "C" { #include "redis/redis_aux.h" @@ -48,8 +49,7 @@ using namespace std; ABSL_FLAG(string, dir, "", "working directory"); ABSL_FLAG(string, dbfilename, "dump", "the filename to save/load the DB"); ABSL_FLAG(string, requirepass, "", "password for AUTH authentication"); -ABSL_FLAG(string, save_schedule, "", "glob spec for the time to save a snapshot which matches HH:MM 24h time"); -ABSL_FLAG(bool, save_schedule_use_utc, false, "use UTC when specifying the time to save a snapshot"); +ABSL_FLAG(string, save_schedule, "", "glob spec for the UTC time to save a snapshot which matches HH:MM 24h time"); ABSL_DECLARE_FLAG(uint32_t, port); ABSL_DECLARE_FLAG(bool, cache_mode); @@ -150,77 +150,63 @@ class LinuxWriteWrapper : public io::WriteFile { } // namespace -bool IsValidSaveScheduleNibble(string_view time, unsigned int max, unsigned int min_len = 2) { - size_t digit_mask = std::pow(10, time.size() - 1); - for (size_t i = 0; i < time.length(); ++i, digit_mask /= 10) { - // ignore wildcards as they are always valid for their placeholder - if (time[i] == '*') continue; - // if it is not a number and not a '*' invalid string - if (time[i] < '0' || time[i] > '9') return false; - // get the expected digits from both items - unsigned int digit = max / digit_mask; - unsigned int time_digit = time[i] - '0'; - // the validation only needs to continue as long as digit == time_digit - // take for example max 24, if time is 1x any x will still be less than the max - if (digit > time_digit) return true; - if (digit < time_digit) return false; - - max = max % digit_mask; +bool IsValidSaveScheduleNibble(string_view time, unsigned int max) { + /* + * a nibble is valid iff there exists one time that matches the pattern + * and that time is <= max. For any wildcard the minimum value is 0. + * Therefore the minimum time the pattern can match is the time with + * all *s replaced with 0s. If this time is > max all other times that + * match the pattern are > max and the pattern is invalid. Otherwise + * there exists at least one valid nibble specified by this pattern + * + * Note the edge case of "*" is equivalent to "**". While using this + * approach "*" and "**" both map to 0. + */ + unsigned int min_match = 0; + for (size_t i = 0; i < time.size(); ++i) { + min_match *= 10; + min_match += time[i] == '*' ? 0 : time[i] - '0'; } - return true; + return min_match <= max; } -bool IsValidSaveSchedule(string_view time) { - if (time.length() < 3 || time.length() > 5) return false; - - size_t separator_idx = 0; - while (separator_idx < 3 && time[separator_idx] != ':') ++separator_idx; +std::optional ParseSaveSchedule(string_view time) { + if (time.length() < 3 || time.length() > 5) return std::nullopt; + size_t separator_idx = time.find(':'); // the time cannot start with ':' and it must be present in the first 3 characters of any time - if (separator_idx == 3 || separator_idx == 0) return false; - - auto hour_view = string_view(time.data(), separator_idx); - auto min_view = string_view(time.data() + separator_idx + 1, time.length() - separator_idx - 1); + if (separator_idx == string::npos || separator_idx == 0 || separator_idx >= 3) return std::nullopt; - // any hour is >= 0 and <= 23 - if (hour_view.length() < 1 || hour_view.length() > 2) return false; + SnapshotSpec spec{string(time.substr(0, separator_idx)), string(time.substr(separator_idx + 1))}; // a minute should be 2 digits as it is zero padded, unless it is a '*' in which case this greedily can // make up both digits - if ((min_view.length() < 2 && min_view != "*") || min_view.length() > 2) return false; + if (spec.minute_spec != "*" && spec.minute_spec.length() != 2) return std::nullopt; - return IsValidSaveScheduleNibble(hour_view, 23, 1) - && IsValidSaveScheduleNibble(min_view, 59, 2); + return IsValidSaveScheduleNibble(spec.hour_spec, 23) && + IsValidSaveScheduleNibble(spec.minute_spec, 59) ? + std::optional(spec) : std::nullopt; } -bool DoesTimeMatchSpecifier(string_view::const_reverse_iterator begin, string_view::const_reverse_iterator end, unsigned int time) { +bool DoesTimeNibbleMatchSpecifier(string_view time_spec, unsigned int current_time) { // single greedy wildcard matches everything - if (*begin == '*' && begin + 1 == end) return true; - // otherwise start from the least significant digit of the string - // check if the current digit in the matcher is a wildcard or if it matches the digit specified - while (begin < end) { - if (*begin != '*' && *begin != '0' + (time % 10)) return false; - ++begin; - time /= 10; + if (time_spec == "*") return true; + // all times are specified by 2 digit numbers + for (int i = time_spec.length() - 1; i >= 0; --i) { + // if the current digit is not a wildcard and it does not match the digit in the current time it does not match + if (time_spec[i] != '*' && (current_time % 10) != (time_spec[i] - '0')) return false; + current_time /= 10; } - return true; + return current_time == 0; } -bool DoesTimeMatchSpecifier(string_view time, unsigned int hour, unsigned int min) { - std::string_view::const_iterator it; - for (it = time.begin(); it != time.end() && *it != ':'; ++it); - if (!DoesTimeMatchSpecifier(std::make_reverse_iterator(it), time.rend(), hour)) { - return false; - } - - ++it; - if (!DoesTimeMatchSpecifier(time.rbegin(), std::make_reverse_iterator(it), min)) { - return false; - } - - return true; -} +bool DoesTimeMatchSpecifier(const SnapshotSpec &spec, time_t now) { + unsigned hour = (now / 3600) % 24; + unsigned min = (now / 60) % 60; + return DoesTimeNibbleMatchSpecifier(spec.hour_spec, hour) && + DoesTimeNibbleMatchSpecifier(spec.minute_spec, min); +} ServerFamily::ServerFamily(Service* service) : service_(*service) { start_time_ = time(NULL); @@ -276,14 +262,16 @@ void ServerFamily::Init(util::AcceptServer* acceptor, util::ListenerInterface* m } string save_time = GetFlag(FLAGS_save_schedule); - if (!save_time.empty() && IsValidSaveSchedule(save_time)) { - snapshot_fiber_ = service_.proactor_pool().GetNextProactor()->LaunchFiber([save_time = std::move(save_time), this] { - SnapshotScheduling(std::move(save_time)); - }); - } - // if the argument is not empty it is an invalid format so print a warning - else if (!save_time.empty()) { - LOG(WARNING)<<"Invalid snapshot time specifier "< spec = ParseSaveSchedule(save_time); + if (spec) { + snapshot_fiber_ = service_.proactor_pool().GetNextProactor()->LaunchFiber([save_spec = std::move(spec.value()), this] { + SnapshotScheduling(std::move(save_spec)); + }); + } + else { + LOG(WARNING)<<"Invalid snapshot time specifier "<save_time; } - if ((last_save / 60) == (absl::ToTimeT(now) / 60)) { + if ((last_save / 60) == (now / 60)) { continue; } diff --git a/src/server/server_family.h b/src/server/server_family.h index 69d5933b9ec6..87962a29a4db 100644 --- a/src/server/server_family.h +++ b/src/server/server_family.h @@ -51,6 +51,11 @@ struct LastSaveInfo { std::vector> freq_map; // RDB_TYPE_xxx -> count mapping. }; +struct SnapshotSpec { + std::string hour_spec; + std::string minute_spec; +}; + class ServerFamily { public: ServerFamily(Service* service); @@ -117,7 +122,7 @@ class ServerFamily { void Load(const std::string& file_name); - void SnapshotScheduling(const std::string &&time); + void SnapshotScheduling(const SnapshotSpec &&time); boost::fibers::fiber load_fiber_, snapshot_fiber_; diff --git a/src/server/snapshot_test.cc b/src/server/snapshot_test.cc index d6bcc7d96a08..231b9d258b84 100644 --- a/src/server/snapshot_test.cc +++ b/src/server/snapshot_test.cc @@ -7,6 +7,9 @@ #include "base/gtest.h" #include "server/test_utils.h" +#include +#include + using namespace testing; using namespace std; using namespace util; @@ -19,53 +22,62 @@ class SnapshotTest : public Test { protected: }; -bool IsValidSaveSchedule(string_view time); -bool DoesTimeMatchSpecifier(string_view time, unsigned int hour, unsigned int min); +std::optional ParseSaveSchedule(string_view time); +bool DoesTimeMatchSpecifier(const SnapshotSpec&, time_t); + +bool DoesTimeMatchSpecifier(string_view time_spec, unsigned int hour, unsigned int min) { + auto spec = ParseSaveSchedule(time_spec); + if (!spec) return false; + + time_t now = ((hour * 60) + min) * 60; + + return DoesTimeMatchSpecifier(spec.value(), now); +} TEST_F(SnapshotTest, InvalidTimes) { - EXPECT_FALSE(IsValidSaveSchedule("24:00")); - EXPECT_FALSE(IsValidSaveSchedule("00:60")); - EXPECT_FALSE(IsValidSaveSchedule("100:00")); - EXPECT_FALSE(IsValidSaveSchedule("00:100")); + EXPECT_FALSE(ParseSaveSchedule("24:00")); + EXPECT_FALSE(ParseSaveSchedule("00:60")); + EXPECT_FALSE(ParseSaveSchedule("100:00")); + EXPECT_FALSE(ParseSaveSchedule("00:100")); // invalid times with regex - EXPECT_FALSE(IsValidSaveSchedule("23:6*")); + EXPECT_FALSE(ParseSaveSchedule("23:6*")); // Minutes must be zero padded - EXPECT_FALSE(IsValidSaveSchedule("00:9")); + EXPECT_FALSE(ParseSaveSchedule("00:9")); // No separators or start with separator - EXPECT_FALSE(IsValidSaveSchedule(":12")); - EXPECT_FALSE(IsValidSaveSchedule("1234")); - EXPECT_FALSE(IsValidSaveSchedule("1")); + EXPECT_FALSE(ParseSaveSchedule(":12")); + EXPECT_FALSE(ParseSaveSchedule("1234")); + EXPECT_FALSE(ParseSaveSchedule("1")); // Negative numbers / non numeric characters - EXPECT_FALSE(IsValidSaveSchedule("-1:-2")); - EXPECT_FALSE(IsValidSaveSchedule("12:34b")); + EXPECT_FALSE(ParseSaveSchedule("-1:-2")); + EXPECT_FALSE(ParseSaveSchedule("12:34b")); // Wildcards for full times - EXPECT_FALSE(IsValidSaveSchedule("12*:09")); - EXPECT_FALSE(IsValidSaveSchedule("23:45*")); + EXPECT_FALSE(ParseSaveSchedule("12*:09")); + EXPECT_FALSE(ParseSaveSchedule("23:45*")); } TEST_F(SnapshotTest, ValidTimes) { // Test endpoints - EXPECT_TRUE(IsValidSaveSchedule("23:59")); - EXPECT_TRUE(IsValidSaveSchedule("00:00")); + EXPECT_TRUE(ParseSaveSchedule("23:59")); + EXPECT_TRUE(ParseSaveSchedule("00:00")); // hours don't need to be zero padded - EXPECT_TRUE(IsValidSaveSchedule("0:00")); + EXPECT_TRUE(ParseSaveSchedule("0:00")); // wildcard checks - EXPECT_TRUE(IsValidSaveSchedule("1*:09")); - EXPECT_TRUE(IsValidSaveSchedule("*9:23")); - EXPECT_TRUE(IsValidSaveSchedule("23:*1")); - EXPECT_TRUE(IsValidSaveSchedule("18:1*")); + EXPECT_TRUE(ParseSaveSchedule("1*:09")); + EXPECT_TRUE(ParseSaveSchedule("*9:23")); + EXPECT_TRUE(ParseSaveSchedule("23:*1")); + EXPECT_TRUE(ParseSaveSchedule("18:1*")); // Greedy wildcards - EXPECT_TRUE(IsValidSaveSchedule("*:12")); - EXPECT_TRUE(IsValidSaveSchedule("9:*")); - EXPECT_TRUE(IsValidSaveSchedule("09:*")); - EXPECT_TRUE(IsValidSaveSchedule("*:*")); + EXPECT_TRUE(ParseSaveSchedule("*:12")); + EXPECT_TRUE(ParseSaveSchedule("9:*")); + EXPECT_TRUE(ParseSaveSchedule("09:*")); + EXPECT_TRUE(ParseSaveSchedule("*:*")); } TEST_F(SnapshotTest, TimeMatches) { @@ -74,6 +86,7 @@ TEST_F(SnapshotTest, TimeMatches) { EXPECT_TRUE(DoesTimeMatchSpecifier("2:04", 2, 4)); EXPECT_FALSE(DoesTimeMatchSpecifier("12:34", 2, 4)); + EXPECT_FALSE(DoesTimeMatchSpecifier("2:34", 12, 34)); EXPECT_FALSE(DoesTimeMatchSpecifier("2:34", 3, 34)); EXPECT_FALSE(DoesTimeMatchSpecifier("2:04", 3, 5));