Skip to content

Commit

Permalink
feat(server): Implemented periodic snapshotting (dragonflydb#161)
Browse files Browse the repository at this point in the history 
Code cleanup & CONTRIBUTORS.md modifcation

Signed-off-by: Braydn <[email protected]>
  • Loading branch information
@braydnm
braydnm committed Aug 23, 2022
1 parent bd3ba4c commit a31c5cf
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 98 deletions.
1 change: 1 addition & 0 deletions CONTRIBUTORS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

* **[Philipp Born](https://github.com/tamcore)**
* Helm Chart
* **[Braydn Moore](https://github.com/braydnm)**
* **[Ryan Russell](https://github.com/ryanrussell)**
* Docs & Code Readability
* **[Ali-Akber Saifee](https://github.com/alisaifee)**
128 changes: 57 additions & 71 deletions src/server/server_family.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include <chrono>
#include <filesystem>
#include <optional>

extern "C" {
#include "redis/redis_aux.h"
Expand Down Expand Up @@ -48,8 +49,7 @@ using namespace std;
ABSL_FLAG(string, dir, "", "working directory");
ABSL_FLAG(string, dbfilename, "dump", "the filename to save/load the DB");
ABSL_FLAG(string, requirepass, "", "password for AUTH authentication");
ABSL_FLAG(string, save_schedule, "", "glob spec for the time to save a snapshot which matches HH:MM 24h time");
ABSL_FLAG(bool, save_schedule_use_utc, false, "use UTC when specifying the time to save a snapshot");
ABSL_FLAG(string, save_schedule, "", "glob spec for the UTC time to save a snapshot which matches HH:MM 24h time");

ABSL_DECLARE_FLAG(uint32_t, port);
ABSL_DECLARE_FLAG(bool, cache_mode);
Expand Down Expand Up @@ -150,77 +150,63 @@ class LinuxWriteWrapper : public io::WriteFile {

} // namespace

bool IsValidSaveScheduleNibble(string_view time, unsigned int max, unsigned int min_len = 2) {
size_t digit_mask = std::pow(10, time.size() - 1);
for (size_t i = 0; i < time.length(); ++i, digit_mask /= 10) {
// ignore wildcards as they are always valid for their placeholder
if (time[i] == '*') continue;
// if it is not a number and not a '*' invalid string
if (time[i] < '0' || time[i] > '9') return false;
// get the expected digits from both items
unsigned int digit = max / digit_mask;
unsigned int time_digit = time[i] - '0';
// the validation only needs to continue as long as digit == time_digit
// take for example max 24, if time is 1x any x will still be less than the max
if (digit > time_digit) return true;
if (digit < time_digit) return false;

max = max % digit_mask;
bool IsValidSaveScheduleNibble(string_view time, unsigned int max) {
/*
* a nibble is valid iff there exists one time that matches the pattern
* and that time is <= max. For any wildcard the minimum value is 0.
* Therefore the minimum time the pattern can match is the time with
* all *s replaced with 0s. If this time is > max all other times that
* match the pattern are > max and the pattern is invalid. Otherwise
* there exists at least one valid nibble specified by this pattern
*
* Note the edge case of "*" is equivalent to "**". While using this
* approach "*" and "**" both map to 0.
*/
unsigned int min_match = 0;
for (size_t i = 0; i < time.size(); ++i) {
min_match *= 10;
min_match += time[i] == '*' ? 0 : time[i] - '0';
}

return true;
return min_match <= max;
}

bool IsValidSaveSchedule(string_view time) {
if (time.length() < 3 || time.length() > 5) return false;

size_t separator_idx = 0;
while (separator_idx < 3 && time[separator_idx] != ':') ++separator_idx;
std::optional<SnapshotSpec> ParseSaveSchedule(string_view time) {
if (time.length() < 3 || time.length() > 5) return std::nullopt;

size_t separator_idx = time.find(':');
// the time cannot start with ':' and it must be present in the first 3 characters of any time
if (separator_idx == 3 || separator_idx == 0) return false;

auto hour_view = string_view(time.data(), separator_idx);
auto min_view = string_view(time.data() + separator_idx + 1, time.length() - separator_idx - 1);
if (separator_idx == string::npos || separator_idx == 0 || separator_idx >= 3) return std::nullopt;

// any hour is >= 0 and <= 23
if (hour_view.length() < 1 || hour_view.length() > 2) return false;
SnapshotSpec spec{string(time.substr(0, separator_idx)), string(time.substr(separator_idx + 1))};
// a minute should be 2 digits as it is zero padded, unless it is a '*' in which case this greedily can
// make up both digits
if ((min_view.length() < 2 && min_view != "*") || min_view.length() > 2) return false;
if (spec.minute_spec != "*" && spec.minute_spec.length() != 2) return std::nullopt;

return IsValidSaveScheduleNibble(hour_view, 23, 1)
&& IsValidSaveScheduleNibble(min_view, 59, 2);
return IsValidSaveScheduleNibble(spec.hour_spec, 23) &&
IsValidSaveScheduleNibble(spec.minute_spec, 59) ?
std::optional<SnapshotSpec>(spec) : std::nullopt;
}

bool DoesTimeMatchSpecifier(string_view::const_reverse_iterator begin, string_view::const_reverse_iterator end, unsigned int time) {
bool DoesTimeNibbleMatchSpecifier(string_view time_spec, unsigned int current_time) {
// single greedy wildcard matches everything
if (*begin == '*' && begin + 1 == end) return true;
// otherwise start from the least significant digit of the string
// check if the current digit in the matcher is a wildcard or if it matches the digit specified
while (begin < end) {
if (*begin != '*' && *begin != '0' + (time % 10)) return false;
++begin;
time /= 10;
if (time_spec == "*") return true;
// all times are specified by 2 digit numbers
for (int i = time_spec.length() - 1; i >= 0; --i) {
// if the current digit is not a wildcard and it does not match the digit in the current time it does not match
if (time_spec[i] != '*' && (current_time % 10) != (time_spec[i] - '0')) return false;
current_time /= 10;
}

return true;
return current_time == 0;
}

bool DoesTimeMatchSpecifier(string_view time, unsigned int hour, unsigned int min) {
std::string_view::const_iterator it;
for (it = time.begin(); it != time.end() && *it != ':'; ++it);
if (!DoesTimeMatchSpecifier(std::make_reverse_iterator(it), time.rend(), hour)) {
return false;
}

++it;
if (!DoesTimeMatchSpecifier(time.rbegin(), std::make_reverse_iterator(it), min)) {
return false;
}

return true;
}
bool DoesTimeMatchSpecifier(const SnapshotSpec &spec, time_t now) {
unsigned hour = (now / 3600) % 24;
unsigned min = (now / 60) % 60;
return DoesTimeNibbleMatchSpecifier(spec.hour_spec, hour) &&
DoesTimeNibbleMatchSpecifier(spec.minute_spec, min);
}

ServerFamily::ServerFamily(Service* service) : service_(*service) {
start_time_ = time(NULL);
Expand Down Expand Up @@ -276,14 +262,16 @@ void ServerFamily::Init(util::AcceptServer* acceptor, util::ListenerInterface* m
}

string save_time = GetFlag(FLAGS_save_schedule);
if (!save_time.empty() && IsValidSaveSchedule(save_time)) {
snapshot_fiber_ = service_.proactor_pool().GetNextProactor()->LaunchFiber([save_time = std::move(save_time), this] {
SnapshotScheduling(std::move(save_time));
});
}
// if the argument is not empty it is an invalid format so print a warning
else if (!save_time.empty()) {
LOG(WARNING)<<"Invalid snapshot time specifier "<<save_time;
if (!save_time.empty()) {
std::optional<SnapshotSpec> spec = ParseSaveSchedule(save_time);
if (spec) {
snapshot_fiber_ = service_.proactor_pool().GetNextProactor()->LaunchFiber([save_spec = std::move(spec.value()), this] {
SnapshotScheduling(std::move(save_spec));
});
}
else {
LOG(WARNING)<<"Invalid snapshot time specifier "<<save_time;
}
}

is_running_ = true;
Expand Down Expand Up @@ -358,16 +346,14 @@ void ServerFamily::Load(const std::string& load_path) {
});
}

void ServerFamily::SnapshotScheduling(const string &&time) {
auto timezone = GetFlag(FLAGS_save_schedule_use_utc) ? absl::UTCTimeZone() : absl::LocalTimeZone();
void ServerFamily::SnapshotScheduling(const SnapshotSpec &&spec) {
while (is_running_) {
std::unique_lock lk(snapshot_mu_);
snapshot_timer_cv_.wait_for(lk, std::chrono::seconds(20));

absl::Time now = absl::Now();
absl::TimeZone::CivilInfo tz = timezone.At(now);

if (!DoesTimeMatchSpecifier(time, tz.cs.hour(), tz.cs.minute())) {
time_t now = std::time(NULL);

if (!DoesTimeMatchSpecifier(spec, now)) {
continue;
}

Expand All @@ -378,7 +364,7 @@ void ServerFamily::SnapshotScheduling(const string &&time) {
last_save = lsinfo_->save_time;
}

if ((last_save / 60) == (absl::ToTimeT(now) / 60)) {
if ((last_save / 60) == (now / 60)) {
continue;
}

Expand Down
7 changes: 6 additions & 1 deletion src/server/server_family.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ struct LastSaveInfo {
std::vector<std::pair<std::string_view, size_t>> freq_map; // RDB_TYPE_xxx -> count mapping.
};

struct SnapshotSpec {
std::string hour_spec;
std::string minute_spec;
};

class ServerFamily {
public:
ServerFamily(Service* service);
Expand Down Expand Up @@ -117,7 +122,7 @@ class ServerFamily {

void Load(const std::string& file_name);

void SnapshotScheduling(const std::string &&time);
void SnapshotScheduling(const SnapshotSpec &&time);

boost::fibers::fiber load_fiber_, snapshot_fiber_;

Expand Down
65 changes: 39 additions & 26 deletions src/server/snapshot_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#include "base/gtest.h"
#include "server/test_utils.h"

#include <iostream>
#include <chrono>

using namespace testing;
using namespace std;
using namespace util;
Expand All @@ -19,53 +22,62 @@ class SnapshotTest : public Test {
protected:
};

bool IsValidSaveSchedule(string_view time);
bool DoesTimeMatchSpecifier(string_view time, unsigned int hour, unsigned int min);
std::optional<SnapshotSpec> ParseSaveSchedule(string_view time);
bool DoesTimeMatchSpecifier(const SnapshotSpec&, time_t);

bool DoesTimeMatchSpecifier(string_view time_spec, unsigned int hour, unsigned int min) {
auto spec = ParseSaveSchedule(time_spec);
if (!spec) return false;

time_t now = ((hour * 60) + min) * 60;

return DoesTimeMatchSpecifier(spec.value(), now);
}

TEST_F(SnapshotTest, InvalidTimes) {
EXPECT_FALSE(IsValidSaveSchedule("24:00"));
EXPECT_FALSE(IsValidSaveSchedule("00:60"));
EXPECT_FALSE(IsValidSaveSchedule("100:00"));
EXPECT_FALSE(IsValidSaveSchedule("00:100"));
EXPECT_FALSE(ParseSaveSchedule("24:00"));
EXPECT_FALSE(ParseSaveSchedule("00:60"));
EXPECT_FALSE(ParseSaveSchedule("100:00"));
EXPECT_FALSE(ParseSaveSchedule("00:100"));

// invalid times with regex
EXPECT_FALSE(IsValidSaveSchedule("23:6*"));
EXPECT_FALSE(ParseSaveSchedule("23:6*"));

// Minutes must be zero padded
EXPECT_FALSE(IsValidSaveSchedule("00:9"));
EXPECT_FALSE(ParseSaveSchedule("00:9"));

// No separators or start with separator
EXPECT_FALSE(IsValidSaveSchedule(":12"));
EXPECT_FALSE(IsValidSaveSchedule("1234"));
EXPECT_FALSE(IsValidSaveSchedule("1"));
EXPECT_FALSE(ParseSaveSchedule(":12"));
EXPECT_FALSE(ParseSaveSchedule("1234"));
EXPECT_FALSE(ParseSaveSchedule("1"));

// Negative numbers / non numeric characters
EXPECT_FALSE(IsValidSaveSchedule("-1:-2"));
EXPECT_FALSE(IsValidSaveSchedule("12:34b"));
EXPECT_FALSE(ParseSaveSchedule("-1:-2"));
EXPECT_FALSE(ParseSaveSchedule("12:34b"));

// Wildcards for full times
EXPECT_FALSE(IsValidSaveSchedule("12*:09"));
EXPECT_FALSE(IsValidSaveSchedule("23:45*"));
EXPECT_FALSE(ParseSaveSchedule("12*:09"));
EXPECT_FALSE(ParseSaveSchedule("23:45*"));
}

TEST_F(SnapshotTest, ValidTimes) {
// Test endpoints
EXPECT_TRUE(IsValidSaveSchedule("23:59"));
EXPECT_TRUE(IsValidSaveSchedule("00:00"));
EXPECT_TRUE(ParseSaveSchedule("23:59"));
EXPECT_TRUE(ParseSaveSchedule("00:00"));
// hours don't need to be zero padded
EXPECT_TRUE(IsValidSaveSchedule("0:00"));
EXPECT_TRUE(ParseSaveSchedule("0:00"));

// wildcard checks
EXPECT_TRUE(IsValidSaveSchedule("1*:09"));
EXPECT_TRUE(IsValidSaveSchedule("*9:23"));
EXPECT_TRUE(IsValidSaveSchedule("23:*1"));
EXPECT_TRUE(IsValidSaveSchedule("18:1*"));
EXPECT_TRUE(ParseSaveSchedule("1*:09"));
EXPECT_TRUE(ParseSaveSchedule("*9:23"));
EXPECT_TRUE(ParseSaveSchedule("23:*1"));
EXPECT_TRUE(ParseSaveSchedule("18:1*"));

// Greedy wildcards
EXPECT_TRUE(IsValidSaveSchedule("*:12"));
EXPECT_TRUE(IsValidSaveSchedule("9:*"));
EXPECT_TRUE(IsValidSaveSchedule("09:*"));
EXPECT_TRUE(IsValidSaveSchedule("*:*"));
EXPECT_TRUE(ParseSaveSchedule("*:12"));
EXPECT_TRUE(ParseSaveSchedule("9:*"));
EXPECT_TRUE(ParseSaveSchedule("09:*"));
EXPECT_TRUE(ParseSaveSchedule("*:*"));
}

TEST_F(SnapshotTest, TimeMatches) {
Expand All @@ -74,6 +86,7 @@ TEST_F(SnapshotTest, TimeMatches) {
EXPECT_TRUE(DoesTimeMatchSpecifier("2:04", 2, 4));

EXPECT_FALSE(DoesTimeMatchSpecifier("12:34", 2, 4));
EXPECT_FALSE(DoesTimeMatchSpecifier("2:34", 12, 34));
EXPECT_FALSE(DoesTimeMatchSpecifier("2:34", 3, 34));
EXPECT_FALSE(DoesTimeMatchSpecifier("2:04", 3, 5));

Expand Down

0 comments on commit a31c5cf

Please sign in to comment.