diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c471dc7b..24c377484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log ## Next Build +* Abandon old config and migrate to gflags. ### Major Features and Improvements diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b2cb7275..8e454223e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -334,7 +334,6 @@ target_link_libraries(antlr_opencypher_parser_lib antlr4_static) # all memgraph src files set(memgraph_src_files - ${src_dir}/config/config.cpp ${src_dir}/dbms/dbms.cpp # ${src_dir}/dbms/cleaner.cpp ${src_dir}/utils/numerics/saturate.cpp @@ -370,6 +369,7 @@ set(memgraph_src_files ${src_dir}/database/graph_db_accessor.cpp ${src_dir}/data_structures/concurrent/skiplist_gc.cpp ${src_dir}/query/stripper.cpp + ${src_dir}/query/engine.cpp ${src_dir}/query/console.cpp ${src_dir}/query/frontend/ast/cypher_main_visitor.cpp ${src_dir}/query/typed_value.cpp diff --git a/config/compiler.yaml b/config/compiler.yaml deleted file mode 100644 index 2aa5b86d8..000000000 --- a/config/compiler.yaml +++ /dev/null @@ -1,28 +0,0 @@ -########################### -# MEMGRAPH DEFAULT CONFIG # -########################### - -# NOTE: all paths are relative to the run folder -# (where the executable is runned) - -# path to the codes which will be compiled -compile_path: "./compiled/" - -# path to the template (cpp) for codes generation -template_cpp_path: "./template/plan_template_cpp" - -# path to the folder with snapshots -snapshots_path: "snapshots" - -# cleaning cycle interval -# if set to -1 the GC will not run -cleaning_cycle_sec: "30" - -# snapshot cycle interval -snapshot_cycle_sec: "60" - -# max number of snapshots which will be kept on the disk at some point -max_retained_snapshots: "3" - -# by default query engine runs in interpret mode -interpret: false diff --git a/config/hardcoded_queries_memgraph.yaml b/config/hardcoded_queries_memgraph.yaml deleted file mode 100644 index 2aa5b86d8..000000000 --- a/config/hardcoded_queries_memgraph.yaml +++ /dev/null @@ -1,28 +0,0 @@ -########################### -# MEMGRAPH DEFAULT CONFIG # -########################### - -# NOTE: all paths are relative to the run folder -# (where the executable is runned) - -# path to the codes which will be compiled -compile_path: "./compiled/" - -# path to the template (cpp) for codes generation -template_cpp_path: "./template/plan_template_cpp" - -# path to the folder with snapshots -snapshots_path: "snapshots" - -# cleaning cycle interval -# if set to -1 the GC will not run -cleaning_cycle_sec: "30" - -# snapshot cycle interval -snapshot_cycle_sec: "60" - -# max number of snapshots which will be kept on the disk at some point -max_retained_snapshots: "3" - -# by default query engine runs in interpret mode -interpret: false diff --git a/config/interpreter.yaml b/config/interpreter.yaml deleted file mode 100644 index a815b8ff1..000000000 --- a/config/interpreter.yaml +++ /dev/null @@ -1,28 +0,0 @@ -########################### -# MEMGRAPH DEFAULT CONFIG # -########################### - -# NOTE: all paths are relative to the run folder -# (where the executable is runned) - -# path to the codes which will be compiled -compile_path: "./compiled/" - -# path to the template (cpp) for codes generation -template_cpp_path: "./template/plan_template_cpp" - -# path to the folder with snapshots -snapshots_path: "snapshots" - -# cleaning cycle interval -# if set to -1 the GC will not run -cleaning_cycle_sec: "30" - -# snapshot cycle interval -snapshot_cycle_sec: "60" - -# max number of snapshots which will be kept on the disk at some point -max_retained_snapshots: "3" - -# by default query engine runs in interpret mode -interpret: true diff --git a/config/memgraph.yaml b/config/testing similarity index 65% rename from config/memgraph.yaml rename to config/testing index aceb19d42..1538281c0 100644 --- a/config/memgraph.yaml +++ b/config/testing @@ -1,36 +1,34 @@ -########################### -# MEMGRAPH DEFAULT CONFIG # -########################### +# MEMGRAPH DEFAULT TESTING CONFIG # NOTE: all paths are relative to the run folder # (where the executable is runned) # path to the codes which will be compiled -compile_path: "./compiled/" +--compile_path="./compiled/" # path to the template (cpp) for codes generation -template_cpp_path: "./template/plan_template_cpp" +--template_cpp_path="./template/plan_template_cpp" # path to the folder with snapshots -snapshots_path: "snapshots" +--snapshots_path="snapshots" # cleaning cycle interval # if set to -1 the GC will not run -cleaning_cycle_sec: "30" +--cleaning_cycle_sec=30 # snapshot cycle interval # if set to -1 the snapshooter will not run -snapshot_cycle_sec: "-1" +--snapshot_cycle_sec=-1 # create snapshot disabled on db destruction -snapshot_db_destruction: false +--snapshot_db_destruction=false # max number of snapshots which will be kept on the disk at some point # if set to -1 the max number of snapshots is unlimited -max_retained_snapshots: "-1" +--max_retained_snapshots=-1 # by default query engine runs in interpret mode -interpret: true +--interpret=true # database recovering is disabled by default -recovery: false +--recovery=false diff --git a/src/config/config.hpp b/src/config/config.hpp deleted file mode 100644 index 8571d9e2e..000000000 --- a/src/config/config.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once - -#include "utils/config/config.hpp" - -#include -#include - -namespace config { - -// this class is used as a Definition class of config::Config class from utils -// number of elements should be small, -// it depends on implementation of config::Config class -// in other words number of fields in Definition class should be related -// to the number of config keys -class MemgraphConfig { - public: - static const char *env_config_key; - static const char *default_file_path; - static std::set arguments; -}; - -// -- all possible Memgraph's keys -- -constexpr const char *COMPILE_PATH = "compile_path"; -constexpr const char *TEMPLATE_CPP_PATH = "template_cpp_path"; -constexpr const char *SNAPSHOTS_PATH = "snapshots_path"; -constexpr const char *CLEANING_CYCLE_SEC = "cleaning_cycle_sec"; -constexpr const char *SNAPSHOT_CYCLE_SEC = "snapshot_cycle_sec"; -constexpr const char *SNAPSHOT_DB_DESTRUCTION = "snapshot_db_destruction"; -constexpr const char *MAX_RETAINED_SNAPSHOTS = "max_retained_snapshots"; -constexpr const char *INTERPRET = "interpret"; -constexpr const char *RECOVERY = "recovery"; -// -- all possible Memgraph's keys -- - -inline long long to_int(const std::string &s) { return stoll(s); } -// TODO: move this to register args because it doesn't make sense to convert -// str to bool for every lookup -inline bool to_bool(std::string &s) { - std::transform(s.begin(), s.end(), s.begin(), ::tolower); - std::istringstream is(s); - bool b; - is >> std::boolalpha >> b; - return b; -} -} - -// code uses this define for key access -// _KEY_ is value from all possible keys that are listed above -#define CONFIG_REGISTER_ARGS(ARGC, ARGV) \ - config::Config::instance().register_args(ARGC, ARGV); -#define CONFIG(_KEY_) config::Config::instance()[_KEY_] - -#define CONFIG_INTEGER(_KEY_) config::to_int(CONFIG(_KEY_)) -#define CONFIG_BOOL(_KEY_) config::to_bool(CONFIG(_KEY_)) diff --git a/src/database/graph_db.cpp b/src/database/graph_db.cpp index a61484f80..abf4b06de 100644 --- a/src/database/graph_db.cpp +++ b/src/database/graph_db.cpp @@ -1,6 +1,7 @@ #include -#include "config/config.hpp" +#include "gflags/gflags.h" + #include "database/creation_exception.hpp" #include "database/graph_db.hpp" #include "database/graph_db_accessor.hpp" @@ -9,22 +10,27 @@ #include "storage/edge.hpp" #include "storage/garbage_collector.hpp" -const int DEFAULT_CLEANING_CYCLE_SEC = 30; // 30 seconds -const std::string DEFAULT_SNAPSHOT_FOLDER = "snapshots"; -const int DEFAULT_MAX_RETAINED_SNAPSHOTS = -1; // unlimited number of snapshots -const int DEFAULT_SNAPSHOT_CYCLE_SEC = -1; // off +DEFINE_int32(GC_CYCLE_SEC, 30, + "Amount of time between starts of two cleaning cycles in seconds. " + "-1 to turn off."); +DEFINE_int32(MAX_RETAINED_SNAPSHOTS, -1, + "Number of retained snapshots, -1 means without limit."); +DEFINE_int32(SNAPSHOT_CYCLE_SEC, -1, + "Amount of time between starts of two snapshooters in seconds. -1 " + "to turn off."); +DEFINE_bool(SNAPSHOT_ON_DB_DESTRUCTION, false, + "Snapshot on database destruction."); + +DECLARE_string(SNAPSHOT_DIRECTORY); GraphDb::GraphDb(const std::string &name, const fs::path &snapshot_db_dir) : name_(name), gc_vertices_(vertices_, vertex_record_deleter_, vertex_version_list_deleter_), gc_edges_(edges_, edge_record_deleter_, edge_version_list_deleter_) { - const std::string time_str = CONFIG(config::CLEANING_CYCLE_SEC); - int pause = DEFAULT_CLEANING_CYCLE_SEC; - if (!time_str.empty()) pause = CONFIG_INTEGER(config::CLEANING_CYCLE_SEC); // Pause of -1 means we shouldn't run the GC. - if (pause != -1) { - gc_scheduler_.Run(std::chrono::seconds(pause), [this]() { + if (FLAGS_GC_CYCLE_SEC != -1) { + gc_scheduler_.Run(std::chrono::seconds(FLAGS_GC_CYCLE_SEC), [this]() { // main garbage collection logic // see wiki documentation for logic explanation const auto next_id = this->tx_engine.count() + 1; @@ -51,36 +57,17 @@ GraphDb::GraphDb(const std::string &name, const fs::path &snapshot_db_dir) RecoverDatabase(snapshot_db_dir); StartSnapshooting(); - } void GraphDb::StartSnapshooting() { - const std::string max_retained_snapshots_str = - CONFIG(config::MAX_RETAINED_SNAPSHOTS); - const std::string snapshot_cycle_sec_str = - CONFIG(config::SNAPSHOT_CYCLE_SEC); - const std::string snapshot_folder_str = CONFIG(config::SNAPSHOTS_PATH); - - max_retained_snapshots_ = DEFAULT_MAX_RETAINED_SNAPSHOTS; - if (!max_retained_snapshots_str.empty()) - max_retained_snapshots_ = CONFIG_INTEGER(config::MAX_RETAINED_SNAPSHOTS); - - snapshot_cycle_sec_ = DEFAULT_SNAPSHOT_CYCLE_SEC; - if (!snapshot_cycle_sec_str.empty()) - snapshot_cycle_sec_ = CONFIG_INTEGER(config::SNAPSHOT_CYCLE_SEC); - - snapshot_folder_ = DEFAULT_SNAPSHOT_FOLDER; - if (!snapshot_folder_str.empty()) snapshot_folder_ = snapshot_folder_str; - - snapshot_db_destruction_ = CONFIG_BOOL(config::SNAPSHOT_DB_DESTRUCTION); - - if (snapshot_cycle_sec_ != -1) { + if (FLAGS_SNAPSHOT_CYCLE_SEC != -1) { auto create_snapshot = [this]() -> void { GraphDbAccessor db_accessor(*this); - snapshooter_.MakeSnapshot(db_accessor, fs::path(snapshot_folder_) / name_, - max_retained_snapshots_); + snapshooter_.MakeSnapshot(db_accessor, + fs::path(FLAGS_SNAPSHOT_DIRECTORY) / name_, + FLAGS_MAX_RETAINED_SNAPSHOTS); }; - snapshot_creator_.Run(std::chrono::seconds(snapshot_cycle_sec_), + snapshot_creator_.Run(std::chrono::seconds(FLAGS_SNAPSHOT_CYCLE_SEC), create_snapshot); } } @@ -110,10 +97,11 @@ GraphDb::~GraphDb() { snapshot_creator_.Stop(); // Create last database snapshot - if (snapshot_db_destruction_) { + if (FLAGS_SNAPSHOT_ON_DB_DESTRUCTION == true) { GraphDbAccessor db_accessor(*this); - snapshooter_.MakeSnapshot(db_accessor, fs::path(snapshot_folder_) / name_, - max_retained_snapshots_); + snapshooter_.MakeSnapshot(db_accessor, + fs::path(FLAGS_SNAPSHOT_DIRECTORY) / name_, + FLAGS_MAX_RETAINED_SNAPSHOTS); } // Delete vertices and edges which weren't collected before, also deletes diff --git a/src/database/graph_db.hpp b/src/database/graph_db.hpp index 85138d8f3..643f76181 100644 --- a/src/database/graph_db.hpp +++ b/src/database/graph_db.hpp @@ -104,10 +104,6 @@ class GraphDb { // snapshooter Snapshooter snapshooter_; - std::string snapshot_folder_; - int max_retained_snapshots_; - int snapshot_cycle_sec_; - bool snapshot_db_destruction_; // Schedulers Scheduler gc_scheduler_; diff --git a/src/dbms/dbms.cpp b/src/dbms/dbms.cpp index 2bc8e3acc..608719584 100644 --- a/src/dbms/dbms.cpp +++ b/src/dbms/dbms.cpp @@ -1,5 +1,11 @@ +#include "gflags/gflags.h" + #include "dbms/dbms.hpp" +DEFINE_string(SNAPSHOT_DIRECTORY, "snapshots", + "Relative path to directory in which to save snapshots."); +DEFINE_bool(RECOVER_ON_STARTUP, false, "Recover database on startup."); + std::unique_ptr Dbms::active() { return std::make_unique( *active_db.load(std::memory_order_acquire)); diff --git a/src/dbms/dbms.hpp b/src/dbms/dbms.hpp index 90c34613b..a03c42b1f 100644 --- a/src/dbms/dbms.hpp +++ b/src/dbms/dbms.hpp @@ -4,27 +4,28 @@ #include #include -#include "config/config.hpp" +#include "gflags/gflags.h" + #include "data_structures/concurrent/concurrent_map.hpp" #include "database/graph_db.hpp" #include "database/graph_db_accessor.hpp" #include "durability/recovery.hpp" -//#include "dbms/cleaner.hpp" +DECLARE_string(SNAPSHOT_DIRECTORY); +DECLARE_bool(RECOVER_ON_STARTUP); namespace fs = std::experimental::filesystem; -const std::string DEFAULT_SNAPSHOT_FOLDER = "snapshots"; - class Dbms { public: Dbms() { - if (CONFIG_BOOL(config::RECOVERY)) { - auto accessor = dbs.access(); - std::string snapshot_folder = CONFIG(config::SNAPSHOTS_PATH); - if (snapshot_folder.empty()) snapshot_folder = DEFAULT_SNAPSHOT_FOLDER; - for (auto &snapshot_db : fs::directory_iterator(snapshot_folder)) { - // create db and set it active - active(snapshot_db.path().filename(), snapshot_db); + if (FLAGS_RECOVER_ON_STARTUP) { + if (fs::exists(fs::path(FLAGS_SNAPSHOT_DIRECTORY))) { + auto accessor = dbs.access(); + for (auto &snapshot_db : + fs::directory_iterator(FLAGS_SNAPSHOT_DIRECTORY)) { + // create db and set it active + active(snapshot_db.path().filename(), snapshot_db); + } } } diff --git a/src/durability/snapshooter.cpp b/src/durability/snapshooter.cpp index 50faaf07b..ab63126c6 100644 --- a/src/durability/snapshooter.cpp +++ b/src/durability/snapshooter.cpp @@ -1,11 +1,12 @@ -#include "durability/snapshooter.hpp" #include + #include "communication/bolt/v1/encoder/base_encoder.hpp" -#include "config/config.hpp" #include "database/graph_db_accessor.hpp" #include "durability/file_writer_buffer.hpp" #include "utils/datetime/timestamp.hpp" +#include "durability/snapshooter.hpp" + bool Snapshooter::MakeSnapshot(GraphDbAccessor &db_accessor_, const fs::path &snapshot_folder, const int max_retained_snapshots) { diff --git a/src/memgraph_bolt.cpp b/src/memgraph_bolt.cpp index 317aada9f..316a47e4a 100644 --- a/src/memgraph_bolt.cpp +++ b/src/memgraph_bolt.cpp @@ -17,7 +17,6 @@ #include "logging/default.hpp" #include "logging/streams/stdout.hpp" -#include "utils/config/config.hpp" #include "utils/signals/handler.hpp" #include "utils/stacktrace/log.hpp" #include "utils/terminate_handler.hpp" @@ -46,9 +45,50 @@ void throw_and_stacktace(std::string message) { logger.info(stacktrace.dump()); } +// Load flags in this order, the last one has the highest priority: +// 1) /etc/memgraph/config +// 2) ~/.memgraph/config +// 3) env - MEMGRAPH_CONFIG +// 4) command line flags + +void load_config(int &argc, char **&argv) { + std::vector configs = {fs::path("/etc/memgraph/config")}; + if (getenv("HOME") != nullptr) + configs.emplace_back(fs::path(getenv("HOME")) / + fs::path(".memgraph/config")); + if (getenv("MEMGRAPH_CONFIG") != nullptr) + configs.emplace_back(fs::path(getenv("MEMGRAPH_CONFIG"))); + + std::vector flagfile_arguments; + for (const auto &config : configs) + if (fs::exists(config)) { + flagfile_arguments.emplace_back( + std::string("--flagfile=" + config.generic_string())); + } + + int custom_argc = static_cast(flagfile_arguments.size()) + 1; + char **custom_argv = new char *[custom_argc]; + + custom_argv[0] = strdup(std::string("memgraph").c_str()); + for (int i = 0; i < (int)flagfile_arguments.size(); ++i) { + custom_argv[i + 1] = strdup(flagfile_arguments[i].c_str()); + } + + // setup flags from config flags + gflags::ParseCommandLineFlags(&custom_argc, &custom_argv, false); + + // unconsumed arguments have to be freed to avoid memory leak since they are + // strdup-ed. + for (int i = 0; i < custom_argc; ++i) free(custom_argv[i]); + delete[] custom_argv; + + // setup flags from command line + gflags::ParseCommandLineFlags(&argc, &argv, true); +} + int main(int argc, char **argv) { fs::current_path(fs::path(argv[0]).parent_path()); - gflags::ParseCommandLineFlags(&argc, &argv, true); + load_config(argc, argv); // Logging init. #ifdef SYNC_LOGGER logging::init_sync(); @@ -78,9 +118,6 @@ int main(int argc, char **argv) { std::exit(EXIT_FAILURE); }); - // Register args. - CONFIG_REGISTER_ARGS(argc, argv); - // Initialize endpoint. endpoint_t endpoint; try { diff --git a/src/query/engine.cpp b/src/query/engine.cpp new file mode 100644 index 000000000..b1b66770b --- /dev/null +++ b/src/query/engine.cpp @@ -0,0 +1,6 @@ +#include "query/engine.hpp" + +DEFINE_bool(INTERPRET, true, + "Use interpretor instead of compiler for query execution."); +DEFINE_string(COMPILE_DIRECTORY, "./compiled/", + "Directory in which to write compiled libraries."); diff --git a/src/query/engine.hpp b/src/query/engine.hpp index f309a3600..a8ef23f7a 100644 --- a/src/query/engine.hpp +++ b/src/query/engine.hpp @@ -3,7 +3,6 @@ #include namespace fs = std::experimental::filesystem; -#include "config/config.hpp" #include "data_structures/concurrent/concurrent_map.hpp" #include "database/graph_db.hpp" #include "logging/loggable.hpp" @@ -15,6 +14,9 @@ namespace fs = std::experimental::filesystem; #include "query/preprocessor.hpp" #include "utils/dynamic_lib.hpp" +DECLARE_bool(INTERPRET); +DECLARE_string(COMPILE_DIRECTORY); + /** * Responsible for query execution. * @@ -64,7 +66,7 @@ class QueryEngine : public Loggable { */ auto Run(const std::string &query, GraphDbAccessor &db_accessor, Stream &stream) { - if (CONFIG_BOOL(config::INTERPRET)) { + if (FLAGS_INTERPRET) { query::Interpret(query, db_accessor, stream); return true; } @@ -155,13 +157,13 @@ class QueryEngine : public Loggable { return query_plan_it->second->instance(); // find hardcoded query plan if exists - auto hardcoded_path = fs::path(CONFIG(config::COMPILE_PATH) + "hardcode/" + + auto hardcoded_path = fs::path(FLAGS_COMPILE_DIRECTORY + "hardcode/" + std::to_string(stripped.hash) + ".cpp"); if (fs::exists(hardcoded_path)) return LoadCpp(hardcoded_path, stripped.hash); // generate query plan - auto generated_path = fs::path(CONFIG(config::COMPILE_PATH) + + auto generated_path = fs::path(FLAGS_COMPILE_DIRECTORY + std::to_string(stripped.hash) + ".cpp"); query::frontend::opencypher::Parser parser(stripped.query); @@ -194,7 +196,7 @@ class QueryEngine : public Loggable { // and that is a problem because at this point we want brand new // dynamic lib. That is the tmp solution. The right solution would be // to deal with this problem in DynamicLib - auto path_so = CONFIG(config::COMPILE_PATH) + std::to_string(hash) + "_" + + auto path_so = FLAGS_COMPILE_DIRECTORY + std::to_string(hash) + "_" + (std::string)Timestamp::now() + ".so"; plan_compiler.Compile(path_cpp, path_so); diff --git a/tests/integration/query_engine.cpp b/tests/integration/query_engine.cpp index 95b31ea38..bec85d075 100644 --- a/tests/integration/query_engine.cpp +++ b/tests/integration/query_engine.cpp @@ -2,10 +2,12 @@ #include "gflags/gflags.h" -#include "config/config.hpp" #include "dbms/dbms.hpp" #include "query_engine_common.hpp" +DECLARE_bool(INTERPRET); +DECLARE_string(COMPILE_DIRECTORY); + using namespace std::chrono_literals; using namespace tests::integration; @@ -30,10 +32,10 @@ int main(int argc, char *argv[]) { auto log = init_logging("IntegrationQueryEngine"); // Manually set config compile_path to avoid loading whole config file with // the test. - CONFIG(config::COMPILE_PATH) = "../compiled/"; + FLAGS_COMPILE_DIRECTORY = "../compiled/"; // Set the interpret to false to avoid calling the interpreter which doesn't // support all the queries yet. - CONFIG(config::INTERPRET) = "false"; + FLAGS_INTERPRET = false; Dbms dbms; StreamT stream(std::cout); QueryEngineT query_engine; diff --git a/tests/manual/console_test.cpp b/tests/manual/console_test.cpp index 6714e9a1d..9320f3953 100644 --- a/tests/manual/console_test.cpp +++ b/tests/manual/console_test.cpp @@ -1,5 +1,7 @@ #include +#include "gflags/gflags.h" + #include "dbms/dbms.hpp" #include "query/console.hpp" #include "query/interpreter.hpp" @@ -24,12 +26,12 @@ void random_generate(Dbms &dbms, uint node_count, int edge_factor = 5) { } int main(int argc, char *argv[]) { - REGISTER_ARGS(argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); // parse the first cmd line argument as the count of nodes to randomly create uint node_count = 0; if (argc > 1) { - node_count = (uint) std::stoul(argv[1]); + node_count = (uint)std::stoul(argv[1]); permanent_assert(node_count < 10000000, "More then 10M nodes requested, that's too much"); } diff --git a/tests/unit/bolt_session.cpp b/tests/unit/bolt_session.cpp index 70fcdb47e..9eae29b98 100644 --- a/tests/unit/bolt_session.cpp +++ b/tests/unit/bolt_session.cpp @@ -1,18 +1,19 @@ -#include "bolt_common.hpp" +#include "gflags/gflags.h" +#include "bolt_common.hpp" #include "communication/bolt/v1/encoder/result_stream.hpp" #include "communication/bolt/v1/session.hpp" -#include "config/config.hpp" #include "query/engine.hpp" +DECLARE_bool(INTERPRET); // Shortcuts for writing variable initializations in tests -#define INIT_VARS Dbms dbms;\ - TestSocket socket(10);\ - QueryEngine query_engine;\ - SessionT session(std::move(socket), dbms, query_engine);\ - std::vector &output = session.socket_.output; - +#define INIT_VARS \ + Dbms dbms; \ + TestSocket socket(10); \ + QueryEngine query_engine; \ + SessionT session(std::move(socket), dbms, query_engine); \ + std::vector &output = session.socket_.output; using ResultStreamT = communication::bolt::ResultStream; using StateT = communication::bolt::State; - // Sample testdata that has correct inputs and outputs. -const uint8_t handshake_req[] = { - 0x60, 0x60, 0xb0, 0x17, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t handshake_req[] = {0x60, 0x60, 0xb0, 0x17, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; const uint8_t handshake_resp[] = {0x00, 0x00, 0x00, 0x01}; const uint8_t init_req[] = { - 0xb2, 0x01, 0xd0, 0x15, 0x6c, 0x69, 0x62, 0x6e, 0x65, 0x6f, 0x34, 0x6a, - 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x31, 0x2e, 0x32, 0x2e, - 0x31, 0xa3, 0x86, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, 0x85, 0x62, 0x61, - 0x73, 0x69, 0x63, 0x89, 0x70, 0x72, 0x69, 0x6e, 0x63, 0x69, 0x70, 0x61, - 0x6c, 0x80, 0x8b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x80}; + 0xb2, 0x01, 0xd0, 0x15, 0x6c, 0x69, 0x62, 0x6e, 0x65, 0x6f, 0x34, + 0x6a, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x31, 0x2e, + 0x32, 0x2e, 0x31, 0xa3, 0x86, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x65, + 0x85, 0x62, 0x61, 0x73, 0x69, 0x63, 0x89, 0x70, 0x72, 0x69, 0x6e, + 0x63, 0x69, 0x70, 0x61, 0x6c, 0x80, 0x8b, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x80}; const uint8_t init_resp[] = {0x00, 0x03, 0xb1, 0x70, 0xa0, 0x00, 0x00}; const uint8_t run_req_header[] = {0xb2, 0x10, 0xd1}; const uint8_t pullall_req[] = {0xb0, 0x3f}; @@ -42,7 +42,6 @@ const uint8_t ackfailure_req[] = {0xb0, 0x0e}; const uint8_t success_resp[] = {0x00, 0x03, 0xb1, 0x70, 0xa0, 0x00, 0x00}; const uint8_t ignored_resp[] = {0x00, 0x02, 0xb0, 0x7e, 0x00, 0x00}; - // Write bolt chunk header (length) void WriteChunkHeader(SessionT &session, uint16_t len) { len = bswap(len); @@ -52,16 +51,14 @@ void WriteChunkHeader(SessionT &session, uint16_t len) { } // Write bolt chunk tail (two zeros) -void WriteChunkTail(SessionT &session) { - WriteChunkHeader(session, 0); -} +void WriteChunkTail(SessionT &session) { WriteChunkHeader(session, 0); } // Check that the server responded with a failure message void CheckFailureMessage(std::vector &output) { ASSERT_GE(output.size(), 6); // skip the first two bytes because they are the chunk header - ASSERT_EQ(output[2], 0xB1); // tiny struct 1 - ASSERT_EQ(output[3], 0x7F); // signature failure + ASSERT_EQ(output[2], 0xB1); // tiny struct 1 + ASSERT_EQ(output[3], 0x7F); // signature failure } // Execute and check a correct handshake @@ -78,7 +75,8 @@ void ExecuteHandshake(SessionT &session, std::vector &output) { } // Write bolt chunk and execute command -void ExecuteCommand(SessionT &session, const uint8_t *data, size_t len, bool chunk = true) { +void ExecuteCommand(SessionT &session, const uint8_t *data, size_t len, + bool chunk = true) { if (chunk) WriteChunkHeader(session, len); auto buff = session.Allocate(); memcpy(buff.data, data, len); @@ -117,21 +115,19 @@ void WriteRunRequest(SessionT &session, const char *str) { // write empty map for parameters buff = session.Allocate(); - buff.data[0] = 0xA0; // TinyMap0 + buff.data[0] = 0xA0; // TinyMap0 session.Written(1); // write chunk tail WriteChunkTail(session); } - TEST(BoltSession, HandshakeWrongPreamble) { INIT_VARS; auto buff = session.Allocate(); // copy 0x00000001 four times - for (int i = 0; i < 4; ++i) - memcpy(buff.data + i * 4, handshake_req + 4, 4); + for (int i = 0; i < 4; ++i) memcpy(buff.data + i * 4, handshake_req + 4, 4); session.Written(20); session.Execute(); @@ -162,7 +158,6 @@ TEST(BoltSession, HandshakeInTwoPackets) { CheckOutput(output, handshake_resp, 4); } - TEST(BoltSession, HandshakeWriteFail) { INIT_VARS; session.socket_.SetWriteSuccess(false); @@ -178,7 +173,6 @@ TEST(BoltSession, HandshakeOK) { ExecuteHandshake(session, output); } - TEST(BoltSession, InitWrongSignature) { INIT_VARS; ExecuteHandshake(session, output); @@ -235,7 +229,6 @@ TEST(BoltSession, InitOK) { ExecuteInit(session, output); } - TEST(BoltSession, ExecuteRunWrongMarker) { INIT_VARS; @@ -457,7 +450,7 @@ TEST(BoltSession, ErrorOK) { const uint8_t *dataset[] = {ackfailure_req, reset_req}; for (int i = 0; i < 2; ++i) { - // first test with socket write success, then with socket write fail + // first test with socket write success, then with socket write fail for (int j = 0; j < 2; ++j) { INIT_VARS; @@ -523,7 +516,7 @@ TEST(BoltSession, MultipleChunksInOneExecute) { // Count chunks in output int len, num = 0; - while(output.size() > 0) { + while (output.size() > 0) { len = (output[0] << 8) + output[1]; output.erase(output.begin(), output.begin() + len + 4); ++num; @@ -579,13 +572,12 @@ TEST(BoltSession, InvalidChunk) { CheckFailureMessage(output); } - int main(int argc, char **argv) { logging::init_sync(); logging::log->pipe(std::make_unique()); // Set the interpret to true to avoid calling the compiler which only // supports a limited set of queries. - CONFIG(config::INTERPRET) = "true"; + FLAGS_INTERPRET = true; ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/dbms_recovery.cpp b/tests/unit/dbms_recovery.cpp index ae8386c2d..f56849aed 100644 --- a/tests/unit/dbms_recovery.cpp +++ b/tests/unit/dbms_recovery.cpp @@ -1,7 +1,14 @@ #include -#include "dbms/dbms.hpp" + +#include "gflags/gflags.h" #include "gtest/gtest.h" +#include "dbms/dbms.hpp" + +DECLARE_bool(RECOVERY_ON_STARTUP); +DECLARE_string(SNAPSHOT_DIRECTORY); +DECLARE_int32(SNAPSHOT_CYCLE_SEC); + namespace fs = std::experimental::filesystem; const fs::path SNAPSHOTS_DBMS_RECOVERY_ALL_DB = std::tmpnam(nullptr); @@ -25,25 +32,17 @@ void CleanDbDir() { class DbmsRecoveryTest : public ::testing::Test { protected: - virtual void TearDown() { - CleanDbDir(); - CONFIG(config::SNAPSHOTS_PATH) = snapshots_path_setup_; - CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_; - } + virtual void TearDown() { CleanDbDir(); } virtual void SetUp() { CleanDbDir(); - snapshots_path_setup_ = CONFIG(config::SNAPSHOTS_PATH); - snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC); - CONFIG(config::SNAPSHOTS_PATH) = SNAPSHOTS_DBMS_RECOVERY_ALL_DB; - CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1"; + FLAGS_SNAPSHOT_DIRECTORY = SNAPSHOTS_DBMS_RECOVERY_ALL_DB; + FLAGS_SNAPSHOT_CYCLE_SEC = -1; } - std::string snapshots_path_setup_; - std::string snapshot_cycle_sec_setup_; }; void CreateSnapshot() { - CONFIG(config::RECOVERY) = "false"; + FLAGS_RECOVER_ON_STARTUP = false; Dbms dbms; auto dba = dbms.active(); @@ -56,12 +55,13 @@ void CreateSnapshot() { dba->advance_command(); Snapshooter snapshooter; - snapshooter.MakeSnapshot(*dba.get(), SNAPSHOTS_DBMS_RECOVERY_DEFAULT_DB_DIR, - 1); + EXPECT_EQ(snapshooter.MakeSnapshot(*dba.get(), + SNAPSHOTS_DBMS_RECOVERY_DEFAULT_DB_DIR, 1), + true); } void RecoverDbms() { - CONFIG(config::RECOVERY) = "true"; + FLAGS_RECOVER_ON_STARTUP = true; Dbms dbms; auto dba = dbms.active(); @@ -84,7 +84,7 @@ void RecoverDbms() { edges.push_back(edge); edge_count++; } - EXPECT_EQ(edge_count, 2); + ASSERT_EQ(edge_count, 2); EXPECT_EQ(edges[0].to() == edges[1].to(), true); EXPECT_EQ(edges[0].from() == edges[1].from(), false); } diff --git a/tests/unit/mvcc_gc.cpp b/tests/unit/mvcc_gc.cpp index aff1d2a96..07964921f 100644 --- a/tests/unit/mvcc_gc.cpp +++ b/tests/unit/mvcc_gc.cpp @@ -5,7 +5,6 @@ #include #include -#include "config/config.hpp" #include "data_structures/concurrent/skiplist.hpp" #include "logging/logger.hpp" #include "logging/streams/stdout.hpp" diff --git a/tests/unit/recovery.cpp b/tests/unit/recovery.cpp index 058ec2b59..6ba2eba0f 100644 --- a/tests/unit/recovery.cpp +++ b/tests/unit/recovery.cpp @@ -1,12 +1,17 @@ -#include "durability/recovery.hpp" #include #include + +#include "gflags/gflags.h" +#include "gtest/gtest.h" + #include "communication/bolt/v1/decoder/decoder.hpp" #include "dbms/dbms.hpp" #include "durability/file_reader_buffer.hpp" -#include "gtest/gtest.h" +#include "durability/recovery.hpp" #include "utils/assert.hpp" +DECLARE_int32(SNAPSHOT_CYCLE_SEC); + namespace fs = std::experimental::filesystem; const fs::path SNAPSHOTS_RECOVERY_DEFAULT_DB_DIR = std::tmpnam(nullptr); @@ -30,17 +35,12 @@ void CleanDbDir() { class RecoveryTest : public ::testing::Test { protected: - void TearDown() override { - CleanDbDir(); - CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_; - } + void TearDown() override { CleanDbDir(); } void SetUp() override { CleanDbDir(); - snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC); - CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1"; + FLAGS_SNAPSHOT_CYCLE_SEC = -1; } - std::string snapshot_cycle_sec_setup_; const int max_retained_snapshots_ = 10; }; diff --git a/tests/unit/snapshot.cpp b/tests/unit/snapshot.cpp index 99b9c9c11..0cc227e62 100644 --- a/tests/unit/snapshot.cpp +++ b/tests/unit/snapshot.cpp @@ -1,7 +1,14 @@ #include + +#include "gflags/gflags.h" +#include "gtest/gtest.h" + #include "dbms/dbms.hpp" #include "durability/snapshooter.hpp" -#include "gtest/gtest.h" + +DECLARE_bool(SNAPSHOT_ON_DB_DESTRUCTION); +DECLARE_int32(SNAPSHOT_CYCLE_SEC); +DECLARE_string(SNAPSHOT_DIRECTORY); namespace fs = std::experimental::filesystem; @@ -28,15 +35,11 @@ void CleanDbDir() { class SnapshotTest : public ::testing::Test { protected: - virtual void TearDown() { - CleanDbDir(); - CONFIG(config::SNAPSHOT_CYCLE_SEC) = snapshot_cycle_sec_setup_; - } + virtual void TearDown() { CleanDbDir(); } virtual void SetUp() { CleanDbDir(); - snapshot_cycle_sec_setup_ = CONFIG(config::SNAPSHOT_CYCLE_SEC); - CONFIG(config::SNAPSHOT_CYCLE_SEC) = "-1"; + FLAGS_SNAPSHOT_CYCLE_SEC = -1; } std::string snapshot_cycle_sec_setup_; }; @@ -97,8 +100,8 @@ TEST_F(SnapshotTest, CreateSnapshotWithUnlimitedMaxRetainedSnapshots) { TEST_F(SnapshotTest, TestSnapshotFileOnDbDestruct) { { - CONFIG(config::SNAPSHOTS_PATH) = SNAPSHOTS_FOLDER_ALL_DB; - CONFIG(config::SNAPSHOT_DB_DESTRUCTION) = "true"; + FLAGS_SNAPSHOT_DIRECTORY = SNAPSHOTS_FOLDER_ALL_DB; + FLAGS_SNAPSHOT_ON_DB_DESTRUCTION = true; Dbms dbms; auto dba = dbms.active(); }