diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index 0fd4a930b..40ee3e7dd 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -249,7 +249,9 @@ jobs: - name: Run GQL Behave tests run: | - cd tests/gql_behave + cd tests + ./setup.sh /opt/toolchain-v4/activate + cd gql_behave ./continuous_integration - name: Save quality assurance status @@ -417,11 +419,12 @@ jobs: source ve3/bin/activate pip install -r requirements.txt ./main.py --benchmark-name "macro_benchmark" \ - --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \ + --benchmark-results "../../tests/macro_benchmark/.harness_summary" \ --github-run-id "${{ github.run_id }}" \ --github-run-number "${{ github.run_number }}" \ --head-branch-name "${{ env.BRANCH_NAME }}" + # TODO (andi) No need for path flags and for --disk-storage and --in-memory-analytical - name: Run mgbench run: | cd tests/mgbench @@ -434,7 +437,7 @@ jobs: source ve3/bin/activate pip install -r requirements.txt ./main.py --benchmark-name "mgbench" \ - --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \ + --benchmark-results "../../tests/mgbench/benchmark_result.json" \ --github-run-id "${{ github.run_id }}" \ --github-run-number "${{ github.run_number }}" \ --head-branch-name "${{ env.BRANCH_NAME }}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a3a53fcc2..d6258f2c8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,8 @@ add_subdirectory(auth) add_subdirectory(audit) add_subdirectory(dbms) add_subdirectory(flags) +add_subdirectory(distributed) +add_subdirectory(replication) string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) diff --git a/src/communication/bolt/v1/session.hpp b/src/communication/bolt/v1/session.hpp index bcceddf45..4d58a4737 100644 --- a/src/communication/bolt/v1/session.hpp +++ b/src/communication/bolt/v1/session.hpp @@ -208,7 +208,7 @@ class Session { Version version_; - virtual std::string GetDatabaseName() const = 0; + virtual std::string GetCurrentDB() const = 0; std::string UUID() const { return session_uuid_; } private: diff --git a/src/communication/bolt/v1/states/handlers.hpp b/src/communication/bolt/v1/states/handlers.hpp index 4243a7fb9..eceb578f6 100644 --- a/src/communication/bolt/v1/states/handlers.hpp +++ b/src/communication/bolt/v1/states/handlers.hpp @@ -208,7 +208,7 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) { DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state"); - spdlog::debug("[Run - {}] '{}'", session.GetDatabaseName(), query.ValueString()); + spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString()); try { // Interpret can throw. @@ -272,7 +272,7 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) { return HandleFailure(session, e); } - spdlog::debug("[Run - {}] '{}'", session.GetDatabaseName(), query.ValueString()); + spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString()); try { // Interpret can throw. diff --git a/src/dbms/database.hpp b/src/dbms/database.hpp index a95cf6d91..33b1dec5e 100644 --- a/src/dbms/database.hpp +++ b/src/dbms/database.hpp @@ -60,6 +60,11 @@ class Database { return storage_->Access(override_isolation_level); } + std::unique_ptr UniqueAccess( + std::optional override_isolation_level = {}) { + return storage_->UniqueAccess(override_isolation_level); + } + /** * @brief Unique storage identified (name) * diff --git a/src/distributed/CMakeLists.txt b/src/distributed/CMakeLists.txt new file mode 100644 index 000000000..ab85b13fd --- /dev/null +++ b/src/distributed/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(mg-distributed) +add_library(mg::distributed ALIAS mg-distributed) +target_include_directories(mg-distributed PUBLIC include ) +target_sources(mg-distributed PRIVATE lamport_clock.cpp) diff --git a/src/distributed/include/distributed/lamport_clock.hpp b/src/distributed/include/distributed/lamport_clock.hpp new file mode 100644 index 000000000..f3e91e47a --- /dev/null +++ b/src/distributed/include/distributed/lamport_clock.hpp @@ -0,0 +1,61 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include +#include +#include +#include + +namespace memgraph::distributed { + +// forward declare, for strong timestamps +template +struct LamportClock; + +template +struct timestamp { + friend std::strong_ordering operator<=>(timestamp const &, timestamp const &) = default; + + private: + friend struct LamportClock; + + explicit timestamp(uint64_t value) : value_{value} {} + uint64_t value_; +}; + +constexpr struct internal_t { +} internal; +constexpr struct send_t { +} send; +constexpr struct receive_t { +} receive; + +template +struct LamportClock { + using timestamp_t = timestamp; + + auto get_timestamp(internal_t) -> timestamp_t { return timestamp_t{++internal}; }; + auto get_timestamp(send_t) -> timestamp_t { return timestamp_t{++internal}; }; + auto get_timestamp(receive_t, timestamp_t received_timestamp) -> timestamp_t { + while (true) { + auto local_current = internal.load(std::memory_order_acquire); + auto next = std::max(received_timestamp.value_, local_current) + 1; + bool res = internal.compare_exchange_weak(local_current, next, std::memory_order_acq_rel); + if (res) return timestamp_t{next}; + } + }; + + private: + std::atomic internal = 0; +}; + +} // namespace memgraph::distributed diff --git a/src/distributed/lamport_clock.cpp b/src/distributed/lamport_clock.cpp new file mode 100644 index 000000000..9635f50c7 --- /dev/null +++ b/src/distributed/lamport_clock.cpp @@ -0,0 +1,11 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#include "distributed/lamport_clock.hpp" diff --git a/src/glue/SessionHL.cpp b/src/glue/SessionHL.cpp index 87481eb20..0d0fd7b0f 100644 --- a/src/glue/SessionHL.cpp +++ b/src/glue/SessionHL.cpp @@ -114,9 +114,9 @@ std::string SessionHL::GetDefaultDB() { } #endif -std::string SessionHL::GetDatabaseName() const { - if (!interpreter_.db_acc_) return ""; - const auto *db = interpreter_.db_acc_->get(); +std::string SessionHL::GetCurrentDB() const { + if (!interpreter_.current_db_.db_acc_) return ""; + const auto *db = interpreter_.current_db_.db_acc_->get(); return db->id(); } @@ -127,18 +127,23 @@ std::optional SessionHL::GetServerNameForInit() { bool SessionHL::Authenticate(const std::string &username, const std::string &password) { bool res = true; + interpreter_.ResetUser(); { auto locked_auth = auth_->Lock(); if (locked_auth->HasUsers()) { user_ = locked_auth->Authenticate(username, password); - res = user_.has_value(); + if (user_.has_value()) { + interpreter_.SetUser(user_->username()); + } else { + res = false; + } } } #ifdef MG_ENTERPRISE // Start off with the default database - interpreter_.SetCurrentDB(GetDefaultDB()); + interpreter_.SetCurrentDB(GetDefaultDB(), false); #endif - implicit_db_.emplace(GetDatabaseName()); + implicit_db_.emplace(GetCurrentDB()); return res; } @@ -159,7 +164,7 @@ std::map SessionHL::Pull(Sess std::optional n, std::optional qid) { // TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt) - auto *db = interpreter_.db_acc_->get(); + auto *db = interpreter_.current_db_.db_acc_->get(); try { TypedValueResultStream stream(encoder, db->storage()); return DecodeSummary(interpreter_.Pull(&stream, n, qid)); @@ -184,14 +189,14 @@ std::pair, std::optional> SessionHL::Interpret( #ifdef MG_ENTERPRISE // TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt) - auto *db = interpreter_.db_acc_->get(); + auto *db = interpreter_.current_db_.db_acc_->get(); if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) { audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query, memgraph::storage::PropertyValue(params_pv), db->id()); } #endif try { - auto result = interpreter_.Prepare(query, params_pv, username, ToQueryExtras(extra), UUID()); + auto result = interpreter_.Prepare(query, params_pv, ToQueryExtras(extra)); const std::string db_name = result.db ? *result.db : ""; if (user_ && !AuthChecker::IsUserAuthorized(*user_, result.privileges, db_name)) { interpreter_.Abort(); @@ -230,7 +235,7 @@ void SessionHL::Configure(const std::map SessionHL::DecodeSummary( const std::map &summary) { // TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt) - auto *db = interpreter_.db_acc_->get(); + auto *db = interpreter_.current_db_.db_acc_->get(); std::map decoded_summary; for (const auto &kv : summary) { auto maybe_value = ToBoltValue(kv.second, *db->storage(), memgraph::storage::View::NEW); diff --git a/src/glue/SessionHL.hpp b/src/glue/SessionHL.hpp index d6c095a04..cc53ae08a 100644 --- a/src/glue/SessionHL.hpp +++ b/src/glue/SessionHL.hpp @@ -68,7 +68,7 @@ class SessionHL final : public memgraph::communication::bolt::Session GetServerNameForInit() override; - std::string GetDatabaseName() const override; + std::string GetCurrentDB() const override; private: std::map DecodeSummary( diff --git a/src/kvstore/CMakeLists.txt b/src/kvstore/CMakeLists.txt index b3a119ed5..fee8b7b05 100644 --- a/src/kvstore/CMakeLists.txt +++ b/src/kvstore/CMakeLists.txt @@ -4,4 +4,5 @@ find_package(ZLIB REQUIRED) # STATIC library used to store key-value pairs add_library(mg-kvstore STATIC kvstore.cpp) +add_library(mg::kvstore ALIAS mg-kvstore) target_link_libraries(mg-kvstore stdc++fs mg-utils rocksdb BZip2::BZip2 ZLIB::ZLIB gflags) diff --git a/src/memgraph.cpp b/src/memgraph.cpp index f28b463ab..370d04bc5 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -360,7 +360,7 @@ int main(int argc, char **argv) { auto db_acc_opt = db_gatekeeper.access(); MG_ASSERT(db_acc_opt, "Failed to access the main database"); auto &db_acc = *db_acc_opt; - memgraph::query::InterpreterContext interpreter_context_(interp_config, nullptr, auth_handler.get(), + memgraph::query::InterpreterContext interpreter_context_(interp_config, &db_gatekeeper, auth_handler.get(), auth_checker.get()); #endif MG_ASSERT(db_acc, "Failed to access the main database"); diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index a66516c34..45fd2414b 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -38,7 +38,8 @@ set(mg_query_sources graph.cpp db_accessor.cpp auth_query_handler.cpp - interpreter_context.cpp) + interpreter_context.cpp +) add_library(mg-query STATIC ${mg_query_sources}) target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include) diff --git a/src/query/config.hpp b/src/query/config.hpp index 59a34e263..64e2da5bb 100644 --- a/src/query/config.hpp +++ b/src/query/config.hpp @@ -19,7 +19,7 @@ struct InterpreterConfig { bool allow_load_csv{true}; } query; - // The same as \ref memgraph::storage::replication::ReplicationClientConfig + // The same as \ref memgraph::replication::ReplicationClientConfig std::chrono::seconds replication_replica_check_frequency{1}; std::string default_kafka_bootstrap_servers; diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index 3a38127c2..102fe5d4c 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -459,6 +459,8 @@ class DbAccessor final { } storage::Result> RemoveVertex(VertexAccessor *vertex_accessor) { + accessor_->PrefetchOutEdges(vertex_accessor->impl_); + accessor_->PrefetchInEdges(vertex_accessor->impl_); auto res = accessor_->DeleteVertex(&vertex_accessor->impl_); if (res.HasError()) { return res.GetError(); @@ -533,7 +535,7 @@ class DbAccessor final { void AdvanceCommand() { accessor_->AdvanceCommand(); } - utils::BasicResult Commit() { return accessor_->Commit(); } + utils::BasicResult Commit() { return accessor_->Commit(); } void Abort() { accessor_->Abort(); } @@ -554,20 +556,12 @@ class DbAccessor final { return accessor_->GetIndexStats(label, property); } - std::vector> ClearLabelPropertyIndexStats() { - return accessor_->ClearLabelPropertyIndexStats(); - } - - std::vector ClearLabelIndexStats() { return accessor_->ClearLabelIndexStats(); } - std::vector> DeleteLabelPropertyIndexStats( - const std::span labels) { - return accessor_->DeleteLabelPropertyIndexStats(labels); + const storage::LabelId &label) { + return accessor_->DeleteLabelPropertyIndexStats(label); } - std::vector DeleteLabelIndexStats(const std::span labels) { - return accessor_->DeleteLabelIndexStats(labels); - } + bool DeleteLabelIndexStats(const storage::LabelId &label) { return accessor_->DeleteLabelIndexStats(label); } void SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats) { accessor_->SetIndexStats(label, stats); @@ -602,6 +596,44 @@ class DbAccessor final { storage::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); } const std::string &id() const { return accessor_->id(); } + + utils::BasicResult CreateIndex(storage::LabelId label) { + return accessor_->CreateIndex(label); + } + + utils::BasicResult CreateIndex(storage::LabelId label, + storage::PropertyId property) { + return accessor_->CreateIndex(label, property); + } + + utils::BasicResult DropIndex(storage::LabelId label) { + return accessor_->DropIndex(label); + } + + utils::BasicResult DropIndex(storage::LabelId label, + storage::PropertyId property) { + return accessor_->DropIndex(label, property); + } + + utils::BasicResult CreateExistenceConstraint( + storage::LabelId label, storage::PropertyId property) { + return accessor_->CreateExistenceConstraint(label, property); + } + + utils::BasicResult DropExistenceConstraint( + storage::LabelId label, storage::PropertyId property) { + return accessor_->DropExistenceConstraint(label, property); + } + + utils::BasicResult + CreateUniqueConstraint(storage::LabelId label, const std::set &properties) { + return accessor_->CreateUniqueConstraint(label, properties); + } + + storage::UniqueConstraints::DeletionStatus DropUniqueConstraint(storage::LabelId label, + const std::set &properties) { + return accessor_->DropUniqueConstraint(label, properties); + } }; class SubgraphDbAccessor final { diff --git a/src/query/frontend/ast/ast.cpp b/src/query/frontend/ast/ast.cpp index 09e54fdcf..c5e4c84c4 100644 --- a/src/query/frontend/ast/ast.cpp +++ b/src/query/frontend/ast/ast.cpp @@ -223,7 +223,11 @@ constexpr utils::TypeInfo query::Unwind::kType{utils::TypeId::AST_UNWIND, "Unwin constexpr utils::TypeInfo query::AuthQuery::kType{utils::TypeId::AST_AUTH_QUERY, "AuthQuery", &query::Query::kType}; -constexpr utils::TypeInfo query::InfoQuery::kType{utils::TypeId::AST_INFO_QUERY, "InfoQuery", &query::Query::kType}; +constexpr utils::TypeInfo query::DatabaseInfoQuery::kType{utils::TypeId::AST_DATABASE_INFO_QUERY, "DatabaseInfoQuery", + &query::Query::kType}; + +constexpr utils::TypeInfo query::SystemInfoQuery::kType{utils::TypeId::AST_SYSTEM_INFO_QUERY, "SystemInfoQuery", + &query::Query::kType}; constexpr utils::TypeInfo query::Constraint::kType{utils::TypeId::AST_CONSTRAINT, "Constraint", nullptr}; diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index 201f4e230..d6ff1fb24 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -2897,19 +2897,37 @@ const std::vector kPrivilegesAll = {AuthQuery::Privilege:: AuthQuery::Privilege::MULTI_DATABASE_EDIT, AuthQuery::Privilege::MULTI_DATABASE_USE}; -class InfoQuery : public memgraph::query::Query { +class DatabaseInfoQuery : public memgraph::query::Query { public: static const utils::TypeInfo kType; const utils::TypeInfo &GetTypeInfo() const override { return kType; } - enum class InfoType { STORAGE, INDEX, CONSTRAINT, BUILD }; + enum class InfoType { INDEX, CONSTRAINT }; DEFVISITABLE(QueryVisitor); - memgraph::query::InfoQuery::InfoType info_type_; + memgraph::query::DatabaseInfoQuery::InfoType info_type_; - InfoQuery *Clone(AstStorage *storage) const override { - InfoQuery *object = storage->Create(); + DatabaseInfoQuery *Clone(AstStorage *storage) const override { + DatabaseInfoQuery *object = storage->Create(); + object->info_type_ = info_type_; + return object; + } +}; + +class SystemInfoQuery : public memgraph::query::Query { + public: + static const utils::TypeInfo kType; + const utils::TypeInfo &GetTypeInfo() const override { return kType; } + + enum class InfoType { STORAGE, BUILD }; + + DEFVISITABLE(QueryVisitor); + + memgraph::query::SystemInfoQuery::InfoType info_type_; + + SystemInfoQuery *Clone(AstStorage *storage) const override { + SystemInfoQuery *object = storage->Create(); object->info_type_ = info_type_; return object; } diff --git a/src/query/frontend/ast/ast_visitor.hpp b/src/query/frontend/ast/ast_visitor.hpp index 89b405663..793c15a95 100644 --- a/src/query/frontend/ast/ast_visitor.hpp +++ b/src/query/frontend/ast/ast_visitor.hpp @@ -82,7 +82,8 @@ class AuthQuery; class ExplainQuery; class ProfileQuery; class IndexQuery; -class InfoQuery; +class DatabaseInfoQuery; +class SystemInfoQuery; class ConstraintQuery; class RegexMatch; class DumpQuery; @@ -140,10 +141,10 @@ class ExpressionVisitor template class QueryVisitor - : public utils::Visitor {}; + : public utils::Visitor {}; } // namespace memgraph::query diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index c3f401bda..32a3f4c11 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -112,25 +112,36 @@ antlrcpp::Any CypherMainVisitor::visitProfileQuery(MemgraphCypher::ProfileQueryC return profile_query; } -antlrcpp::Any CypherMainVisitor::visitInfoQuery(MemgraphCypher::InfoQueryContext *ctx) { - MG_ASSERT(ctx->children.size() == 2, "InfoQuery should have exactly two children!"); - auto *info_query = storage_->Create(); +antlrcpp::Any CypherMainVisitor::visitDatabaseInfoQuery(MemgraphCypher::DatabaseInfoQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 2, "DatabaseInfoQuery should have exactly two children!"); + auto *info_query = storage_->Create(); + query_ = info_query; + if (ctx->indexInfo()) { + info_query->info_type_ = DatabaseInfoQuery::InfoType::INDEX; + return info_query; + } + if (ctx->constraintInfo()) { + info_query->info_type_ = DatabaseInfoQuery::InfoType::CONSTRAINT; + return info_query; + } + // Should never get here + throw utils::NotYetImplemented("Database info query: '{}'", ctx->getText()); +} + +antlrcpp::Any CypherMainVisitor::visitSystemInfoQuery(MemgraphCypher::SystemInfoQueryContext *ctx) { + MG_ASSERT(ctx->children.size() == 2, "SystemInfoQuery should have exactly two children!"); + auto *info_query = storage_->Create(); query_ = info_query; if (ctx->storageInfo()) { - info_query->info_type_ = InfoQuery::InfoType::STORAGE; + info_query->info_type_ = SystemInfoQuery::InfoType::STORAGE; return info_query; - } else if (ctx->indexInfo()) { - info_query->info_type_ = InfoQuery::InfoType::INDEX; - return info_query; - } else if (ctx->constraintInfo()) { - info_query->info_type_ = InfoQuery::InfoType::CONSTRAINT; - return info_query; - } else if (ctx->buildInfo()) { - info_query->info_type_ = InfoQuery::InfoType::BUILD; - return info_query; - } else { - throw utils::NotYetImplemented("Info query: '{}'", ctx->getText()); } + if (ctx->buildInfo()) { + info_query->info_type_ = SystemInfoQuery::InfoType::BUILD; + return info_query; + } + // Should never get here + throw utils::NotYetImplemented("System info query: '{}'", ctx->getText()); } antlrcpp::Any CypherMainVisitor::visitConstraintQuery(MemgraphCypher::ConstraintQueryContext *ctx) { diff --git a/src/query/frontend/ast/cypher_main_visitor.hpp b/src/query/frontend/ast/cypher_main_visitor.hpp index 3bcafcbaf..9e828674f 100644 --- a/src/query/frontend/ast/cypher_main_visitor.hpp +++ b/src/query/frontend/ast/cypher_main_visitor.hpp @@ -157,9 +157,14 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor { antlrcpp::Any visitProfileQuery(MemgraphCypher::ProfileQueryContext *ctx) override; /** - * @return InfoQuery* + * @return DatabaseInfoQuery* */ - antlrcpp::Any visitInfoQuery(MemgraphCypher::InfoQueryContext *ctx) override; + antlrcpp::Any visitDatabaseInfoQuery(MemgraphCypher::DatabaseInfoQueryContext *ctx) override; + + /** + * @return SystemInfoQuery* + */ + antlrcpp::Any visitSystemInfoQuery(MemgraphCypher::SystemInfoQueryContext *ctx) override; /** * @return Constraint diff --git a/src/query/frontend/opencypher/grammar/Cypher.g4 b/src/query/frontend/opencypher/grammar/Cypher.g4 index ea9a9f5f7..53f1fc765 100644 --- a/src/query/frontend/opencypher/grammar/Cypher.g4 +++ b/src/query/frontend/opencypher/grammar/Cypher.g4 @@ -27,7 +27,8 @@ query : cypherQuery | indexQuery | explainQuery | profileQuery - | infoQuery + | databaseInfoQuery + | systemInfoQuery | constraintQuery ; @@ -48,7 +49,9 @@ constraintInfo : CONSTRAINT INFO ; buildInfo : BUILD INFO ; -infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo | buildInfo) ; +databaseInfoQuery : SHOW ( indexInfo | constraintInfo ) ; + +systemInfoQuery : SHOW ( storageInfo | buildInfo ) ; explainQuery : EXPLAIN cypherQuery ; diff --git a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 index deed2a3e6..e05ef7550 100644 --- a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 +++ b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 @@ -128,7 +128,8 @@ query : cypherQuery | indexQuery | explainQuery | profileQuery - | infoQuery + | databaseInfoQuery + | systemInfoQuery | constraintQuery | authQuery | dumpQuery diff --git a/src/query/frontend/semantic/required_privileges.cpp b/src/query/frontend/semantic/required_privileges.cpp index 93447498b..b5b75e26e 100644 --- a/src/query/frontend/semantic/required_privileges.cpp +++ b/src/query/frontend/semantic/required_privileges.cpp @@ -35,18 +35,14 @@ class PrivilegeExtractor : public QueryVisitor, public HierarchicalTreeVis void Visit(ProfileQuery &query) override { query.cypher_query_->Accept(dynamic_cast(*this)); } - void Visit(InfoQuery &info_query) override { + void Visit(DatabaseInfoQuery &info_query) override { switch (info_query.info_type_) { - case InfoQuery::InfoType::INDEX: + case DatabaseInfoQuery::InfoType::INDEX: // TODO: This should be INDEX | STATS, but we don't have support for // *or* with privileges. AddPrivilege(AuthQuery::Privilege::INDEX); break; - case InfoQuery::InfoType::STORAGE: - case InfoQuery::InfoType::BUILD: - AddPrivilege(AuthQuery::Privilege::STATS); - break; - case InfoQuery::InfoType::CONSTRAINT: + case DatabaseInfoQuery::InfoType::CONSTRAINT: // TODO: This should be CONSTRAINT | STATS, but we don't have support // for *or* with privileges. AddPrivilege(AuthQuery::Privilege::CONSTRAINT); @@ -54,6 +50,15 @@ class PrivilegeExtractor : public QueryVisitor, public HierarchicalTreeVis } } + void Visit(SystemInfoQuery &info_query) override { + switch (info_query.info_type_) { + case SystemInfoQuery::InfoType::STORAGE: + case SystemInfoQuery::InfoType::BUILD: + AddPrivilege(AuthQuery::Privilege::STATS); + break; + } + } + void Visit(ConstraintQuery &constraint_query) override { AddPrivilege(AuthQuery::Privilege::CONSTRAINT); } void Visit(CypherQuery &query) override { diff --git a/src/query/frontend/semantic/symbol_generator.cpp b/src/query/frontend/semantic/symbol_generator.cpp index 709df16c0..5fbc5d8a3 100644 --- a/src/query/frontend/semantic/symbol_generator.cpp +++ b/src/query/frontend/semantic/symbol_generator.cpp @@ -518,6 +518,10 @@ bool SymbolGenerator::PreVisit(Exists &exists) { throw utils::NotYetImplemented("WITH can not be used with exists, but only during matching!"); } + if (scope.in_return) { + throw utils::NotYetImplemented("RETURN can not be used with exists, but only during matching!"); + } + scope.in_exists = true; const auto &symbol = CreateAnonymousSymbol(); diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index a4f3ce5b8..6f9873b18 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -65,6 +65,7 @@ #include "query/stream/streams.hpp" #include "query/trigger.hpp" #include "query/typed_value.hpp" +#include "replication/config.hpp" #include "spdlog/spdlog.h" #include "storage/v2/disk/storage.hpp" #include "storage/v2/edge.hpp" @@ -72,7 +73,7 @@ #include "storage/v2/id_types.hpp" #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/property_value.hpp" -#include "storage/v2/replication/config.hpp" +#include "storage/v2/storage_error.hpp" #include "storage/v2/storage_mode.hpp" #include "utils/algorithm.hpp" #include "utils/build_info.hpp" @@ -97,6 +98,8 @@ #include "dbms/dbms_handler.hpp" #include "query/auth_query_handler.hpp" #include "query/interpreter_context.hpp" +#include "replication/state.hpp" +#include "storage/v2/replication/replication_handler.hpp" namespace memgraph::metrics { extern Event ReadQuery; @@ -112,6 +115,29 @@ extern const Event CommitedTransactions; extern const Event RollbackedTransactions; extern const Event ActiveTransactions; } // namespace memgraph::metrics +void memgraph::query::CurrentDB::SetupDatabaseTransaction( + std::optional override_isolation_level, bool could_commit, bool unique) { + auto &db_acc = *db_acc_; + if (unique) { + db_transactional_accessor_ = db_acc->UniqueAccess(override_isolation_level); + } else { + db_transactional_accessor_ = db_acc->Access(override_isolation_level); + } + execution_db_accessor_.emplace(db_transactional_accessor_.get()); + + if (db_acc->trigger_store()->HasTriggers() && could_commit) { + trigger_context_collector_.emplace(db_acc->trigger_store()->GetEventTypes()); + } +} +void memgraph::query::CurrentDB::CleanupDBTransaction(bool abort) { + if (abort && db_transactional_accessor_) { + db_transactional_accessor_->Abort(); + } + db_transactional_accessor_.reset(); + execution_db_accessor_.reset(); + trigger_context_collector_.reset(); +} +// namespace memgraph::metrics namespace memgraph::query { @@ -120,7 +146,6 @@ constexpr auto kAlwaysFalse = false; namespace { template - void Sort(std::vector &vec) { std::sort(vec.begin(), vec.end()); } @@ -132,13 +157,15 @@ void Sort(std::vector &vec) { } // NOLINTNEXTLINE (misc-unused-parameters) -bool Same(const TypedValue &lv, const TypedValue &rv) { +[[maybe_unused]] bool Same(const TypedValue &lv, const TypedValue &rv) { return TypedValue(lv).ValueString() == TypedValue(rv).ValueString(); } // NOLINTNEXTLINE (misc-unused-parameters) bool Same(const TypedValue &lv, const std::string &rv) { return std::string(TypedValue(lv).ValueString()) == rv; } // NOLINTNEXTLINE (misc-unused-parameters) -bool Same(const std::string &lv, const TypedValue &rv) { return lv == std::string(TypedValue(rv).ValueString()); } +[[maybe_unused]] bool Same(const std::string &lv, const TypedValue &rv) { + return lv == std::string(TypedValue(rv).ValueString()); +} // NOLINTNEXTLINE (misc-unused-parameters) bool Same(const std::string &lv, const std::string &rv) { return lv == rv; } @@ -224,24 +251,40 @@ bool IsAllShortestPathsQuery(const std::vector &claus return false; } +inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode) -> replication::ReplicationMode { + switch (sync_mode) { + case ReplicationQuery::SyncMode::ASYNC: { + return replication::ReplicationMode::ASYNC; + } + case ReplicationQuery::SyncMode::SYNC: { + return replication::ReplicationMode::SYNC; + } + } + // TODO: C++23 std::unreachable() + return replication::ReplicationMode::ASYNC; +} + class ReplQueryHandler final : public query::ReplicationQueryHandler { public: - explicit ReplQueryHandler(storage::Storage *db) : db_(db) {} + explicit ReplQueryHandler(storage::Storage *db) : db_(db), handler_{db_->repl_state_, *db_} {} /// @throw QueryRuntimeException if an error ocurred. void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { if (replication_role == ReplicationQuery::ReplicationRole::MAIN) { - if (!db_->SetMainReplicationRole()) { + if (!handler_.SetReplicationRoleMain()) { throw QueryRuntimeException("Couldn't set role to main!"); } } else { if (!port || *port < 0 || *port > std::numeric_limits::max()) { throw QueryRuntimeException("Port number invalid!"); } - if (!db_->SetReplicaRole(storage::replication::ReplicationServerConfig{ - .ip_address = storage::replication::kDefaultReplicationServerIp, - .port = static_cast(*port), - })) { + + auto const config = memgraph::replication::ReplicationServerConfig{ + .ip_address = memgraph::replication::kDefaultReplicationServerIp, + .port = static_cast(*port), + }; + + if (!handler_.SetReplicationRoleReplica(config)) { throw QueryRuntimeException("Couldn't set role to replica!"); } } @@ -249,10 +292,10 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. ReplicationQuery::ReplicationRole ShowReplicationRole() const override { - switch (db_->GetReplicationRole()) { - case storage::replication::ReplicationRole::MAIN: + switch (handler_.GetRole()) { + case memgraph::replication::ReplicationRole::MAIN: return ReplicationQuery::ReplicationRole::MAIN; - case storage::replication::ReplicationRole::REPLICA: + case memgraph::replication::ReplicationRole::REPLICA: return ReplicationQuery::ReplicationRole::REPLICA; } throw QueryRuntimeException("Couldn't show replication role - invalid role set!"); @@ -262,39 +305,29 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { void RegisterReplica(const std::string &name, const std::string &socket_address, const ReplicationQuery::SyncMode sync_mode, const std::chrono::seconds replica_check_frequency) override { - if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { + if (handler_.IsReplica()) { // replica can't register another replica throw QueryRuntimeException("Replica can't register another replica!"); } - if (name == storage::replication::kReservedReplicationRoleName) { + if (name == memgraph::replication::kReservedReplicationRoleName) { throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!"); } - storage::replication::ReplicationMode repl_mode; - switch (sync_mode) { - case ReplicationQuery::SyncMode::ASYNC: { - repl_mode = storage::replication::ReplicationMode::ASYNC; - break; - } - case ReplicationQuery::SyncMode::SYNC: { - repl_mode = storage::replication::ReplicationMode::SYNC; - break; - } - } + auto repl_mode = convertToReplicationMode(sync_mode); auto maybe_ip_and_port = - io::network::Endpoint::ParseSocketOrIpAddress(socket_address, storage::replication::kDefaultReplicationPort); + io::network::Endpoint::ParseSocketOrIpAddress(socket_address, memgraph::replication::kDefaultReplicationPort); if (maybe_ip_and_port) { auto [ip, port] = *maybe_ip_and_port; - auto ret = db_->RegisterReplica( - storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID, - storage::replication::ReplicationClientConfig{.name = name, - .mode = repl_mode, - .ip_address = ip, - .port = port, - .replica_check_frequency = replica_check_frequency, - .ssl = std::nullopt}); + auto config = replication::ReplicationClientConfig{.name = name, + .mode = repl_mode, + .ip_address = ip, + .port = port, + .replica_check_frequency = replica_check_frequency, + .ssl = std::nullopt}; + using storage::RegistrationMode; + auto ret = handler_.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, config); if (ret.HasError()) { throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name)); } @@ -303,20 +336,26 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { } } - /// @throw QueryRuntimeException if an error ocurred. - void DropReplica(const std::string &replica_name) override { - if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { - // replica can't unregister a replica - throw QueryRuntimeException("Replica can't unregister a replica!"); - } - if (!db_->UnregisterReplica(replica_name)) { - throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name)); + /// @throw QueryRuntimeException if an error occurred. + void DropReplica(std::string_view replica_name) override { + auto const result = handler_.UnregisterReplica(replica_name); + switch (result) { + using enum memgraph::storage::UnregisterReplicaResult; + case NOT_MAIN: + throw QueryRuntimeException("Replica can't unregister a replica!"); + case COULD_NOT_BE_PERSISTED: + [[fallthrough]]; + case CAN_NOT_UNREGISTER: + throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name)); + case SUCCESS: + break; } } using Replica = ReplicationQueryHandler::Replica; std::vector ShowReplicas() const override { - if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { + auto const &replState = db_->repl_state_; + if (replState.IsReplica()) { // replica can't show registered replicas (it shouldn't have any) throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!"); } @@ -330,10 +369,10 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { replica.name = repl_info.name; replica.socket_address = repl_info.endpoint.SocketAddress(); switch (repl_info.mode) { - case storage::replication::ReplicationMode::SYNC: + case memgraph::replication::ReplicationMode::SYNC: replica.sync_mode = ReplicationQuery::SyncMode::SYNC; break; - case storage::replication::ReplicationMode::ASYNC: + case memgraph::replication::ReplicationMode::ASYNC: replica.sync_mode = ReplicationQuery::SyncMode::ASYNC; break; } @@ -366,6 +405,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { private: storage::Storage *db_; + storage::ReplicationHandler handler_; }; /// returns false if the replication role can't be set @@ -719,7 +759,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & fmt::format("Replica {} is registered.", repl_query->replica_name_)); return callback; } - case ReplicationQuery::Action::DROP_REPLICA: { const auto &name = repl_query->replica_name_; callback.fn = [handler = ReplQueryHandler{storage}, name]() mutable { @@ -730,7 +769,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & fmt::format("Replica {} is dropped.", repl_query->replica_name_)); return callback; } - case ReplicationQuery::Action::SHOW_REPLICAS: { callback.header = { "name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master", @@ -783,10 +821,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters & } } -std::optional StringPointerToOptional(const std::string *str) { - return str == nullptr ? std::nullopt : std::make_optional(*str); -} - stream::CommonStreamInfo GetCommonStreamInfo(StreamQuery *stream_query, ExpressionVisitor &evaluator) { return { .batch_interval = GetOptionalValue(stream_query->batch_interval_, evaluator) @@ -809,7 +843,7 @@ std::vector EvaluateTopicNames(ExpressionVisitor &evalu Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, ExpressionVisitor &evaluator, memgraph::dbms::DatabaseAccess db_acc, InterpreterContext *interpreter_context, - const std::string *username) { + const std::optional &username) { static constexpr std::string_view kDefaultConsumerGroup = "mg_consumer"; std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup : stream_query->consumer_group_}; @@ -839,7 +873,7 @@ Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, Exp return [db_acc = std::move(db_acc), interpreter_context, stream_name = stream_query->stream_name_, topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_), consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info), - bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username), + bootstrap_servers = std::move(bootstrap), owner = username, configs = get_config_map(stream_query->configs_, "Configs"), credentials = get_config_map(stream_query->credentials_, "Credentials"), default_server = interpreter_context->config.default_kafka_bootstrap_servers]() mutable { @@ -861,7 +895,7 @@ Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, Exp Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, ExpressionVisitor &evaluator, memgraph::dbms::DatabaseAccess db, InterpreterContext *interpreter_context, - const std::string *username) { + const std::optional &username) { auto service_url = GetOptionalStringValue(stream_query->service_url_, evaluator); if (service_url && service_url->empty()) { throw SemanticException("Service URL must not be an empty string!"); @@ -871,8 +905,7 @@ Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, Ex return [db = std::move(db), interpreter_context, stream_name = stream_query->stream_name_, topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_), - common_stream_info = std::move(common_stream_info), service_url = std::move(service_url), - owner = StringPointerToOptional(username), + common_stream_info = std::move(common_stream_info), service_url = std::move(service_url), owner = username, default_service = interpreter_context->config.default_pulsar_service_url]() mutable { std::string url = service_url ? std::move(*service_url) : std::move(default_service); db->streams()->Create( @@ -886,7 +919,7 @@ Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, Ex Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters ¶meters, memgraph::dbms::DatabaseAccess &db_acc, InterpreterContext *interpreter_context, - const std::string *username, std::vector *notifications) { + const std::optional &username, std::vector *notifications) { // TODO: MemoryResource for EvaluationContext, it should probably be passed as // the argument to Callback. EvaluationContext evaluation_context; @@ -1353,18 +1386,19 @@ bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage, const query::plan::ReadWriteTypeChecker::RWType query_type) { if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL || storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) { - return (storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) && - (query_type == RWType::W || query_type == RWType::RW); + auto const &replState = storage->repl_state_; + return replState.IsReplica() && (query_type == RWType::W || query_type == RWType::RW); } return false; } -storage::replication::ReplicationRole GetReplicaRole(storage::Storage *storage) { +bool IsReplica(storage::Storage *storage) { if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL || storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) { - return storage->GetReplicationRole(); + auto const &replState = storage->repl_state_; + return replState.IsReplica(); } - return storage::replication::ReplicationRole::MAIN; + return false; } } // namespace @@ -1385,13 +1419,13 @@ Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_ #ifndef MG_ENTERPRISE auto db_acc = interpreter_context_->db_gatekeeper->access(); MG_ASSERT(db_acc, "Database accessor needs to be valid"); - db_acc_ = std::move(db_acc); + current_db_.db_acc_ = std::move(db_acc); #endif } Interpreter::Interpreter(InterpreterContext *interpreter_context, memgraph::dbms::DatabaseAccess db) - : db_acc_(std::move(db)), interpreter_context_(interpreter_context) { - MG_ASSERT(db_acc_, "Database accessor needs to be valid"); + : current_db_{std::move(db)}, interpreter_context_(interpreter_context) { + MG_ASSERT(current_db_.db_acc_, "Database accessor needs to be valid"); MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL"); } @@ -1412,37 +1446,31 @@ auto DetermineTxTimeout(std::optional tx_timeout_ms, InterpreterConfig return TxTimeout{}; } +auto CreateTimeoutTimer(QueryExtras const &extras, InterpreterConfig const &config) + -> std::shared_ptr { + if (auto const timeout = DetermineTxTimeout(extras.tx_timeout, config)) { + return std::make_shared(timeout.ValueUnsafe().count()); + } + return {}; +} + PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper, QueryExtras const &extras) { std::function handler; if (query_upper == "BEGIN") { + query_executions_.clear(); + transaction_queries_->clear(); // TODO: Evaluate doing move(extras). Currently the extras is very small, but this will be important if it ever // becomes large. handler = [this, extras = extras] { if (in_explicit_transaction_) { throw ExplicitTransactionUsageException("Nested transactions are not supported."); } - - memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTransactions); - + SetupInterpreterTransaction(extras); in_explicit_transaction_ = true; expect_rollback_ = false; - metadata_ = GenOptional(extras.metadata_pv); - - auto const timeout = DetermineTxTimeout(extras.tx_timeout, interpreter_context_->config); - explicit_transaction_timer_ = - timeout ? std::make_shared(timeout.ValueUnsafe().count()) : nullptr; - - if (!db_acc_) throw DatabaseContextRequiredException("No current database for transaction defined."); - - auto &db_acc = *db_acc_; - db_accessor_ = db_acc->Access(GetIsolationLevelOverride()); - execution_db_accessor_.emplace(db_accessor_.get()); - transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release); - - if (db_acc->trigger_store()->HasTriggers()) { - trigger_context_collector_.emplace(db_acc->trigger_store()->GetEventTypes()); - } + if (!current_db_.db_acc_) throw DatabaseContextRequiredException("No current database for transaction defined."); + SetupDatabaseTransaction(true); }; } else if (query_upper == "COMMIT") { handler = [this] { @@ -1465,7 +1493,7 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper, expect_rollback_ = false; in_explicit_transaction_ = false; metadata_ = std::nullopt; - explicit_transaction_timer_.reset(); + current_timeout_timer_.reset(); }; } else if (query_upper == "ROLLBACK") { handler = [this] { @@ -1479,7 +1507,7 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper, expect_rollback_ = false; in_explicit_transaction_ = false; metadata_ = std::nullopt; - explicit_transaction_timer_.reset(); + current_timeout_timer_.reset(); }; } else { LOG_FATAL("Should not get here -- unknown transaction query!"); @@ -1510,36 +1538,36 @@ inline static void TryCaching(const AstStorage &ast_storage, FrameChangeCollecto } } +bool IsLoadCsvQuery(const std::vector &clauses) { + return std::any_of(clauses.begin(), clauses.end(), + [](memgraph::query::Clause const *clause) { return clause->GetTypeInfo() == LoadCsv::kType; }); +} + bool IsCallBatchedProcedureQuery(const std::vector &clauses) { EvaluationContext evaluation_context; - return std::ranges::any_of(clauses, [&evaluation_context](const auto &clause) -> bool { - if (clause->GetTypeInfo() == CallProcedure::kType) { - auto *call_procedure_clause = utils::Downcast(clause); + return std::ranges::any_of(clauses, [&evaluation_context](memgraph::query::Clause *clause) -> bool { + if (!(clause->GetTypeInfo() == CallProcedure::kType)) return false; + auto *call_procedure_clause = utils::Downcast(clause); - const auto &maybe_found = memgraph::query::procedure::FindProcedure( - procedure::gModuleRegistry, call_procedure_clause->procedure_name_, evaluation_context.memory); - if (!maybe_found) { - throw QueryRuntimeException("There is no procedure named '{}'.", call_procedure_clause->procedure_name_); - } - const auto &[module, proc] = *maybe_found; - if (proc->info.is_batched) { - spdlog::trace("Using PoolResource for batched query procedure"); - return true; - } + const auto &maybe_found = memgraph::query::procedure::FindProcedure( + procedure::gModuleRegistry, call_procedure_clause->procedure_name_, evaluation_context.memory); + if (!maybe_found) { + throw QueryRuntimeException("There is no procedure named '{}'.", call_procedure_clause->procedure_name_); } - return false; + const auto &[module, proc] = *maybe_found; + if (!proc->info.is_batched) return false; + spdlog::trace("Using PoolResource for batched query procedure"); + return true; }); - - return false; } PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map *summary, - InterpreterContext *interpreter_context, DbAccessor *dba, + InterpreterContext *interpreter_context, CurrentDB ¤t_db, utils::MemoryResource *execution_memory, std::vector *notifications, - const std::string *username, std::atomic *transaction_status, - std::shared_ptr tx_timer, auto *plan_cache, - TriggerContextCollector *trigger_context_collector = nullptr, + std::optional const &username, + std::atomic *transaction_status, + std::shared_ptr tx_timer, FrameChangeCollector *frame_change_collector = nullptr) { auto *cypher_query = utils::Downcast(parsed_query.query); @@ -1570,6 +1598,14 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::mapget()->plan_cache() : nullptr; + auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, parsed_query.parameters, plan_cache, dba); @@ -1590,11 +1626,13 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory, - StringPointerToOptional(username), transaction_status, std::move(tx_timer), - trigger_context_collector, memory_limit, use_monotonic_memory, - frame_change_collector->IsTrackingValues() ? frame_change_collector : nullptr); + // TODO: pass current DB into plan, in future current can change during pull + auto *trigger_context_collector = + current_db.trigger_context_collector_ ? &*current_db.trigger_context_collector_ : nullptr; + auto pull_plan = std::make_shared( + plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory, username, transaction_status, + std::move(tx_timer), trigger_context_collector, memory_limit, use_monotonic_memory, + frame_change_collector->IsTrackingValues() ? frame_change_collector : nullptr); return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), [pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary]( AnyStream *stream, std::optional n) -> std::optional { @@ -1607,7 +1645,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map *summary, - InterpreterContext *interpreter_context, DbAccessor *dba, auto *plan_cache) { + InterpreterContext *interpreter_context, CurrentDB ¤t_db) { const std::string kExplainQueryStart = "explain "; MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart), "Expected stripped query to start with '{}'", kExplainQueryStart); @@ -1625,9 +1663,14 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map(parsed_inner_query.query); MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN"); - auto cypher_query_plan = CypherQueryToPlan( - parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, - parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? plan_cache : nullptr, dba); + MG_ASSERT(current_db.execution_db_accessor_, "Explain query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + + auto *plan_cache = parsed_inner_query.is_cacheable ? current_db.db_acc_->get()->plan_cache() : nullptr; + + auto cypher_query_plan = + CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), + cypher_query, parsed_inner_query.parameters, plan_cache, dba); std::stringstream printed_plan; plan::PrettyPrint(*dba, &cypher_query_plan->plan(), &printed_plan); @@ -1653,9 +1696,10 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map *summary, InterpreterContext *interpreter_context, - DbAccessor *dba, utils::MemoryResource *execution_memory, const std::string *username, + CurrentDB ¤t_db, utils::MemoryResource *execution_memory, + std::optional const &username, std::atomic *transaction_status, - std::shared_ptr tx_timer, auto *plan_cache, + std::shared_ptr tx_timer, FrameChangeCollector *frame_change_collector) { const std::string kProfileQueryStart = "profile "; @@ -1713,19 +1757,22 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context}; const auto memory_limit = EvaluateMemoryLimit(evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_); - auto cypher_query_plan = CypherQueryToPlan( - parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query, - parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? plan_cache : nullptr, dba); + MG_ASSERT(current_db.execution_db_accessor_, "Profile query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + + auto *plan_cache = parsed_inner_query.is_cacheable ? current_db.db_acc_->get()->plan_cache() : nullptr; + auto cypher_query_plan = + CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), + cypher_query, parsed_inner_query.parameters, plan_cache, dba); TryCaching(cypher_query_plan->ast_storage(), frame_change_collector); auto rw_type_checker = plan::ReadWriteTypeChecker(); - auto optional_username = StringPointerToOptional(username); rw_type_checker.InferRWType(const_cast(cypher_query_plan->plan())); return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"}, std::move(parsed_query.required_privileges), [plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), - summary, dba, interpreter_context, execution_memory, memory_limit, optional_username, + summary, dba, interpreter_context, execution_memory, memory_limit, username, // We want to execute the query we are profiling lazily, so we delay // the construction of the corresponding context. stats_and_total_time = std::optional{}, @@ -1735,9 +1782,9 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra // No output symbols are given so that nothing is streamed. if (!stats_and_total_time) { stats_and_total_time = - PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, - optional_username, transaction_status, std::move(tx_timer), nullptr, - memory_limit, use_monotonic_memory, + PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, username, + transaction_status, std::move(tx_timer), nullptr, memory_limit, + use_monotonic_memory, frame_change_collector->IsTrackingValues() ? frame_change_collector : nullptr) .Pull(stream, {}, {}, summary); pull_plan = std::make_shared(ProfilingStatsToTable(*stats_and_total_time)); @@ -1755,18 +1802,20 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra rw_type_checker.type}; } -PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map *summary, DbAccessor *dba, - utils::MemoryResource *execution_memory) { - return PreparedQuery{{"QUERY"}, - std::move(parsed_query.required_privileges), - [pull_plan = std::make_shared(dba)]( - AnyStream *stream, std::optional n) -> std::optional { - if (pull_plan->Pull(stream, n)) { - return QueryHandlerResult::COMMIT; - } - return std::nullopt; - }, - RWType::R}; +PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, CurrentDB ¤t_db) { + MG_ASSERT(current_db.execution_db_accessor_, "Dump query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + auto plan = std::make_shared(dba); + return PreparedQuery{ + {"QUERY"}, + std::move(parsed_query.required_privileges), + [pull_plan = std::move(plan)](AnyStream *stream, std::optional n) -> std::optional { + if (pull_plan->Pull(stream, n)) { + return QueryHandlerResult::COMMIT; + } + return std::nullopt; + }, + RWType::R}; } std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreateStatistics( @@ -1832,12 +1881,14 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat // Iterate over all label property indexed vertices std::for_each( index_info.begin(), index_info.end(), - [execution_db_accessor, &label_property_counter, &vertex_degree_counter, view](const LPIndex &index_info) { - auto vertices = execution_db_accessor->Vertices(view, index_info.first, index_info.second); + [execution_db_accessor, &label_property_counter, &vertex_degree_counter, view](const LPIndex &index_element) { + auto &lp_counter = label_property_counter[index_element]; + auto &vd_counter = vertex_degree_counter[index_element]; + auto vertices = execution_db_accessor->Vertices(view, index_element.first, index_element.second); std::for_each(vertices.begin(), vertices.end(), - [&index_info, &label_property_counter, &vertex_degree_counter, &view](const auto &vertex) { - label_property_counter[index_info][*vertex.GetProperty(view, index_info.second)]++; - vertex_degree_counter[index_info] += *vertex.OutDegree(view) + *vertex.InDegree(view); + [&index_element, &lp_counter, &vd_counter, &view](const auto &vertex) { + lp_counter[*vertex.GetProperty(view, index_element.second)]++; + vd_counter += *vertex.OutDegree(view) + *vertex.InDegree(view); }); }); @@ -1923,18 +1974,78 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphDeleteStatistics( const std::span labels, DbAccessor *execution_db_accessor) { - std::vector> label_prop_results; - std::vector label_results; - if (labels[0] == kAsterisk) { - label_prop_results = execution_db_accessor->ClearLabelPropertyIndexStats(); - label_results = execution_db_accessor->ClearLabelIndexStats(); - } else { - label_prop_results = execution_db_accessor->DeleteLabelPropertyIndexStats(labels); - label_results = execution_db_accessor->DeleteLabelIndexStats(labels); - } + auto erase_not_specified_label_indices = [&labels, execution_db_accessor](auto &index_info) { + if (labels[0] == kAsterisk) { + return; + } + + for (auto it = index_info.cbegin(); it != index_info.cend();) { + if (std::find(labels.begin(), labels.end(), execution_db_accessor->LabelToName(*it)) == labels.end()) { + it = index_info.erase(it); + } else { + ++it; + } + } + }; + + auto erase_not_specified_label_property_indices = [&labels, execution_db_accessor](auto &index_info) { + if (labels[0] == kAsterisk) { + return; + } + + for (auto it = index_info.cbegin(); it != index_info.cend();) { + if (std::find(labels.begin(), labels.end(), execution_db_accessor->LabelToName(it->first)) == labels.end()) { + it = index_info.erase(it); + } else { + ++it; + } + } + }; + + auto populate_label_results = [execution_db_accessor](auto index_info) { + std::vector label_results; + label_results.reserve(index_info.size()); + std::for_each(index_info.begin(), index_info.end(), + [execution_db_accessor, &label_results](const storage::LabelId &label_id) { + const auto res = execution_db_accessor->DeleteLabelIndexStats(label_id); + if (res) label_results.emplace_back(label_id); + }); + + return label_results; + }; + + auto populate_label_property_results = [execution_db_accessor](auto index_info) { + std::vector> label_property_results; + label_property_results.reserve(index_info.size()); + std::for_each(index_info.begin(), index_info.end(), + [execution_db_accessor, + &label_property_results](const std::pair &label_property) { + const auto &res = execution_db_accessor->DeleteLabelPropertyIndexStats(label_property.first); + label_property_results.insert(label_property_results.end(), res.begin(), res.end()); + }); + + return label_property_results; + }; + + auto index_info = execution_db_accessor->ListAllIndices(); + + std::vector label_indices_info = index_info.label; + erase_not_specified_label_indices(label_indices_info); + auto label_results = populate_label_results(label_indices_info); + + std::vector> label_property_indices_info = index_info.label_property; + erase_not_specified_label_property_indices(label_property_indices_info); + auto label_prop_results = populate_label_property_results(label_property_indices_info); std::vector> results; - results.reserve(label_prop_results.size() + label_results.size()); + results.reserve(label_results.size() + label_prop_results.size()); + + std::transform( + label_results.begin(), label_results.end(), std::back_inserter(results), + [execution_db_accessor](const auto &label_index) { + return std::vector{TypedValue(execution_db_accessor->LabelToName(label_index)), TypedValue("")}; + }); + std::transform(label_prop_results.begin(), label_prop_results.end(), std::back_inserter(results), [execution_db_accessor](const auto &label_property_index) { return std::vector{ @@ -1942,11 +2053,6 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphDelet TypedValue(execution_db_accessor->PropertyToName(label_property_index.second))}; }); - std::transform( - label_results.begin(), label_results.end(), std::back_inserter(results), - [execution_db_accessor](const auto &label_index) { - return std::vector{TypedValue(execution_db_accessor->LabelToName(label_index)), TypedValue("")}; - }); return results; } @@ -1976,14 +2082,14 @@ Callback HandleAnalyzeGraphQuery(AnalyzeGraphQuery *analyze_graph_query, DbAcces return callback; } -PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - DbAccessor *execution_db_accessor, auto *plan_cache) { +PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw AnalyzeGraphInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Analyze Graph query expects a current DB"); // Creating an index influences computed plan costs. - auto invalidate_plan_cache = [plan_cache] { + auto invalidate_plan_cache = [plan_cache = current_db.db_acc_->get()->plan_cache()] { auto access = plan_cache->access(); for (auto &kv : access) { access.remove(kv.first); @@ -1993,7 +2099,11 @@ PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explici auto *analyze_graph_query = utils::Downcast(parsed_query.query); MG_ASSERT(analyze_graph_query); - auto callback = HandleAnalyzeGraphQuery(analyze_graph_query, execution_db_accessor); + + MG_ASSERT(current_db.execution_db_accessor_, "Analyze Graph query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + + auto callback = HandleAnalyzeGraphQuery(analyze_graph_query, dba); return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges), [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( @@ -2011,7 +2121,7 @@ PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explici } PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::vector *notifications, storage::Storage *storage, auto *plan_cache) { + std::vector *notifications, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw IndexInMulticommandTxException(); } @@ -2019,14 +2129,22 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans auto *index_query = utils::Downcast(parsed_query.query); std::function handler; + // TODO: we will need transaction for replication + MG_ASSERT(current_db.db_acc_, "Index query expects a current DB"); + auto &db_acc = *current_db.db_acc_; + + MG_ASSERT(current_db.db_transactional_accessor_, "Index query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + // Creating an index influences computed plan costs. - auto invalidate_plan_cache = [plan_cache] { + auto invalidate_plan_cache = [plan_cache = db_acc->plan_cache()] { auto access = plan_cache->access(); for (auto &kv : access) { access.remove(kv.first); } }; + auto *storage = db_acc->storage(); auto label = storage->NameToLabel(index_query->label_.name); std::vector properties; @@ -2050,35 +2168,19 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans index_notification.title = fmt::format("Created index on label {} on properties {}.", index_query->label_.name, properties_stringified); - handler = [storage, label, properties_stringified = std::move(properties_stringified), + // TODO: not just storage + invalidate_plan_cache. Need a DB transaction (for replication) + handler = [dba, label, properties_stringified = std::move(properties_stringified), label_name = index_query->label_.name, properties = std::move(properties), invalidate_plan_cache = std::move(invalidate_plan_cache)](Notification &index_notification) { MG_ASSERT(properties.size() <= 1U); - auto maybe_index_error = - properties.empty() ? storage->CreateIndex(label) : storage->CreateIndex(label, properties[0]); + auto maybe_index_error = properties.empty() ? dba->CreateIndex(label) : dba->CreateIndex(label, properties[0]); utils::OnScopeExit invalidator(invalidate_plan_cache); if (maybe_index_error.HasError()) { - const auto &error = maybe_index_error.GetError(); - std::visit( - [&index_notification, &label_name, &properties_stringified](T &&) { - using ErrorType = std::remove_cvref_t; - if constexpr (std::is_same_v) { - throw ReplicationException( - fmt::format("At least one SYNC replica has not confirmed the creation of the index on label {} " - "on properties {}.", - label_name, properties_stringified)); - } else if constexpr (std::is_same_v) { - index_notification.code = NotificationCode::EXISTENT_INDEX; - index_notification.title = fmt::format("Index on label {} on properties {} already exists.", - label_name, properties_stringified); - } else if constexpr (std::is_same_v) { - throw IndexPersistenceException(); - } else { - static_assert(kAlwaysFalse, "Missing type from variant visitor"); - } - }, - error); + index_notification.code = NotificationCode::EXISTENT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} already exists.", label_name, properties_stringified); + // ABORT? } }; break; @@ -2087,35 +2189,18 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans index_notification.code = NotificationCode::DROP_INDEX; index_notification.title = fmt::format("Dropped index on label {} on properties {}.", index_query->label_.name, utils::Join(properties_string, ", ")); - handler = [storage, label, properties_stringified = std::move(properties_stringified), + // TODO: not just storage + invalidate_plan_cache. Need a DB transaction (for replication) + handler = [dba, label, properties_stringified = std::move(properties_stringified), label_name = index_query->label_.name, properties = std::move(properties), invalidate_plan_cache = std::move(invalidate_plan_cache)](Notification &index_notification) { MG_ASSERT(properties.size() <= 1U); - auto maybe_index_error = - properties.empty() ? storage->DropIndex(label) : storage->DropIndex(label, properties[0]); + auto maybe_index_error = properties.empty() ? dba->DropIndex(label) : dba->DropIndex(label, properties[0]); utils::OnScopeExit invalidator(invalidate_plan_cache); if (maybe_index_error.HasError()) { - const auto &error = maybe_index_error.GetError(); - std::visit( - [&index_notification, &label_name, &properties_stringified](T &&) { - using ErrorType = std::remove_cvref_t; - if constexpr (std::is_same_v) { - throw ReplicationException( - fmt::format("At least one SYNC replica has not confirmed the dropping of the index on label {} " - "on properties {}.", - label_name, properties_stringified)); - } else if constexpr (std::is_same_v) { - index_notification.code = NotificationCode::NONEXISTENT_INDEX; - index_notification.title = fmt::format("Index on label {} on properties {} doesn't exist.", - label_name, properties_stringified); - } else if constexpr (std::is_same_v) { - throw IndexPersistenceException(); - } else { - static_assert(kAlwaysFalse, "Missing type from variant visitor"); - } - }, - error); + index_notification.code = NotificationCode::NONEXISTENT_INDEX; + index_notification.title = + fmt::format("Index on label {} on properties {} doesn't exist.", label_name, properties_stringified); } }; break; @@ -2129,7 +2214,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans AnyStream * /*stream*/, std::optional /*unused*/) mutable { handler(index_notification); notifications->push_back(index_notification); - return QueryHandlerResult::NOTHING; + return QueryHandlerResult::COMMIT; // TODO: Will need to become COMMIT when we fix replication }, RWType::W}; } @@ -2167,12 +2252,15 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa } PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::vector *notifications, storage::Storage *storage, + std::vector *notifications, CurrentDB ¤t_db, const InterpreterConfig &config) { if (in_explicit_transaction) { throw ReplicationModificationInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Replication query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw ReplicationDisabledOnDiskStorage(); } @@ -2197,11 +2285,14 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } -PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_transaction, storage::Storage *storage) { +PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw LockPathModificationInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Lock Path query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw LockPathDisabledOnDiskStorage(); } @@ -2254,12 +2345,14 @@ PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_tr RWType::NONE}; } -PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - storage::Storage *storage) { +PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw FreeMemoryModificationInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Free Memory query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw FreeMemoryDisabledOnDiskStorage(); } @@ -2382,20 +2475,25 @@ Callback ShowTriggers(TriggerStore *trigger_store) { } PreparedQuery PrepareTriggerQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::vector *notifications, TriggerStore *trigger_store, - InterpreterContext *interpreter_context, DbAccessor *dba, + std::vector *notifications, CurrentDB ¤t_db, + InterpreterContext *interpreter_context, const std::map &user_parameters, - const std::string *username) { + std::optional const &username) { if (in_explicit_transaction) { throw TriggerModificationInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Trigger query expects a current DB"); + TriggerStore *trigger_store = current_db.db_acc_->get()->trigger_store(); + MG_ASSERT(current_db.execution_db_accessor_, "Trigger query expects a current DB transaction"); + DbAccessor *dba = &*current_db.execution_db_accessor_; + auto *trigger_query = utils::Downcast(parsed_query.query); MG_ASSERT(trigger_query); std::optional trigger_notification; auto callback = std::invoke([trigger_query, trigger_store, interpreter_context, dba, &user_parameters, - owner = StringPointerToOptional(username), &trigger_notification]() mutable { + owner = username, &trigger_notification]() mutable { switch (trigger_query->action_) { case TriggerQuery::Action::CREATE_TRIGGER: trigger_notification.emplace(SeverityLevel::INFO, NotificationCode::CREATE_TRIGGER, @@ -2432,12 +2530,15 @@ PreparedQuery PrepareTriggerQuery(ParsedQuery parsed_query, bool in_explicit_tra } PreparedQuery PrepareStreamQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::vector *notifications, memgraph::dbms::DatabaseAccess &db_acc, - InterpreterContext *interpreter_context, const std::string *username) { + std::vector *notifications, CurrentDB ¤t_db, + InterpreterContext *interpreter_context, const std::optional &username) { if (in_explicit_transaction) { throw StreamQueryInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Stream query expects a current DB"); + auto &db_acc = *current_db.db_acc_; + auto *stream_query = utils::Downcast(parsed_query.query); MG_ASSERT(stream_query); auto callback = @@ -2502,7 +2603,7 @@ bool SwitchingFromDiskToInMemory(storage::StorageMode current_mode, storage::Sto } PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, - storage::Storage *storage, Interpreter *interpreter) { + CurrentDB ¤t_db, Interpreter *interpreter) { if (in_explicit_transaction) { throw IsolationLevelModificationInMulticommandTxException(); } @@ -2512,24 +2613,33 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in const auto isolation_level = ToStorageIsolationLevel(isolation_level_query->isolation_level_); - auto callback = [isolation_level_query, isolation_level, storage, interpreter]() -> std::function { - switch (isolation_level_query->isolation_level_scope_) { - case IsolationLevelQuery::IsolationLevelScope::GLOBAL: - return [storage, isolation_level] { - if (auto maybe_error = storage->SetIsolationLevel(isolation_level); maybe_error.HasError()) { - switch (maybe_error.GetError()) { - case storage::Storage::SetIsolationLevelError::DisabledForAnalyticalMode: - throw IsolationLevelModificationInAnalyticsException(); - break; - } + std::function callback; + switch (isolation_level_query->isolation_level_scope_) { + case IsolationLevelQuery::IsolationLevelScope::GLOBAL: // TODO:.....not GLOBAL is it?! + { + MG_ASSERT(current_db.db_acc_, "Storage Isolation Level query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + callback = [storage, isolation_level] { + if (auto maybe_error = storage->SetIsolationLevel(isolation_level); maybe_error.HasError()) { + switch (maybe_error.GetError()) { + case storage::Storage::SetIsolationLevelError::DisabledForAnalyticalMode: + throw IsolationLevelModificationInAnalyticsException(); + break; } - }; - case IsolationLevelQuery::IsolationLevelScope::SESSION: - return [interpreter, isolation_level] { interpreter->SetSessionIsolationLevel(isolation_level); }; - case IsolationLevelQuery::IsolationLevelScope::NEXT: - return [interpreter, isolation_level] { interpreter->SetNextTransactionIsolationLevel(isolation_level); }; + } + }; + break; } - }(); + + case IsolationLevelQuery::IsolationLevelScope::SESSION: { + callback = [interpreter, isolation_level] { interpreter->SetSessionIsolationLevel(isolation_level); }; + break; + } + case IsolationLevelQuery::IsolationLevelScope::NEXT: { + callback = [interpreter, isolation_level] { interpreter->SetNextTransactionIsolationLevel(isolation_level); }; + break; + } + } return PreparedQuery{ {}, @@ -2603,29 +2713,32 @@ bool ActiveTransactionsExist(InterpreterContext *interpreter_context) { } PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, - memgraph::dbms::DatabaseAccess &db, InterpreterContext *interpreter_context) { + CurrentDB ¤t_db, InterpreterContext *interpreter_context) { if (in_explicit_transaction) { throw StorageModeModificationInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Storage Mode query expects a current DB"); + memgraph::dbms::DatabaseAccess &db_acc = *current_db.db_acc_; auto *storage_mode_query = utils::Downcast(parsed_query.query); MG_ASSERT(storage_mode_query); const auto requested_mode = ToStorageMode(storage_mode_query->storage_mode_); - auto current_mode = db->GetStorageMode(); + auto current_mode = db_acc->GetStorageMode(); std::function callback; if (current_mode == storage::StorageMode::ON_DISK_TRANSACTIONAL || requested_mode == storage::StorageMode::ON_DISK_TRANSACTIONAL) { - callback = SwitchMemoryDevice(current_mode, requested_mode, db).fn; + callback = SwitchMemoryDevice(current_mode, requested_mode, db_acc).fn; } else { + // TODO: this needs to be filtered to just db_acc->storage() if (ActiveTransactionsExist(interpreter_context)) { spdlog::info( "Storage mode will be modified when there are no other active transactions. Check the status of the " "transactions using 'SHOW TRANSACTIONS' query and ensure no other transactions are active."); } - callback = [requested_mode, storage = db->storage()]() -> std::function { + callback = [requested_mode, storage = db_acc->storage()]() -> std::function { // SetStorageMode will probably be handled at the Database level return [storage, requested_mode] { storage->SetStorageMode(requested_mode); }; }(); @@ -2641,13 +2754,11 @@ PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_ex RWType::NONE}; } -PreparedQuery PrepareEdgeImportModeQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, - storage::Storage *db) { - if (in_explicit_transaction) { - throw EdgeImportModeModificationInMulticommandTxException(); - } +PreparedQuery PrepareEdgeImportModeQuery(ParsedQuery parsed_query, CurrentDB ¤t_db) { + MG_ASSERT(current_db.db_acc_, "Edge Import query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); - if (db->GetStorageMode() != storage::StorageMode::ON_DISK_TRANSACTIONAL) { + if (storage->GetStorageMode() != storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw EdgeImportModeQueryDisabledOnDiskStorage(); } @@ -2655,9 +2766,9 @@ PreparedQuery PrepareEdgeImportModeQuery(ParsedQuery parsed_query, const bool in MG_ASSERT(edge_import_mode_query); const auto requested_status = ToEdgeImportMode(edge_import_mode_query->status_); - auto callback = [requested_status, db]() -> std::function { - return [db, requested_status] { - auto *disk_storage = static_cast(db); + auto callback = [requested_status, storage]() -> std::function { + return [storage, requested_status] { + auto *disk_storage = static_cast(storage); disk_storage->SetEdgeImportMode(requested_status); }; }(); @@ -2673,11 +2784,14 @@ PreparedQuery PrepareEdgeImportModeQuery(ParsedQuery parsed_query, const bool in } PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - storage::Storage *storage) { + CurrentDB ¤t_db) { if (in_explicit_transaction) { throw CreateSnapshotInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Create Snapshot query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw CreateSnapshotDisabledOnDiskStorage(); } @@ -2687,7 +2801,7 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli std::move(parsed_query.required_privileges), [storage](AnyStream * /*stream*/, std::optional /*n*/) -> std::optional { auto *mem_storage = static_cast(storage); - if (auto maybe_error = mem_storage->CreateSnapshot({}); maybe_error.HasError()) { + if (auto maybe_error = mem_storage->CreateSnapshot(storage->repl_state_, {}); maybe_error.HasError()) { switch (maybe_error.GetError()) { case storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica: throw utils::BasicException( @@ -2706,7 +2820,7 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli RWType::NONE}; } -PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, bool in_explicit_transaction, DbAccessor *dba) { +PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, bool in_explicit_transaction) { if (in_explicit_transaction) { throw SettingConfigInMulticommandTxException{}; } @@ -2732,9 +2846,9 @@ PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, bool in_explicit_tra // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) } -std::vector> TransactionQueueQueryHandler::ShowTransactions( - const std::unordered_set &interpreters, const std::optional &username, - bool hasTransactionManagementPrivilege, std::optional &filter_db_acc) { +template +auto ShowTransactions(const std::unordered_set &interpreters, const std::optional &username, + Func &&privilege_checker) -> std::vector> { std::vector> results; results.reserve(interpreters.size()); for (Interpreter *interpreter : interpreters) { @@ -2747,9 +2861,14 @@ std::vector> TransactionQueueQueryHandler::ShowTransacti utils::OnScopeExit clean_status([interpreter]() { interpreter->transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release); }); - if (interpreter->db_acc_ != filter_db_acc) continue; std::optional transaction_id = interpreter->GetTransactionId(); - if (transaction_id.has_value() && (interpreter->username_ == username || hasTransactionManagementPrivilege)) { + + auto get_interpreter_db_name = [&]() -> std::string const & { + static std::string all; + return interpreter->current_db_.db_acc_ ? interpreter->current_db_.db_acc_->get()->id() : all; + }; + if (transaction_id.has_value() && + (interpreter->username_ == username || privilege_checker(get_interpreter_db_name()))) { const auto &typed_queries = interpreter->GetQueries(); results.push_back({TypedValue(interpreter->username_.value_or("")), TypedValue(std::to_string(transaction_id.value())), TypedValue(typed_queries)}); @@ -2768,50 +2887,37 @@ std::vector> TransactionQueueQueryHandler::ShowTransacti Callback HandleTransactionQueueQuery(TransactionQueueQuery *transaction_query, const std::optional &username, const Parameters ¶meters, - InterpreterContext *interpreter_context, - memgraph::query::Interpreter &interpreter) { - EvaluationContext evaluation_context; - evaluation_context.timestamp = QueryTimestamp(); - evaluation_context.parameters = parameters; - auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context}; - - bool hasTransactionManagementPrivilege = interpreter_context->auth_checker->IsUserAuthorized( - username, {query::AuthQuery::Privilege::TRANSACTION_MANAGEMENT}, ""); - - if (!interpreter.db_acc_) { - // TODO: remove in future when we have cypher execution which can happen without any database transaction - // ie. when we have a transaction ID not tied to storage transaction - throw DatabaseContextRequiredException("No current database for transaction defined."); - } + InterpreterContext *interpreter_context) { + auto privilege_checker = [username, auth_checker = interpreter_context->auth_checker](std::string const &db_name) { + return auth_checker->IsUserAuthorized(username, {query::AuthQuery::Privilege::TRANSACTION_MANAGEMENT}, db_name); + }; Callback callback; switch (transaction_query->action_) { case TransactionQueueQuery::Action::SHOW_TRANSACTIONS: { + auto show_transactions = [username, privilege_checker = std::move(privilege_checker)](const auto &interpreters) { + return ShowTransactions(interpreters, username, privilege_checker); + }; callback.header = {"username", "transaction_id", "query", "metadata"}; - callback.fn = [handler = TransactionQueueQueryHandler(), interpreter_context, username, - hasTransactionManagementPrivilege, db_acc = &interpreter.db_acc_]() mutable { - std::vector> results; + callback.fn = [interpreter_context, show_transactions = std::move(show_transactions)] { // Multiple simultaneous SHOW TRANSACTIONS aren't allowed - interpreter_context->interpreters.WithLock( - [&results, handler, username, hasTransactionManagementPrivilege, db_acc](const auto &interpreters) { - results = handler.ShowTransactions(interpreters, username, hasTransactionManagementPrivilege, *db_acc); - }); - return results; + return interpreter_context->interpreters.WithLock(show_transactions); }; break; } case TransactionQueueQuery::Action::TERMINATE_TRANSACTIONS: { + auto evaluation_context = EvaluationContext{.timestamp = QueryTimestamp(), .parameters = parameters}; + auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context}; std::vector maybe_kill_transaction_ids; std::transform(transaction_query->transaction_id_list_.begin(), transaction_query->transaction_id_list_.end(), std::back_inserter(maybe_kill_transaction_ids), [&evaluator](Expression *expression) { return std::string(expression->Accept(evaluator).ValueString()); }); callback.header = {"transaction_id", "killed"}; - callback.fn = [handler = TransactionQueueQueryHandler(), interpreter_context, - maybe_kill_transaction_ids = std::move(maybe_kill_transaction_ids), username, - hasTransactionManagementPrivilege, interpreter = &interpreter]() mutable { - return interpreter_context->KillTransactions(std::move(maybe_kill_transaction_ids), username, - hasTransactionManagementPrivilege, *interpreter); + callback.fn = [interpreter_context, maybe_kill_transaction_ids = std::move(maybe_kill_transaction_ids), username, + privilege_checker = std::move(privilege_checker)]() mutable { + return interpreter_context->TerminateTransactions(std::move(maybe_kill_transaction_ids), username, + std::move(privilege_checker)); }; break; } @@ -2821,16 +2927,11 @@ Callback HandleTransactionQueueQuery(TransactionQueueQuery *transaction_query, } PreparedQuery PrepareTransactionQueueQuery(ParsedQuery parsed_query, const std::optional &username, - bool in_explicit_transaction, InterpreterContext *interpreter_context, - memgraph::query::Interpreter &interpreter) { - if (in_explicit_transaction) { - throw TransactionQueueInMulticommandTxException(); - } - + InterpreterContext *interpreter_context) { auto *transaction_queue_query = utils::Downcast(parsed_query.query); MG_ASSERT(transaction_queue_query); - auto callback = HandleTransactionQueueQuery(transaction_queue_query, username, parsed_query.parameters, - interpreter_context, interpreter); + auto callback = + HandleTransactionQueueQuery(transaction_queue_query, username, parsed_query.parameters, interpreter_context); return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges), [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr{nullptr}]( @@ -2865,48 +2966,26 @@ PreparedQuery PrepareVersionQuery(ParsedQuery parsed_query, bool in_explicit_tra RWType::NONE}; } -PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::map * /*summary*/, storage::Storage *storage, - utils::MemoryResource * /*execution_memory*/, - std::optional interpreter_isolation_level, - std::optional next_transaction_isolation_level) { +PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw InfoInMulticommandTxException(); } - auto *info_query = utils::Downcast(parsed_query.query); + MG_ASSERT(current_db.db_acc_, "Database info query expects a current DB"); + MG_ASSERT(current_db.db_transactional_accessor_, "Database ifo query expects a current DB transaction"); + auto *dba = &*current_db.execution_db_accessor_; + + auto *info_query = utils::Downcast(parsed_query.query); std::vector header; std::function>, QueryHandlerResult>()> handler; switch (info_query->info_type_) { - case InfoQuery::InfoType::STORAGE: - header = {"storage info", "value"}; - - handler = [storage, interpreter_isolation_level, next_transaction_isolation_level] { - auto info = storage->GetInfo(); - std::vector> results{ - {TypedValue("name"), TypedValue(storage->id())}, - {TypedValue("vertex_count"), TypedValue(static_cast(info.vertex_count))}, - {TypedValue("edge_count"), TypedValue(static_cast(info.edge_count))}, - {TypedValue("average_degree"), TypedValue(info.average_degree)}, - {TypedValue("memory_usage"), TypedValue(static_cast(info.memory_usage))}, - {TypedValue("disk_usage"), TypedValue(static_cast(info.disk_usage))}, - {TypedValue("memory_allocated"), TypedValue(static_cast(utils::total_memory_tracker.Amount()))}, - {TypedValue("allocation_limit"), TypedValue(static_cast(utils::total_memory_tracker.HardLimit()))}, - {TypedValue("global_isolation_level"), TypedValue(IsolationLevelToString(storage->GetIsolationLevel()))}, - {TypedValue("session_isolation_level"), TypedValue(IsolationLevelToString(interpreter_isolation_level))}, - {TypedValue("next_session_isolation_level"), - TypedValue(IsolationLevelToString(next_transaction_isolation_level))}, - {TypedValue("storage_mode"), TypedValue(StorageModeToString(storage->GetStorageMode()))}}; - return std::pair{results, QueryHandlerResult::COMMIT}; - }; - break; - case InfoQuery::InfoType::INDEX: + case DatabaseInfoQuery::InfoType::INDEX: { header = {"index type", "label", "property"}; - handler = [storage] { + handler = [storage = current_db.db_acc_->get()->storage(), dba] { const std::string_view label_index_mark{"label"}; const std::string_view label_property_index_mark{"label+property"}; - auto info = storage->ListAllIndices(); + auto info = dba->ListAllIndices(); std::vector> results; results.reserve(info.label.size() + info.label_property.size()); for (const auto &item : info.label) { @@ -2934,13 +3013,14 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa return record_1[2].ValueString() < record_2[2].ValueString(); }); - return std::pair{results, QueryHandlerResult::NOTHING}; + return std::pair{results, QueryHandlerResult::COMMIT}; }; break; - case InfoQuery::InfoType::CONSTRAINT: + } + case DatabaseInfoQuery::InfoType::CONSTRAINT: { header = {"constraint type", "label", "properties"}; - handler = [storage] { - auto info = storage->ListAllConstraints(); + handler = [storage = current_db.db_acc_->get()->storage(), dba] { + auto info = dba->ListAllConstraints(); std::vector> results; results.reserve(info.existence.size() + info.unique.size()); for (const auto &item : info.existence) { @@ -2956,18 +3036,73 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa results.push_back( {TypedValue("unique"), TypedValue(storage->LabelToName(item.first)), TypedValue(std::move(properties))}); } - return std::pair{results, QueryHandlerResult::NOTHING}; + return std::pair{results, QueryHandlerResult::COMMIT}; }; break; + } + } - case InfoQuery::InfoType::BUILD: + return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), + [handler = std::move(handler), action = QueryHandlerResult::NOTHING, + pull_plan = std::shared_ptr(nullptr)]( + AnyStream *stream, std::optional n) mutable -> std::optional { + if (!pull_plan) { + auto [results, action_on_complete] = handler(); + action = action_on_complete; + pull_plan = std::make_shared(std::move(results)); + } + + if (pull_plan->Pull(stream, n)) { + return action; + } + return std::nullopt; + }, + RWType::NONE}; +} + +PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB ¤t_db, + std::optional interpreter_isolation_level, + std::optional next_transaction_isolation_level) { + if (in_explicit_transaction) { + throw InfoInMulticommandTxException(); + } + + auto *info_query = utils::Downcast(parsed_query.query); + std::vector header; + std::function>, QueryHandlerResult>()> handler; + + switch (info_query->info_type_) { + case SystemInfoQuery::InfoType::STORAGE: { + MG_ASSERT(current_db.db_acc_, "System storage info query expects a current DB"); + header = {"storage info", "value"}; + handler = [storage = current_db.db_acc_->get()->storage(), interpreter_isolation_level, + next_transaction_isolation_level] { + auto info = storage->GetInfo(); + std::vector> results{ + {TypedValue("name"), TypedValue(storage->id())}, + {TypedValue("vertex_count"), TypedValue(static_cast(info.vertex_count))}, + {TypedValue("edge_count"), TypedValue(static_cast(info.edge_count))}, + {TypedValue("average_degree"), TypedValue(info.average_degree)}, + {TypedValue("memory_usage"), TypedValue(static_cast(info.memory_usage))}, + {TypedValue("disk_usage"), TypedValue(static_cast(info.disk_usage))}, + {TypedValue("memory_allocated"), TypedValue(static_cast(utils::total_memory_tracker.Amount()))}, + {TypedValue("allocation_limit"), TypedValue(static_cast(utils::total_memory_tracker.HardLimit()))}, + {TypedValue("global_isolation_level"), TypedValue(IsolationLevelToString(storage->GetIsolationLevel()))}, + {TypedValue("session_isolation_level"), TypedValue(IsolationLevelToString(interpreter_isolation_level))}, + {TypedValue("next_session_isolation_level"), + TypedValue(IsolationLevelToString(next_transaction_isolation_level))}, + {TypedValue("storage_mode"), TypedValue(StorageModeToString(storage->GetStorageMode()))}}; + return std::pair{results, QueryHandlerResult::NOTHING}; + }; + } break; + case SystemInfoQuery::InfoType::BUILD: { header = {"build info", "value"}; handler = [] { std::vector> results{ {TypedValue("build_type"), TypedValue(utils::GetBuildInfo().build_name)}}; return std::pair{results, QueryHandlerResult::NOTHING}; }; - break; + } break; } return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges), @@ -2989,11 +3124,16 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa } PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction, - std::vector *notifications, storage::Storage *storage) { + std::vector *notifications, CurrentDB ¤t_db) { if (in_explicit_transaction) { throw ConstraintInMulticommandTxException(); } + MG_ASSERT(current_db.db_acc_, "Constraint query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + MG_ASSERT(current_db.db_transactional_accessor_, "Constraint query expects a DB transactional access"); + auto *dba = &*current_db.execution_db_accessor_; + auto *constraint_query = utils::Downcast(parsed_query.query); std::function handler; @@ -3022,10 +3162,10 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ } constraint_notification.title = fmt::format("Created EXISTS constraint on label {} on properties {}.", constraint_query->constraint_.label.name, properties_stringified); - handler = [storage, label, label_name = constraint_query->constraint_.label.name, + handler = [storage, dba, label, label_name = constraint_query->constraint_.label.name, properties_stringified = std::move(properties_stringified), properties = std::move(properties)](Notification &constraint_notification) { - auto maybe_constraint_error = storage->CreateExistenceConstraint(label, properties[0], {}); + auto maybe_constraint_error = dba->CreateExistenceConstraint(label, properties[0]); if (maybe_constraint_error.HasError()) { const auto &error = maybe_constraint_error.GetError(); @@ -3045,14 +3185,6 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ constraint_notification.title = fmt::format("Constraint EXISTS on label {} on properties {} already exists.", label_name, properties_stringified); - } else if constexpr (std::is_same_v) { - throw ReplicationException( - "At least one SYNC replica has not confirmed the creation of the EXISTS constraint on " - "label " - "{} on properties {}.", - label_name, properties_stringified); - } else if constexpr (std::is_same_v) { - throw ConstraintsPersistenceException(); } else { static_assert(kAlwaysFalse, "Missing type from variant visitor"); } @@ -3072,10 +3204,10 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ constraint_notification.title = fmt::format("Created UNIQUE constraint on label {} on properties {}.", constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [storage, label, label_name = constraint_query->constraint_.label.name, + handler = [storage, dba, label, label_name = constraint_query->constraint_.label.name, properties_stringified = std::move(properties_stringified), property_set = std::move(property_set)](Notification &constraint_notification) { - auto maybe_constraint_error = storage->CreateUniqueConstraint(label, property_set, {}); + auto maybe_constraint_error = dba->CreateUniqueConstraint(label, property_set); if (maybe_constraint_error.HasError()) { const auto &error = maybe_constraint_error.GetError(); std::visit( @@ -3097,13 +3229,6 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ constraint_notification.title = fmt::format("Constraint UNIQUE on label {} and properties {} couldn't be created.", label_name, properties_stringified); - } else if constexpr (std::is_same_v) { - throw ReplicationException( - fmt::format("At least one SYNC replica has not confirmed the creation of the UNIQUE " - "constraint: {}({}).", - label_name, properties_stringified)); - } else if constexpr (std::is_same_v) { - throw ConstraintsPersistenceException(); } else { static_assert(kAlwaysFalse, "Missing type from variant visitor"); } @@ -3146,32 +3271,14 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ constraint_notification.title = fmt::format("Dropped EXISTS constraint on label {} on properties {}.", constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [storage, label, label_name = constraint_query->constraint_.label.name, + handler = [dba, label, label_name = constraint_query->constraint_.label.name, properties_stringified = std::move(properties_stringified), properties = std::move(properties)](Notification &constraint_notification) { - auto maybe_constraint_error = storage->DropExistenceConstraint(label, properties[0], {}); + auto maybe_constraint_error = dba->DropExistenceConstraint(label, properties[0]); if (maybe_constraint_error.HasError()) { - const auto &error = maybe_constraint_error.GetError(); - std::visit( - [&label_name, &properties_stringified, &constraint_notification](T &&) { - using ErrorType = std::remove_cvref_t; - if constexpr (std::is_same_v) { - constraint_notification.code = NotificationCode::NONEXISTENT_CONSTRAINT; - constraint_notification.title = - fmt::format("Constraint EXISTS on label {} on properties {} doesn't exist.", label_name, - properties_stringified); - } else if constexpr (std::is_same_v) { - throw ReplicationException( - fmt::format("At least one SYNC replica has not confirmed the dropping of the EXISTS " - "constraint on label {} on properties {}.", - label_name, properties_stringified)); - } else if constexpr (std::is_same_v) { - throw ConstraintsPersistenceException(); - } else { - static_assert(kAlwaysFalse, "Missing type from variant visitor"); - } - }, - error); + constraint_notification.code = NotificationCode::NONEXISTENT_CONSTRAINT; + constraint_notification.title = fmt::format( + "Constraint EXISTS on label {} on properties {} doesn't exist.", label_name, properties_stringified); } return std::vector>(); }; @@ -3187,29 +3294,10 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ constraint_notification.title = fmt::format("Dropped UNIQUE constraint on label {} on properties {}.", constraint_query->constraint_.label.name, utils::Join(properties_string, ", ")); - handler = [storage, label, label_name = constraint_query->constraint_.label.name, + handler = [dba, label, label_name = constraint_query->constraint_.label.name, properties_stringified = std::move(properties_stringified), property_set = std::move(property_set)](Notification &constraint_notification) { - auto maybe_constraint_error = storage->DropUniqueConstraint(label, property_set, {}); - if (maybe_constraint_error.HasError()) { - const auto &error = maybe_constraint_error.GetError(); - std::visit( - [&label_name, &properties_stringified](T &&) { - using ErrorType = std::remove_cvref_t; - if constexpr (std::is_same_v) { - throw ReplicationException( - fmt::format("At least one SYNC replica has not confirmed the dropping of the UNIQUE " - "constraint on label {} on properties {}.", - label_name, properties_stringified)); - } else if constexpr (std::is_same_v) { - throw ConstraintsPersistenceException(); - } else { - static_assert(kAlwaysFalse, "Missing type from variant visitor"); - } - }, - error); - } - const auto &res = maybe_constraint_error.GetValue(); + auto res = dba->DropUniqueConstraint(label, property_set); switch (res) { case storage::UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES: throw SyntaxException( @@ -3248,23 +3336,18 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_ RWType::NONE}; } -PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, bool in_explicit_transaction, bool in_explicit_db, +PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB ¤t_db, InterpreterContext *interpreter_context, - memgraph::query::Interpreter &interpreter, std::optional> on_change_cb) { #ifdef MG_ENTERPRISE if (!license::global_license_checker.IsEnterpriseValidFast()) { throw QueryException("Trying to use enterprise feature without a valid license."); } // TODO: Remove once replicas support multi-tenant replication - if (!interpreter.db_acc_) - throw DatabaseContextRequiredException("Multi database queries require a defined database."); - if (GetReplicaRole(interpreter.db_acc_->get()->storage()) == storage::replication::ReplicationRole::REPLICA) { + if (!current_db.db_acc_) throw DatabaseContextRequiredException("Multi database queries require a defined database."); + if (IsReplica(current_db.db_acc_->get()->storage())) { throw QueryException("Query forbidden on the replica!"); } - if (in_explicit_transaction) { - throw MultiDatabaseQueryInMulticommandTxException(); - } auto *query = utils::Downcast(parsed_query.query); auto *db_handler = interpreter_context->db_handler; @@ -3310,23 +3393,23 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, bool in_explic }; case MultiDatabaseQuery::Action::USE: - if (in_explicit_db) { + if (current_db.in_explicit_db_) { throw QueryException("Database switching is prohibited if session explicitly defines the used database"); } return PreparedQuery{{"STATUS"}, std::move(parsed_query.required_privileges), - [db_name = query->db_name_, db_handler, &interpreter, on_change_cb]( + [db_name = query->db_name_, db_handler, ¤t_db, on_change_cb]( AnyStream *stream, std::optional n) -> std::optional { std::vector> status; std::string res; try { - if (interpreter.db_acc_ && db_name == interpreter.db_acc_->get()->id()) { + if (current_db.db_acc_ && db_name == current_db.db_acc_->get()->id()) { res = "Already using " + db_name; } else { auto tmp = db_handler->Get(db_name); if (on_change_cb) (*on_change_cb)(db_name); // Will trow if cb fails - interpreter.SetCurrentDB(std::move(tmp)); + current_db.SetCurrentDB(std::move(tmp), false); res = "Using " + db_name; } } catch (const utils::BasicException &e) { @@ -3392,15 +3475,21 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, bool in_explic #endif } -PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, storage::Storage *storage, +PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, CurrentDB ¤t_db, InterpreterContext *interpreter_context, const std::optional &username) { #ifdef MG_ENTERPRISE + + // TODO: split query into two, Databases (no need for current_db), & Current database (uses current_db) + MG_ASSERT(current_db.db_acc_, "Show Database Level query expects a current DB"); + storage::Storage *storage = current_db.db_acc_->get()->storage(); + if (!license::global_license_checker.IsEnterpriseValidFast()) { throw QueryException("Trying to use enterprise feature without a valid license."); } // TODO: Remove once replicas support multi-tenant replication - if (GetReplicaRole(storage) == storage::replication::ReplicationRole::REPLICA) { + auto &replState = storage->repl_state_; + if (replState.IsReplica()) { throw QueryException("SHOW DATABASES forbidden on the replica!"); } @@ -3482,10 +3571,7 @@ PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, storage::Stora #endif } -std::optional Interpreter::GetTransactionId() const { - if (db_accessor_) return db_accessor_->GetTransactionId(); - return {}; -} +std::optional Interpreter::GetTransactionId() const { return current_transaction_; } void Interpreter::BeginTransaction(QueryExtras const &extras) { const auto prepared_query = PrepareTransactionQuery("BEGIN", extras); @@ -3509,59 +3595,37 @@ void Interpreter::RollbackTransaction() { #if MG_ENTERPRISE // Before Prepare or during Prepare, but single-threaded. // TODO: Is there any cleanup? -void Interpreter::SetCurrentDB(std::string_view db_name) { +void Interpreter::SetCurrentDB(std::string_view db_name, bool in_explicit_db) { // Can throw // do we lock here? - db_acc_ = interpreter_context_->db_handler->Get(db_name); -} -void Interpreter::SetCurrentDB(memgraph::dbms::DatabaseAccess new_db) { - // do we lock here? - db_acc_ = std::move(new_db); + current_db_.SetCurrentDB(interpreter_context_->db_handler->Get(db_name), in_explicit_db); } #endif Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, const std::map ¶ms, - const std::string *username, QueryExtras const &extras, - const std::string &session_uuid) { - std::shared_ptr current_timer; - + QueryExtras const &extras) { // TODO: Remove once the interpreter is storage/tx independent and could run without an associated database - if (!db_acc_) throw DatabaseContextRequiredException("Database required for the query."); - auto *db = db_acc_->get(); - - if (!in_explicit_transaction_) { - query_executions_.clear(); - transaction_queries_->clear(); - // Handle user-defined metadata in auto-transactions - metadata_ = GenOptional(extras.metadata_pv); - auto const timeout = DetermineTxTimeout(extras.tx_timeout, interpreter_context_->config); - current_timer = timeout ? std::make_shared(timeout.ValueUnsafe().count()) : nullptr; - } else { - current_timer = explicit_transaction_timer_; + if (!current_db_.db_acc_) { + throw DatabaseContextRequiredException("Database required for the query."); } - // This will be done in the handle transaction query. Our handler can save username and then send it to the kill and - // show transactions. - std::optional user = StringPointerToOptional(username); - username_ = user; - // Handle transaction control queries. - const auto upper_case_query = utils::ToUpperCase(query_string); const auto trimmed_query = utils::Trim(upper_case_query); - if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") { - query_executions_.emplace_back( - std::make_unique(utils::MonotonicBufferResource(kExecutionMemoryBlockSize))); - auto &query_execution = query_executions_.back(); + auto resource = utils::MonotonicBufferResource(kExecutionMemoryBlockSize); + auto prepared_query = PrepareTransactionQuery(trimmed_query, extras); + auto &query_execution = + query_executions_.emplace_back(QueryExecution::Create(std::move(resource), std::move(prepared_query))); std::optional qid = in_explicit_transaction_ ? static_cast(query_executions_.size() - 1) : std::optional{}; - - query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query, extras)); return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid, {}}; } + if (!in_explicit_transaction_) { + transaction_queries_->clear(); + } // Don't save BEGIN, COMMIT or ROLLBACK transaction_queries_->push_back(query_string); @@ -3569,79 +3633,88 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, // an explicit transaction block. if (in_explicit_transaction_) { AdvanceCommand(); - } else if (db_accessor_) { - // If we're not in an explicit transaction block and we have an open - // transaction, abort it since we're about to prepare a new query. - query_executions_.emplace_back( - std::make_unique(utils::MonotonicBufferResource(kExecutionMemoryBlockSize))); - AbortCommand(&query_executions_.back()); + } else { + query_executions_.clear(); + if (current_db_.db_transactional_accessor_ /* && !in_explicit_transaction_*/) { + // If we're not in an explicit transaction block and we have an open + // transaction, abort it since we're about to prepare a new query. + AbortCommand(nullptr); + } + + SetupInterpreterTransaction(extras); } std::unique_ptr *query_execution_ptr = nullptr; try { - query_executions_.emplace_back( - std::make_unique(utils::MonotonicBufferResource(kExecutionMemoryBlockSize))); - query_execution_ptr = &query_executions_.back(); utils::Timer parsing_timer; ParsedQuery parsed_query = ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query); - TypedValue parsing_time{parsing_timer.Elapsed().count()}; + auto parsing_time = parsing_timer.Elapsed().count(); - if ((utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query))) { - CypherQuery *cypher_query = nullptr; - if (utils::Downcast(parsed_query.query)) { - cypher_query = utils::Downcast(parsed_query.query); - } else { - auto *profile_query = utils::Downcast(parsed_query.query); - cypher_query = profile_query->cypher_query_; + CypherQuery const *const cypher_query = [&]() -> CypherQuery * { + if (auto *cypher_query = utils::Downcast(parsed_query.query)) { + return cypher_query; } - if (const auto &clauses = cypher_query->single_query_->clauses_; - IsAllShortestPathsQuery(clauses) || IsCallBatchedProcedureQuery(clauses) || - std::any_of(clauses.begin(), clauses.end(), - [](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) { - // Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage. - // QueryExecution MemoryResource is mostly used for allocations done on Frame and storing `row`s - query_executions_[query_executions_.size() - 1] = std::make_unique(utils::PoolResource( - 128, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource())); - query_execution_ptr = &query_executions_.back(); - spdlog::trace("PrepareCypher has {} encountered all shortest paths, QueryExection will use PoolResource", - IsAllShortestPathsQuery(clauses) ? "" : "not"); + if (auto *profile_query = utils::Downcast(parsed_query.query)) { + return profile_query->cypher_query_; } + return nullptr; + }(); // IILE + + auto const [usePool, hasAllShortestPaths] = [&]() -> std::pair { + if (!cypher_query) { + return {false, false}; + } + auto const &clauses = cypher_query->single_query_->clauses_; + bool hasAllShortestPaths = IsAllShortestPathsQuery(clauses); + // Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage. + bool usePool = hasAllShortestPaths || IsCallBatchedProcedureQuery(clauses) || IsLoadCsvQuery(clauses); + return {usePool, hasAllShortestPaths}; + }(); // IILE + + // Setup QueryExecution + // its MemoryResource is mostly used for allocations done on Frame and storing `row`s + if (usePool) { + spdlog::trace("PrepareCypher has {} encountered all shortest paths, QueryExecution will use PoolResource", + hasAllShortestPaths ? "" : "not"); + query_executions_.emplace_back(QueryExecution::Create(utils::PoolResource(128, kExecutionPoolMaxBlockSize))); + } else { + query_executions_.emplace_back(QueryExecution::Create(utils::MonotonicBufferResource(kExecutionMemoryBlockSize))); } auto &query_execution = query_executions_.back(); + query_execution_ptr = &query_execution; std::optional qid = in_explicit_transaction_ ? static_cast(query_executions_.size() - 1) : std::optional{}; - query_execution->summary["parsing_time"] = std::move(parsing_time); + query_execution->summary["parsing_time"] = parsing_time; // Set a default cost estimate of 0. Individual queries can overwrite this // field with an improved estimate. query_execution->summary["cost_estimate"] = 0.0; // Some queries require an active transaction in order to be prepared. - if (!in_explicit_transaction_ && - (utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || - utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || - utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || - utils::Downcast(parsed_query.query))) { - memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTransactions); - auto &db_acc = *db_acc_; - db_accessor_ = db_acc->Access(GetIsolationLevelOverride()); - execution_db_accessor_.emplace(db_accessor_.get()); - transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release); + // TODO: make a better analysis visitor over the `parsed_query.query` + bool requires_db_transaction = + utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query) || utils::Downcast(parsed_query.query) || + utils::Downcast(parsed_query.query); - if (utils::Downcast(parsed_query.query) && db_acc->trigger_store()->HasTriggers()) { - trigger_context_collector_.emplace(db_acc->trigger_store()->GetEventTypes()); - } + if (!in_explicit_transaction_ && requires_db_transaction) { + // TODO: ATM only a single database, will change when we have multiple database transactions + bool could_commit = utils::Downcast(parsed_query.query) != nullptr; + bool unique = utils::Downcast(parsed_query.query) != nullptr || + utils::Downcast(parsed_query.query) != nullptr; + SetupDatabaseTransaction(could_commit, unique); } - const auto is_cacheable = parsed_query.is_cacheable; - auto *plan_cache = db->plan_cache(); - auto get_plan_cache = [&]() { - return is_cacheable ? plan_cache : nullptr; - }; // Some queries run additional parsing and may need the plan_cache even if the outer query is not cacheable + // TODO: none database transaction (assuming mutually exclusive from DB transactions) + // if (!requires_db_transaction) { + // /* something */ + // } utils::Timer planning_timer; PreparedQuery prepared_query; @@ -3652,76 +3725,89 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, frame_change_collector_.emplace(memory_resource); if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, - &*execution_db_accessor_, memory_resource, &query_execution->notifications, - username, &transaction_status_, std::move(current_timer), get_plan_cache(), - trigger_context_collector_ ? &*trigger_context_collector_ : nullptr, - &*frame_change_collector_); + current_db_, memory_resource, &query_execution->notifications, username_, + &transaction_status_, current_timeout_timer_, &*frame_change_collector_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, - &*execution_db_accessor_, plan_cache); + prepared_query = + PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_, current_db_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareProfileQuery( - std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_, - &*execution_db_accessor_, &query_execution->execution_memory_with_exception, username, &transaction_status_, - std::move(current_timer), plan_cache, &*frame_change_collector_); + prepared_query = + PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, + interpreter_context_, current_db_, &query_execution->execution_memory_with_exception, + username_, &transaction_status_, current_timeout_timer_, &*frame_change_collector_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_, - memory_resource); + prepared_query = PrepareDumpQuery(std::move(parsed_query), current_db_); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_, - &query_execution->notifications, db->storage(), get_plan_cache()); + &query_execution->notifications, current_db_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareAnalyzeGraphQuery(std::move(parsed_query), in_explicit_transaction_, - &*execution_db_accessor_, get_plan_cache()); + prepared_query = PrepareAnalyzeGraphQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { + /// SYSTEM (Replication) PURE prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_); - } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, - db->storage(), &query_execution->execution_memory_with_exception, - interpreter_isolation_level, next_transaction_isolation_level); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareDatabaseInfoQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareSystemInfoQuery(std::move(parsed_query), in_explicit_transaction_, current_db_, + interpreter_isolation_level, next_transaction_isolation_level); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareConstraintQuery(std::move(parsed_query), in_explicit_transaction_, - &query_execution->notifications, db->storage()); + &query_execution->notifications, current_db_); } else if (utils::Downcast(parsed_query.query)) { + /// TODO: make replication DB agnostic prepared_query = PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, - db->storage(), interpreter_context_->config); + current_db_, interpreter_context_->config); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, db->storage()); + prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareFreeMemoryQuery(std::move(parsed_query), in_explicit_transaction_, db->storage()); + prepared_query = PrepareFreeMemoryQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { + /// SYSTEM PURE prepared_query = PrepareShowConfigQuery(std::move(parsed_query), in_explicit_transaction_); } else if (utils::Downcast(parsed_query.query)) { prepared_query = PrepareTriggerQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, - db->trigger_store(), interpreter_context_, &*execution_db_accessor_, params, username); + current_db_, interpreter_context_, params, username_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareStreamQuery(std::move(parsed_query), in_explicit_transaction_, - &query_execution->notifications, *db_acc_, interpreter_context_, username); - } else if (utils::Downcast(parsed_query.query)) { prepared_query = - PrepareIsolationLevelQuery(std::move(parsed_query), in_explicit_transaction_, db->storage(), this); + PrepareStreamQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications, + current_db_, interpreter_context_, username_); + } else if (utils::Downcast(parsed_query.query)) { + prepared_query = PrepareIsolationLevelQuery(std::move(parsed_query), in_explicit_transaction_, current_db_, this); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, db->storage()); + prepared_query = PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, current_db_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_); + /// SYSTEM PURE + prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_); } else if (utils::Downcast(parsed_query.query)) { + /// SYSTEM PURE prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_); } else if (utils::Downcast(parsed_query.query)) { prepared_query = - PrepareStorageModeQuery(std::move(parsed_query), in_explicit_transaction_, *db_acc_, interpreter_context_); + PrepareStorageModeQuery(std::move(parsed_query), in_explicit_transaction_, current_db_, interpreter_context_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareTransactionQueueQuery(std::move(parsed_query), username_, in_explicit_transaction_, - interpreter_context_, *this); + /// INTERPRETER + if (in_explicit_transaction_) { + throw TransactionQueueInMulticommandTxException(); + } + prepared_query = PrepareTransactionQueueQuery(std::move(parsed_query), username_, interpreter_context_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareMultiDatabaseQuery(std::move(parsed_query), in_explicit_transaction_, in_explicit_db_, - interpreter_context_, *this, on_change_); - } else if (utils::Downcast(parsed_query.query)) { + if (in_explicit_transaction_) { + throw MultiDatabaseQueryInMulticommandTxException(); + } + /// SYSTEM (Replication) + INTERPRETER prepared_query = - PrepareShowDatabasesQuery(std::move(parsed_query), db->storage(), interpreter_context_, username_); + PrepareMultiDatabaseQuery(std::move(parsed_query), current_db_, interpreter_context_, on_change_); + } else if (utils::Downcast(parsed_query.query)) { + /// SYSTEM PURE ("SHOW DATABASES") + /// INTERPRETER (TODO: "SHOW DATABASE") + prepared_query = PrepareShowDatabasesQuery(std::move(parsed_query), current_db_, interpreter_context_, username_); } else if (utils::Downcast(parsed_query.query)) { - prepared_query = PrepareEdgeImportModeQuery(std::move(parsed_query), in_explicit_transaction_, db->storage()); + if (in_explicit_transaction_) { + throw EdgeImportModeModificationInMulticommandTxException(); + } + prepared_query = PrepareEdgeImportModeQuery(std::move(parsed_query), current_db_); } else { LOG_FATAL("Should not get here -- unknown query type!"); } @@ -3734,14 +3820,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, UpdateTypeCount(rw_type); - if (IsWriteQueryOnMainMemoryReplica(db->storage(), rw_type)) { + if (IsWriteQueryOnMainMemoryReplica(current_db_.db_acc_->get()->storage(), rw_type)) { query_execution = nullptr; throw QueryException("Write query forbidden on the replica!"); } // Set the target db to the current db (some queries have different target from the current db) if (!query_execution->prepared_query->db) { - query_execution->prepared_query->db = db->id(); + query_execution->prepared_query->db = current_db_.db_acc_->get()->id(); } query_execution->summary["db"] = *query_execution->prepared_query->db; @@ -3753,6 +3839,16 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, throw; } } +void Interpreter::SetupDatabaseTransaction(bool couldCommit, bool unique) { + current_db_.SetupDatabaseTransaction(GetIsolationLevelOverride(), couldCommit, unique); +} +void Interpreter::SetupInterpreterTransaction(const QueryExtras &extras) { + metrics::IncrementCounter(metrics::ActiveTransactions); + transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release); + current_transaction_ = interpreter_context_->id_handler.next(); + metadata_ = GenOptional(extras.metadata_pv); + current_timeout_timer_ = CreateTimeoutTimer(extras, interpreter_context_->config); +} std::vector Interpreter::GetQueries() { auto typed_queries = std::vector(); @@ -3780,19 +3876,16 @@ void Interpreter::Abort() { expect_rollback_ = false; in_explicit_transaction_ = false; metadata_ = std::nullopt; - explicit_transaction_timer_.reset(); + current_timeout_timer_.reset(); + current_transaction_.reset(); memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTransactions); - if (!db_accessor_) return; - - db_accessor_->Abort(); + // if (!current_db_.db_transactional_accessor_) return; + current_db_.CleanupDBTransaction(true); for (auto &qe : query_executions_) { if (qe) qe->CleanRuntimeData(); } - execution_db_accessor_.reset(); - db_accessor_.reset(); - trigger_context_collector_.reset(); frame_change_collector_.reset(); } @@ -3854,6 +3947,8 @@ void RunTriggersAfterCommit(dbms::DatabaseAccess db_acc, InterpreterContext *int } } else if constexpr (std::is_same_v) { throw QueryException("Unable to commit due to serialization error."); + } else if constexpr (std::is_same_v) { + throw QueryException("Unable to commit due to persistance error."); } else { static_assert(kAlwaysFalse, "Missing type from variant visitor"); } @@ -3870,11 +3965,12 @@ void Interpreter::Commit() { // For now, we will not check if there are some unfinished queries. // We should document clearly that all results should be pulled to complete // a query. - if (!db_accessor_) return; + current_transaction_.reset(); + if (!current_db_.db_transactional_accessor_) return; // TODO: Better (or removed) check - if (!db_acc_) return; - auto *db = db_acc_->get(); + if (!current_db_.db_acc_) return; + auto *db = current_db_.db_acc_->get(); /* At this point we must check that the transaction is alive to start committing. The only other possible state is @@ -3896,7 +3992,7 @@ void Interpreter::Commit() { [this]() { transaction_status_.store(TransactionStatus::IDLE, std::memory_order_release); }); auto current_storage_mode = db->GetStorageMode(); - auto creation_mode = db_accessor_->GetCreationStorageMode(); + auto creation_mode = current_db_.db_transactional_accessor_->GetCreationStorageMode(); if (creation_mode != storage::StorageMode::ON_DISK_TRANSACTIONAL && current_storage_mode == storage::StorageMode::ON_DISK_TRANSACTIONAL) { throw QueryException( @@ -3909,9 +4005,9 @@ void Interpreter::Commit() { }); std::optional trigger_context = std::nullopt; - if (trigger_context_collector_) { - trigger_context.emplace(std::move(*trigger_context_collector_).TransformToTriggerContext()); - trigger_context_collector_.reset(); + if (current_db_.trigger_context_collector_) { + trigger_context.emplace(std::move(*current_db_.trigger_context_collector_).TransformToTriggerContext()); + current_db_.trigger_context_collector_.reset(); } if (frame_change_collector_) { @@ -3924,9 +4020,9 @@ void Interpreter::Commit() { utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize}; AdvanceCommand(); try { - trigger.Execute(&*execution_db_accessor_, &execution_memory, flags::run_time::execution_timeout_sec_, - &interpreter_context_->is_shutting_down, &transaction_status_, *trigger_context, - interpreter_context_->auth_checker); + trigger.Execute(&*current_db_.execution_db_accessor_, &execution_memory, + flags::run_time::execution_timeout_sec_, &interpreter_context_->is_shutting_down, + &transaction_status_, *trigger_context, interpreter_context_->auth_checker); } catch (const utils::BasicException &e) { throw utils::BasicException( fmt::format("Trigger '{}' caused the transaction to fail.\nException: {}", trigger.Name(), e.what())); @@ -3939,20 +4035,18 @@ void Interpreter::Commit() { for (auto &qe : query_executions_) { if (qe) qe->CleanRuntimeData(); } - execution_db_accessor_.reset(); - db_accessor_.reset(); - trigger_context_collector_.reset(); + current_db_.CleanupDBTransaction(false); }; utils::OnScopeExit members_reseter(reset_necessary_members); auto commit_confirmed_by_all_sync_repplicas = true; - auto maybe_commit_error = db_accessor_->Commit(); + auto maybe_commit_error = current_db_.db_transactional_accessor_->Commit(); if (maybe_commit_error.HasError()) { const auto &error = maybe_commit_error.GetError(); std::visit( - [&execution_db_accessor = execution_db_accessor_, + [&execution_db_accessor = current_db_.execution_db_accessor_, &commit_confirmed_by_all_sync_repplicas](T &&arg) { using ErrorType = std::remove_cvref_t; if constexpr (std::is_same_v) { @@ -3979,6 +4073,8 @@ void Interpreter::Commit() { } } else if constexpr (std::is_same_v) { throw QueryException("Unable to commit due to serialization error."); + } else if constexpr (std::is_same_v) { + throw QueryException("Unable to commit due to persistance error."); } else { static_assert(kAlwaysFalse, "Missing type from variant visitor"); } @@ -3993,9 +4089,10 @@ void Interpreter::Commit() { // ordered execution of after commit triggers are not guaranteed. if (trigger_context && db->trigger_store()->AfterCommitTriggers().size() > 0) { db->AddTask([this, trigger_context = std::move(*trigger_context), - user_transaction = std::shared_ptr(std::move(db_accessor_))]() mutable { + user_transaction = std::shared_ptr(std::move(current_db_.db_transactional_accessor_))]() mutable { // TODO: Should this take the db_ and not Access()? - RunTriggersAfterCommit(*db_acc_, interpreter_context_, std::move(trigger_context), &this->transaction_status_); + RunTriggersAfterCommit(*current_db_.db_acc_, interpreter_context_, std::move(trigger_context), + &this->transaction_status_); user_transaction->FinalizeTransaction(); SPDLOG_DEBUG("Finished executing after commit triggers"); // NOLINT(bugprone-lambda-function-name) }); @@ -4008,8 +4105,8 @@ void Interpreter::Commit() { } void Interpreter::AdvanceCommand() { - if (!db_accessor_) return; - db_accessor_->AdvanceCommand(); + if (!current_db_.db_transactional_accessor_) return; + current_db_.db_transactional_accessor_->AdvanceCommand(); } void Interpreter::AbortCommand(std::unique_ptr *query_execution) { @@ -4040,5 +4137,7 @@ void Interpreter::SetNextTransactionIsolationLevel(const storage::IsolationLevel void Interpreter::SetSessionIsolationLevel(const storage::IsolationLevel isolation_level) { interpreter_isolation_level.emplace(isolation_level); } +void Interpreter::ResetUser() { username_.reset(); } +void Interpreter::SetUser(std::string_view username) { username_ = username; } } // namespace memgraph::query diff --git a/src/query/interpreter.hpp b/src/query/interpreter.hpp index ccb13373b..515419070 100644 --- a/src/query/interpreter.hpp +++ b/src/query/interpreter.hpp @@ -95,7 +95,7 @@ class ReplicationQueryHandler { const std::chrono::seconds replica_check_frequency) = 0; /// @throw QueryRuntimeException if an error ocurred. - virtual void DropReplica(const std::string &replica_name) = 0; + virtual void DropReplica(std::string_view replica_name) = 0; /// @throw QueryRuntimeException if an error ocurred. virtual std::vector ShowReplicas() const = 0; @@ -139,6 +139,34 @@ struct QueryExtras { std::optional tx_timeout; }; +struct CurrentDB { + CurrentDB() = default; // TODO: remove, we should always have an implicit default obtainable from somewhere + // ATM: it is provided by the DatabaseAccess + // future: should be a name + ptr to dbms_handler, lazy fetch when needed + explicit CurrentDB(memgraph::dbms::DatabaseAccess db_acc) : db_acc_{std::move(db_acc)} {} + + CurrentDB(CurrentDB const &) = delete; + CurrentDB &operator=(CurrentDB const &) = delete; + + void SetupDatabaseTransaction(std::optional override_isolation_level, bool could_commit, + bool unique = false); + void CleanupDBTransaction(bool abort); + void SetCurrentDB(memgraph::dbms::DatabaseAccess new_db, bool in_explicit_db) { + // do we lock here? + db_acc_ = std::move(new_db); + in_explicit_db_ = in_explicit_db; + } + + // TODO: don't provide explicitly via constructor, instead have a lazy way of getting the current/default + // DatabaseAccess + // hence, explict bolt "use DB" in metadata wouldn't necessarily get access unless query required it. + std::optional db_acc_; // Current db (TODO: expand to support multiple) + std::unique_ptr db_transactional_accessor_; + std::optional execution_db_accessor_; + std::optional trigger_context_collector_; + bool in_explicit_db_{false}; +}; + class Interpreter final { public: Interpreter(InterpreterContext *interpreter_context); @@ -158,16 +186,14 @@ class Interpreter final { std::optional username_; bool in_explicit_transaction_{false}; - bool in_explicit_db_{false}; + CurrentDB current_db_; + bool expect_rollback_{false}; - std::shared_ptr explicit_transaction_timer_{}; + std::shared_ptr current_timeout_timer_{}; std::optional> metadata_{}; //!< User defined transaction metadata - std::optional db_acc_; // Current db (TODO: expand to support multiple) - #ifdef MG_ENTERPRISE - void SetCurrentDB(std::string_view db_name); - void SetCurrentDB(memgraph::dbms::DatabaseAccess new_db); + void SetCurrentDB(std::string_view db_name, bool explicit_db); void OnChangeCB(auto cb) { on_change_.emplace(cb); } #endif @@ -179,9 +205,9 @@ class Interpreter final { * * @throw query::QueryException */ - PrepareResult Prepare(const std::string &query, const std::map ¶ms, - const std::string *username, QueryExtras const &extras = {}, - const std::string &session_uuid = {}); + Interpreter::PrepareResult Prepare(const std::string &query, + const std::map ¶ms, + QueryExtras const &extras); /** * Execute the last prepared query and stream *all* of the results into the @@ -243,27 +269,30 @@ class Interpreter final { */ void Abort(); - std::atomic transaction_status_{TransactionStatus::IDLE}; + std::atomic transaction_status_{TransactionStatus::IDLE}; // Tie to current_transaction_ + std::optional current_transaction_; + + void ResetUser(); + + void SetUser(std::string_view username); private: struct QueryExecution { - std::optional prepared_query; std::variant execution_memory; utils::ResourceWithOutOfMemoryException execution_memory_with_exception; + std::optional prepared_query; std::map summary; std::vector notifications; - explicit QueryExecution(utils::MonotonicBufferResource monotonic_memory) - : execution_memory(std::move(monotonic_memory)) { - std::visit( - [&](auto &memory_resource) { - execution_memory_with_exception = utils::ResourceWithOutOfMemoryException(&memory_resource); - }, - execution_memory); - }; + static auto Create(std::variant memory_resource, + std::optional prepared_query = std::nullopt) -> std::unique_ptr { + return std::make_unique(std::move(memory_resource), std::move(prepared_query)); + } - explicit QueryExecution(utils::PoolResource pool_resource) : execution_memory(std::move(pool_resource)) { + explicit QueryExecution(std::variant memory_resource, + std::optional prepared_query) + : execution_memory(std::move(memory_resource)), prepared_query{std::move(prepared_query)} { std::visit( [&](auto &memory_resource) { execution_memory_with_exception = utils::ResourceWithOutOfMemoryException(&memory_resource); @@ -311,12 +340,6 @@ class Interpreter final { InterpreterContext *interpreter_context_; - // This cannot be std::optional because we need to move this accessor later on into a lambda capture - // which is assigned to std::function. std::function requires every object to be copyable, so we - // move this unique_ptr into a shrared_ptr. - std::unique_ptr db_accessor_; - std::optional execution_db_accessor_; - std::optional trigger_context_collector_; std::optional frame_change_collector_; std::optional interpreter_isolation_level; @@ -334,22 +357,8 @@ class Interpreter final { } std::optional> on_change_{}; -}; - -class TransactionQueueQueryHandler { - public: - TransactionQueueQueryHandler() = default; - virtual ~TransactionQueueQueryHandler() = default; - - TransactionQueueQueryHandler(const TransactionQueueQueryHandler &) = default; - TransactionQueueQueryHandler &operator=(const TransactionQueueQueryHandler &) = default; - - TransactionQueueQueryHandler(TransactionQueueQueryHandler &&) = default; - TransactionQueueQueryHandler &operator=(TransactionQueueQueryHandler &&) = default; - - static std::vector> ShowTransactions( - const std::unordered_set &interpreters, const std::optional &username, - bool hasTransactionManagementPrivilege, std::optional &filter_db_acc); + void SetupInterpreterTransaction(const QueryExtras &extras); + void SetupDatabaseTransaction(bool couldCommit, bool unique = false); }; template @@ -411,7 +420,7 @@ std::map Interpreter::Pull(TStream *result_stream, std: // The only cases in which we have nothing to do are those where // we're either in an explicit transaction or the query is such that // a transaction wasn't started on a call to `Prepare()`. - MG_ASSERT(in_explicit_transaction_ || !db_accessor_); + MG_ASSERT(in_explicit_transaction_ || !current_db_.db_transactional_accessor_); break; } // As the transaction is done we can clear all the executions diff --git a/src/query/interpreter_context.cpp b/src/query/interpreter_context.cpp index eeedcf591..c65b43505 100644 --- a/src/query/interpreter_context.cpp +++ b/src/query/interpreter_context.cpp @@ -13,15 +13,15 @@ #include "query/interpreter.hpp" namespace memgraph::query { -std::vector> InterpreterContext::KillTransactions( +std::vector> InterpreterContext::TerminateTransactions( std::vector maybe_kill_transaction_ids, const std::optional &username, - bool hasTransactionManagementPrivilege, Interpreter &calling_interpreter) { + std::function privilege_checker) { auto not_found_midpoint = maybe_kill_transaction_ids.end(); // Multiple simultaneous TERMINATE TRANSACTIONS aren't allowed // TERMINATE and SHOW TRANSACTIONS are mutually exclusive - interpreters.WithLock([¬_found_midpoint, &maybe_kill_transaction_ids, username, hasTransactionManagementPrivilege, - filter_db_acc = &calling_interpreter.db_acc_](const auto &interpreters) { + interpreters.WithLock([¬_found_midpoint, &maybe_kill_transaction_ids, username, + privilege_checker = std::move(privilege_checker)](const auto &interpreters) { for (Interpreter *interpreter : interpreters) { TransactionStatus alive_status = TransactionStatus::ACTIVE; // if it is just checking kill, commit and abort should wait for the end of the check @@ -38,7 +38,6 @@ std::vector> InterpreterContext::KillTransactions( interpreter->transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release); } }); - if (interpreter->db_acc_ != *filter_db_acc) continue; std::optional intr_trans = interpreter->GetTransactionId(); if (!intr_trans.has_value()) continue; @@ -49,7 +48,11 @@ std::vector> InterpreterContext::KillTransactions( // update the maybe_kill_transaction_ids (partitioning not found + killed) --not_found_midpoint; std::iter_swap(it, not_found_midpoint); - if (interpreter->username_ == username || hasTransactionManagementPrivilege) { + auto get_interpreter_db_name = [&]() -> std::string const & { + static std::string all; + return interpreter->current_db_.db_acc_ ? interpreter->current_db_.db_acc_->get()->id() : all; + }; + if (interpreter->username_ == username || privilege_checker(get_interpreter_db_name())) { killed = true; // Note: this is used by the above `clean_status` (OnScopeExit) spdlog::warn("Transaction {} successfully killed", transaction_id); } else { diff --git a/src/query/interpreter_context.hpp b/src/query/interpreter_context.hpp index 010c85220..829fe9c31 100644 --- a/src/query/interpreter_context.hpp +++ b/src/query/interpreter_context.hpp @@ -11,6 +11,8 @@ #pragma once +#include +#include #include #include #include @@ -34,6 +36,8 @@ class Database; namespace memgraph::query { +constexpr uint64_t kInterpreterTransactionInitialId = 1ULL << 63U; + class AuthQueryHandler; class AuthChecker; class Interpreter; @@ -72,14 +76,20 @@ struct InterpreterContext { // TODO: Have a way to read the current database memgraph::utils::Synchronized, memgraph::utils::SpinLock> interpreters; + struct { + auto next() -> uint64_t { return transaction_id++; } + + private: + std::atomic transaction_id = kInterpreterTransactionInitialId; + } id_handler; + /// Function that is used to tell all active interpreters that they should stop /// their ongoing execution. void Shutdown() { is_shutting_down.store(true, std::memory_order_release); } - std::vector> KillTransactions(std::vector maybe_kill_transaction_ids, - const std::optional &username, - bool hasTransactionManagementPrivilege, - Interpreter &calling_interpreter); + std::vector> TerminateTransactions( + std::vector maybe_kill_transaction_ids, const std::optional &username, + std::function privilege_checker); }; } // namespace memgraph::query diff --git a/src/query/plan/read_write_type_checker.cpp b/src/query/plan/read_write_type_checker.cpp index 851465c56..fa11ced9a 100644 --- a/src/query/plan/read_write_type_checker.cpp +++ b/src/query/plan/read_write_type_checker.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -21,7 +21,7 @@ namespace memgraph::query::plan { PRE_VISIT(CreateNode, RWType::W, true) -PRE_VISIT(CreateExpand, RWType::R, true) +PRE_VISIT(CreateExpand, RWType::R, true) // ?? RWType::RW PRE_VISIT(Delete, RWType::W, true) PRE_VISIT(SetProperty, RWType::W, true) diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index 055d47d2c..2ecf54ee2 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -488,6 +488,10 @@ class RuleBasedPlanner { last_op = std::make_unique(std::move(last_op), node1_symbol, view); new_symbols.emplace_back(node1_symbol); + last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table); + last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths); + last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table); + } else if (named_paths.size() == 1U) { last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table); last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths); last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table); diff --git a/src/query/procedure/mg_procedure_impl.cpp b/src/query/procedure/mg_procedure_impl.cpp index 0294b6f73..eddba6d43 100644 --- a/src/query/procedure/mg_procedure_impl.cpp +++ b/src/query/procedure/mg_procedure_impl.cpp @@ -1940,7 +1940,7 @@ void mgp_vertex_destroy(mgp_vertex *v) { DeleteRawMgpObject(v); } mgp_error mgp_vertex_equal(mgp_vertex *v1, mgp_vertex *v2, int *result) { // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) - static_assert(noexcept(*result = *v1 == *v2 ? 1 : 0)); + static_assert(noexcept(*v1 == *v2)); *result = *v1 == *v2 ? 1 : 0; return mgp_error::MGP_ERROR_NO_ERROR; } @@ -2313,7 +2313,7 @@ void mgp_edge_destroy(mgp_edge *e) { DeleteRawMgpObject(e); } mgp_error mgp_edge_equal(mgp_edge *e1, mgp_edge *e2, int *result) { // NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression) - static_assert(noexcept(*result = *e1 == *e2 ? 1 : 0)); + static_assert(noexcept(*e1 == *e2)); *result = *e1 == *e2 ? 1 : 0; return mgp_error::MGP_ERROR_NO_ERROR; } diff --git a/src/query/stream/streams.cpp b/src/query/stream/streams.cpp index 16e7d7522..e93bf06a7 100644 --- a/src/query/stream/streams.cpp +++ b/src/query/stream/streams.cpp @@ -497,7 +497,7 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std #ifdef MG_ENTERPRISE interpreter->OnChangeCB([](auto) { return false; }); // Disable database change #endif - auto accessor = interpreter->db_acc_->get()->Access(); + auto accessor = interpreter->current_db_.db_acc_->get()->Access(); // register new interpreter into interpreter_context interpreter_context->interpreters->insert(interpreter.get()); utils::OnScopeExit interpreter_cleanup{ @@ -527,7 +527,7 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std std::string query{query_value.ValueString()}; spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name); auto prepare_result = - interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr); + interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), {}); if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges, "")) { throw StreamsException{ "Couldn't execute query '{}' for stream '{}' because the owner is not authorized to execute the " diff --git a/src/replication/CMakeLists.txt b/src/replication/CMakeLists.txt new file mode 100644 index 000000000..4170cd6a8 --- /dev/null +++ b/src/replication/CMakeLists.txt @@ -0,0 +1,24 @@ +add_library(mg-replication STATIC) +add_library(mg::replication ALIAS mg-replication) +target_sources(mg-replication + PUBLIC + include/replication/state.hpp + include/replication/epoch.hpp + include/replication/config.hpp + include/replication/mode.hpp + include/replication/role.hpp + include/replication/status.hpp + + PRIVATE + state.cpp + epoch.cpp + config.cpp + status.cpp +) +target_include_directories(mg-replication PUBLIC include) + +find_package(fmt REQUIRED) +target_link_libraries(mg-replication + PUBLIC mg::utils mg::kvstore lib::json + PRIVATE fmt::fmt +) diff --git a/src/replication/config.cpp b/src/replication/config.cpp new file mode 100644 index 000000000..c4bba4341 --- /dev/null +++ b/src/replication/config.cpp @@ -0,0 +1,11 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#include "replication/config.hpp" diff --git a/src/replication/epoch.cpp b/src/replication/epoch.cpp new file mode 100644 index 000000000..8561e8129 --- /dev/null +++ b/src/replication/epoch.cpp @@ -0,0 +1,12 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/epoch.hpp" diff --git a/src/storage/v2/replication/config.hpp b/src/replication/include/replication/config.hpp similarity index 80% rename from src/storage/v2/replication/config.hpp rename to src/replication/include/replication/config.hpp index d96e1ee76..975fb6b4d 100644 --- a/src/storage/v2/replication/config.hpp +++ b/src/replication/include/replication/config.hpp @@ -12,12 +12,17 @@ #pragma once #include +#include #include #include +#include "replication/mode.hpp" -#include "storage/v2/replication/enums.hpp" +namespace memgraph::replication { + +inline constexpr uint16_t kDefaultReplicationPort = 10000; +inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; +inline constexpr auto *kReservedReplicationRoleName{"__replication_role"}; -namespace memgraph::storage::replication { struct ReplicationClientConfig { std::string name; ReplicationMode mode; @@ -51,4 +56,4 @@ struct ReplicationServerConfig { std::optional ssl; }; -} // namespace memgraph::storage::replication +} // namespace memgraph::replication diff --git a/src/replication/include/replication/epoch.hpp b/src/replication/include/replication/epoch.hpp new file mode 100644 index 000000000..9cf04d11c --- /dev/null +++ b/src/replication/include/replication/epoch.hpp @@ -0,0 +1,49 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once +#include +#include + +#include "utils/uuid.hpp" + +namespace memgraph::replication { + +struct ReplicationEpoch { + ReplicationEpoch() : id_(memgraph::utils::GenerateUUID()) {} + ReplicationEpoch(ReplicationEpoch const &) = delete; + ReplicationEpoch(ReplicationEpoch &&) = delete; + ReplicationEpoch &operator=(ReplicationEpoch const &) = delete; + ReplicationEpoch &operator=(ReplicationEpoch &&) = delete; + + auto id() const -> std::string_view { return id_; } + + auto NewEpoch() -> std::string { return std::exchange(id_, memgraph::utils::GenerateUUID()); } + auto SetEpoch(std::string new_epoch) -> std::string { return std::exchange(id_, std::move(new_epoch)); } + + private: + // UUID to distinguish different main instance runs for replication process + // on SAME storage. + // Multiple instances can have same storage UUID and be MAIN at the same time. + // We cannot compare commit timestamps of those instances if one of them + // becomes the replica of the other so we use epoch_id_ as additional + // discriminating property. + // Example of this: + // We have 2 instances of the same storage, S1 and S2. + // S1 and S2 are MAIN and accept their own commits and write them to the WAL. + // At the moment when S1 commited a transaction with timestamp 20, and S2 + // a different transaction with timestamp 15, we change S2's role to REPLICA + // and register it on S1. + // Without using the epoch_id, we don't know that S1 and S2 have completely + // different transactions, we think that the S2 is behind only by 5 commits. + std::string id_; +}; +} // namespace memgraph::replication diff --git a/src/replication/include/replication/mode.hpp b/src/replication/include/replication/mode.hpp new file mode 100644 index 000000000..c1afe2b1f --- /dev/null +++ b/src/replication/include/replication/mode.hpp @@ -0,0 +1,16 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once +#include +namespace memgraph::replication { +enum class ReplicationMode : std::uint8_t { SYNC, ASYNC }; +} diff --git a/src/replication/include/replication/role.hpp b/src/replication/include/replication/role.hpp new file mode 100644 index 000000000..54fbab9f0 --- /dev/null +++ b/src/replication/include/replication/role.hpp @@ -0,0 +1,18 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +namespace memgraph::replication { + +enum class ReplicationRole : uint8_t { MAIN, REPLICA }; +} diff --git a/src/replication/include/replication/state.hpp b/src/replication/include/replication/state.hpp new file mode 100644 index 000000000..ac5b2841c --- /dev/null +++ b/src/replication/include/replication/state.hpp @@ -0,0 +1,69 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include + +#include "kvstore/kvstore.hpp" +#include "replication/config.hpp" +#include "replication/epoch.hpp" +#include "replication/mode.hpp" +#include "replication/role.hpp" +#include "utils/result.hpp" + +namespace memgraph::replication { + +enum class RolePersisted : uint8_t { UNKNOWN_OR_NO, YES }; + +struct ReplicationState { + ReplicationState(std::optional durability_dir); + + ReplicationState(ReplicationState const &) = delete; + ReplicationState(ReplicationState &&) = delete; + ReplicationState &operator=(ReplicationState const &) = delete; + ReplicationState &operator=(ReplicationState &&) = delete; + + void SetRole(ReplicationRole role) { return replication_role_.store(role); } + auto GetRole() const -> ReplicationRole { return replication_role_.load(); } + bool IsMain() const { return replication_role_ == ReplicationRole::MAIN; } + bool IsReplica() const { return replication_role_ == ReplicationRole::REPLICA; } + + auto GetEpoch() const -> const ReplicationEpoch & { return epoch_; } + auto GetEpoch() -> ReplicationEpoch & { return epoch_; } + + enum class FetchReplicationError : uint8_t { + NOTHING_FETCHED, + PARSE_ERROR, + }; + using ReplicationDataReplica = ReplicationServerConfig; + using ReplicationDataMain = std::vector; + using ReplicationData = std::variant; + using FetchReplicationResult = utils::BasicResult; + auto FetchReplicationData() -> FetchReplicationResult; + + bool ShouldPersist() const { return nullptr != durability_; } + bool TryPersistRoleMain(); + bool TryPersistRoleReplica(const ReplicationServerConfig &config); + bool TryPersistUnregisterReplica(std::string_view &name); + bool TryPersistRegisteredReplica(const ReplicationClientConfig &config); + + private: + ReplicationEpoch epoch_; + std::atomic replication_role_{ReplicationRole::MAIN}; + std::unique_ptr durability_; + std::atomic role_persisted = RolePersisted::UNKNOWN_OR_NO; +}; + +} // namespace memgraph::replication diff --git a/src/storage/v2/replication/replication_persistence_helper.hpp b/src/replication/include/replication/status.hpp similarity index 72% rename from src/storage/v2/replication/replication_persistence_helper.hpp rename to src/replication/include/replication/status.hpp index df6cb03b9..b158cf919 100644 --- a/src/storage/v2/replication/replication_persistence_helper.hpp +++ b/src/replication/include/replication/status.hpp @@ -12,21 +12,16 @@ #pragma once #include -#include +#include #include #include -#include +#include "json/json.hpp" -#include "storage/v2/replication/config.hpp" -#include "storage/v2/replication/enums.hpp" - -namespace memgraph::storage::replication { - -inline constexpr auto *kReservedReplicationRoleName{"__replication_role"}; -inline constexpr uint16_t kDefaultReplicationPort = 10000; -inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; +#include "replication/config.hpp" +#include "replication/role.hpp" +namespace memgraph::replication { struct ReplicationStatus { std::string name; std::string ip_address; @@ -40,6 +35,5 @@ struct ReplicationStatus { }; nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status); - std::optional JSONToReplicationStatus(nlohmann::json &&data); -} // namespace memgraph::storage::replication +} // namespace memgraph::replication diff --git a/src/replication/state.cpp b/src/replication/state.cpp new file mode 100644 index 000000000..45984d317 --- /dev/null +++ b/src/replication/state.cpp @@ -0,0 +1,144 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "replication/state.hpp" + +#include "replication/status.hpp" //TODO: don't use status for durability +#include "utils/file.hpp" + +constexpr auto kReplicationDirectory = std::string_view{"replication"}; + +namespace memgraph::replication { + +ReplicationState::ReplicationState(std::optional durability_dir) { + if (!durability_dir) return; + auto repl_dir = *std::move(durability_dir); + repl_dir /= kReplicationDirectory; + utils::EnsureDirOrDie(repl_dir); + durability_ = std::make_unique(std::move(repl_dir)); +} +bool ReplicationState::TryPersistRoleReplica(const ReplicationServerConfig &config) { + if (!ShouldPersist()) return true; + // Only thing that matters here is the role saved as REPLICA and the listening port + auto data = ReplicationStatusToJSON(ReplicationStatus{.name = kReservedReplicationRoleName, + .ip_address = config.ip_address, + .port = config.port, + .sync_mode = ReplicationMode::SYNC, + .replica_check_frequency = std::chrono::seconds(0), + .ssl = std::nullopt, + .role = ReplicationRole::REPLICA}); + + if (durability_->Put(kReservedReplicationRoleName, data.dump())) { + role_persisted = RolePersisted::YES; + return true; + } + spdlog::error("Error when saving REPLICA replication role in settings."); + return false; +} +bool ReplicationState::TryPersistRoleMain() { + if (!ShouldPersist()) return true; + // Only thing that matters here is the role saved as MAIN + auto data = ReplicationStatusToJSON(ReplicationStatus{.name = kReservedReplicationRoleName, + .ip_address = "", + .port = 0, + .sync_mode = ReplicationMode::SYNC, + .replica_check_frequency = std::chrono::seconds(0), + .ssl = std::nullopt, + .role = ReplicationRole::MAIN}); + + if (durability_->Put(kReservedReplicationRoleName, data.dump())) { + role_persisted = RolePersisted::YES; + return true; + } + spdlog::error("Error when saving MAIN replication role in settings."); + return false; +} +bool ReplicationState::TryPersistUnregisterReplica(std::string_view &name) { + if (!ShouldPersist()) return true; + if (durability_->Delete(name)) return true; + spdlog::error("Error when removing replica {} from settings.", name); + return false; +} +auto ReplicationState::FetchReplicationData() -> FetchReplicationResult { + if (!ShouldPersist()) return FetchReplicationError::NOTHING_FETCHED; + const auto replication_data = durability_->Get(kReservedReplicationRoleName); + if (!replication_data.has_value()) { + return FetchReplicationError::NOTHING_FETCHED; + } + + const auto maybe_replication_status = JSONToReplicationStatus(nlohmann::json::parse(*replication_data)); + if (!maybe_replication_status.has_value()) { + return FetchReplicationError::PARSE_ERROR; + } + + // To get here this must be the case + role_persisted = memgraph::replication::RolePersisted::YES; + + const auto replication_status = *maybe_replication_status; + auto role = replication_status.role.value_or(ReplicationRole::MAIN); + switch (role) { + case ReplicationRole::REPLICA: { + return {ReplicationServerConfig{ + .ip_address = kDefaultReplicationServerIp, + .port = replication_status.port, + }}; + } + case ReplicationRole::MAIN: { + auto res = ReplicationState::ReplicationDataMain{}; + res.reserve(durability_->Size() - 1); + for (const auto &[replica_name, replica_data] : *durability_) { + if (replica_name == kReservedReplicationRoleName) { + continue; + } + + const auto maybe_replica_status = JSONToReplicationStatus(nlohmann::json::parse(replica_data)); + if (!maybe_replica_status.has_value()) { + return FetchReplicationError::PARSE_ERROR; + } + + auto replica_status = *maybe_replica_status; + if (replica_status.name != replica_name) { + return FetchReplicationError::PARSE_ERROR; + } + res.emplace_back(ReplicationClientConfig{ + .name = replica_status.name, + .mode = replica_status.sync_mode, + .ip_address = replica_status.ip_address, + .port = replica_status.port, + .replica_check_frequency = replica_status.replica_check_frequency, + .ssl = replica_status.ssl, + }); + } + return {std::move(res)}; + } + } +} +bool ReplicationState::TryPersistRegisteredReplica(const ReplicationClientConfig &config) { + if (!ShouldPersist()) return true; + + // If any replicas are persisted then Role must be persisted + if (role_persisted != RolePersisted::YES) { + DMG_ASSERT(IsMain(), "MAIN is expected"); + if (!TryPersistRoleMain()) return false; + } + + auto data = ReplicationStatusToJSON(ReplicationStatus{.name = config.name, + .ip_address = config.ip_address, + .port = config.port, + .sync_mode = config.mode, + .replica_check_frequency = config.replica_check_frequency, + .ssl = config.ssl, + .role = ReplicationRole::REPLICA}); + if (durability_->Put(config.name, data.dump())) return true; + spdlog::error("Error when saving replica {} in settings.", config.name); + return false; +} +} // namespace memgraph::replication diff --git a/src/storage/v2/replication/replication_persistence_helper.cpp b/src/replication/status.cpp similarity index 76% rename from src/storage/v2/replication/replication_persistence_helper.cpp rename to src/replication/status.cpp index 262938a82..711b1f955 100644 --- a/src/storage/v2/replication/replication_persistence_helper.cpp +++ b/src/replication/status.cpp @@ -8,24 +8,21 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include "replication/status.hpp" -#include "storage/v2/replication/replication_persistence_helper.hpp" - -#include "storage/v2/replication/enums.hpp" +#include "fmt/format.h" #include "utils/logging.hpp" -namespace { -inline constexpr auto *kReplicaName = "replica_name"; -inline constexpr auto *kIpAddress = "replica_ip_address"; -inline constexpr auto *kPort = "replica_port"; -inline constexpr auto *kSyncMode = "replica_sync_mode"; -inline constexpr auto *kCheckFrequency = "replica_check_frequency"; -inline constexpr auto *kSSLKeyFile = "replica_ssl_key_file"; -inline constexpr auto *kSSLCertFile = "replica_ssl_cert_file"; -inline constexpr auto *kReplicationRole = "replication_role"; -} // namespace +constexpr auto *kReplicaName = "replica_name"; +constexpr auto *kIpAddress = "replica_ip_address"; +constexpr auto *kPort = "replica_port"; +constexpr auto *kSyncMode = "replica_sync_mode"; +constexpr auto *kCheckFrequency = "replica_check_frequency"; +constexpr auto *kSSLKeyFile = "replica_ssl_key_file"; +constexpr auto *kSSLCertFile = "replica_ssl_cert_file"; +constexpr auto *kReplicationRole = "replication_role"; -namespace memgraph::storage::replication { +namespace memgraph::replication { nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status) { auto data = nlohmann::json::object(); @@ -51,7 +48,6 @@ nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status) { return data; } - std::optional JSONToReplicationStatus(nlohmann::json &&data) { ReplicationStatus replica_status; @@ -73,13 +69,13 @@ std::optional JSONToReplicationStatus(nlohmann::json &&data) MG_ASSERT(key_file.is_null() == cert_file.is_null()); if (!key_file.is_null()) { - replica_status.ssl = replication::ReplicationClientConfig::SSL{}; + replica_status.ssl = ReplicationClientConfig::SSL{}; data.at(kSSLKeyFile).get_to(replica_status.ssl->key_file); data.at(kSSLCertFile).get_to(replica_status.ssl->cert_file); } if (data.find(kReplicationRole) != data.end()) { - replica_status.role = replication::ReplicationRole::MAIN; + replica_status.role = ReplicationRole::MAIN; data.at(kReplicationRole).get_to(replica_status.role.value()); } } catch (const nlohmann::json::type_error &exception) { @@ -92,4 +88,4 @@ std::optional JSONToReplicationStatus(nlohmann::json &&data) return replica_status; } -} // namespace memgraph::storage::replication +} // namespace memgraph::replication diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 7c9c567ed..a6eaebe24 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -36,13 +36,13 @@ add_library(mg-storage-v2 STATIC replication/replication_server.cpp replication/serialization.cpp replication/slk.cpp - replication/replication_persistence_helper.cpp replication/rpc.cpp - replication/replication.cpp + replication/replication_storage_state.cpp + replication/replication_handler.cpp inmemory/replication/replication_server.cpp inmemory/replication/replication_client.cpp ) -target_link_libraries(mg-storage-v2 Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk) +target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk) # Until we get LTO there is an advantage to do some unity builds set_target_properties(mg-storage-v2 diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 4631f0762..6f322eb14 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -320,8 +320,9 @@ DiskStorage::~DiskStorage() { kvstore_->options_.comparator = nullptr; } -DiskStorage::DiskAccessor::DiskAccessor(DiskStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode) - : Accessor(storage, isolation_level, storage_mode), config_(storage->config_.items) { +DiskStorage::DiskAccessor::DiskAccessor(auto tag, DiskStorage *storage, IsolationLevel isolation_level, + StorageMode storage_mode) + : Accessor(tag, storage, isolation_level, storage_mode), config_(storage->config_.items) { rocksdb::WriteOptions write_options; auto txOptions = rocksdb::TransactionOptions{.set_snapshot = true}; disk_transaction_ = storage->kvstore_->db_->BeginTransaction(write_options, txOptions); @@ -1091,7 +1092,7 @@ void DiskStorage::SetEdgeImportMode(EdgeImportMode edge_import_status) { } EdgeImportMode DiskStorage::GetEdgeImportMode() const { - std::shared_lock storage_guard_(main_lock_); + std::shared_lock storage_guard_(main_lock_); return edge_import_status_; } @@ -1445,24 +1446,24 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromDisk(const std::string &edge) { return true; } -[[nodiscard]] utils::BasicResult +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( const Vertex &vertex, std::vector> &unique_storage) const { if (auto existence_constraint_validation_result = storage_->constraints_.existence_constraints_->Validate(vertex); existence_constraint_validation_result.has_value()) { - return StorageDataManipulationError{existence_constraint_validation_result.value()}; + return StorageManipulationError{existence_constraint_validation_result.value()}; } auto *disk_unique_constraints = static_cast(storage_->constraints_.unique_constraints_.get()); if (auto unique_constraint_validation_result = disk_unique_constraints->Validate(vertex, unique_storage); unique_constraint_validation_result.has_value()) { - return StorageDataManipulationError{unique_constraint_validation_result.value()}; + return StorageManipulationError{unique_constraint_validation_result.value()}; } return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushVertices( +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushVertices( const auto &vertex_acc, std::vector> &unique_storage) { auto *disk_unique_constraints = static_cast(storage_->constraints_.unique_constraints_.get()); @@ -1485,26 +1486,25 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( /// NOTE: this deletion has to come before writing, otherwise RocksDB thinks that all entries are deleted if (auto maybe_old_disk_key = utils::GetOldDiskKeyOrNull(vertex.delta); maybe_old_disk_key.has_value()) { if (!DeleteVertexFromDisk(maybe_old_disk_key.value())) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } if (!WriteVertexToDisk(vertex)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } if (!disk_unique_constraints->SyncVertexToUniqueConstraintsStorage(vertex, *commit_timestamp_) || !disk_label_index->SyncVertexToLabelIndexStorage(vertex, *commit_timestamp_) || !disk_label_property_index->SyncVertexToLabelPropertyIndexStorage(vertex, *commit_timestamp_)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } return {}; } -[[nodiscard]] utils::BasicResult -DiskStorage::DiskAccessor::ClearDanglingVertices() { +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::ClearDanglingVertices() { auto *disk_unique_constraints = static_cast(storage_->constraints_.unique_constraints_.get()); auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); @@ -1516,12 +1516,12 @@ DiskStorage::DiskAccessor::ClearDanglingVertices() { !disk_label_index->DeleteVerticesWithRemovedIndexingLabel(transaction_.start_timestamp, *commit_timestamp_) || !disk_label_property_index->DeleteVerticesWithRemovedIndexingLabel(transaction_.start_timestamp, *commit_timestamp_)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushIndexCache() { +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushIndexCache() { std::vector> unique_storage; auto flush_index = [&](auto &index_storage) -> utils::BasicResult { @@ -1552,7 +1552,7 @@ DiskStorage::DiskAccessor::ClearDanglingVertices() { return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedVertices() { +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedVertices() { auto *disk_unique_constraints = static_cast(storage_->constraints_.unique_constraints_.get()); auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); @@ -1564,23 +1564,23 @@ DiskStorage::DiskAccessor::ClearDanglingVertices() { !disk_unique_constraints->ClearDeletedVertex(vertex_gid, *commit_timestamp_) || !disk_label_index->ClearDeletedVertex(vertex_gid, *commit_timestamp_) || !disk_label_property_index->ClearDeletedVertex(vertex_gid, *commit_timestamp_)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedEdges() { +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedEdges() { for (const auto &edge_to_delete : edges_to_delete_) { if (!DeleteEdgeFromDisk(edge_to_delete)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushModifiedEdges( +[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushModifiedEdges( const auto &edge_acc) { for (const auto &modified_edge : transaction_.modified_edges_) { const storage::Gid &gid = modified_edge.first; @@ -1592,21 +1592,21 @@ DiskStorage::DiskAccessor::ClearDanglingVertices() { /// If the object was created then flush it, otherwise since properties on edges are false /// edge wasn't modified for sure. if (action == Delta::Action::DELETE_OBJECT && !WriteEdgeToDisk(ser_edge_key, "")) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } else { // If the delta is DELETE_OBJECT, the edge is just created so there is nothing to delete. // If the edge was deserialized, only properties can be modified -> key stays the same as when deserialized // so we can delete it. if (action == Delta::Action::DELETE_DESERIALIZED_OBJECT && !DeleteEdgeFromDisk(ser_edge_key)) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } const auto &edge = edge_acc.find(gid); MG_ASSERT(edge != edge_acc.end(), "Database in invalid state, commit not possible! Please restart your DB and start the import again."); if (!WriteEdgeToDisk(ser_edge_key, utils::SerializeProperties(edge->properties))) { - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } } } @@ -1661,7 +1661,7 @@ DiskStorage::CheckExistingVerticesBeforeCreatingUniqueConstraint(LabelId label, } // NOLINTNEXTLINE(google-default-arguments) -utils::BasicResult DiskStorage::DiskAccessor::Commit( +utils::BasicResult DiskStorage::DiskAccessor::Commit( const std::optional desired_commit_timestamp) { MG_ASSERT(is_transaction_active_, "The transaction is already terminated!"); MG_ASSERT(!transaction_.must_abort, "The transaction can't be committed!"); @@ -1669,10 +1669,84 @@ utils::BasicResult DiskStorage::DiskAccessor auto *disk_storage = static_cast(storage_); bool edge_import_mode_active = disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE; - if (transaction_.deltas.use().empty() || - (!edge_import_mode_active && - std::all_of(transaction_.deltas.use().begin(), transaction_.deltas.use().end(), - [](const Delta &delta) { return delta.action == Delta::Action::DELETE_DESERIALIZED_OBJECT; }))) { + if (!transaction_.md_deltas.empty()) { + // This is usually done by the MVCC, but it does not handle the metadata deltas + transaction_.EnsureCommitTimestampExists(); + std::unique_lock engine_guard(storage_->engine_lock_); + commit_timestamp_.emplace(disk_storage->CommitTimestamp(desired_commit_timestamp)); + transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); + + for (const auto &md_delta : transaction_.md_deltas) { + switch (md_delta.action) { + case MetadataDelta::Action::LABEL_INDEX_CREATE: { + if (!disk_storage->PersistLabelIndexCreation(md_delta.label)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_CREATE: { + const auto &info = md_delta.label_property; + if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintCreation(info.label, info.property, + label_property_index_str)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::LABEL_INDEX_DROP: { + if (!disk_storage->PersistLabelIndexDeletion(md_delta.label)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_DROP: { + const auto &info = md_delta.label_property; + if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintDeletion(info.label, info.property, + label_property_index_str)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::LABEL_INDEX_STATS_SET: { + throw utils::NotYetImplemented("SetIndexStats(stats) is not implemented for DiskStorage."); + } break; + case MetadataDelta::Action::LABEL_INDEX_STATS_CLEAR: { + throw utils::NotYetImplemented("ClearIndexStats(stats) is not implemented for DiskStorage."); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_STATS_SET: { + throw utils::NotYetImplemented("SetIndexStats(stats) is not implemented for DiskStorage."); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_STATS_CLEAR: { + throw utils::NotYetImplemented("ClearIndexStats(stats) is not implemented for DiskStorage."); + } break; + case MetadataDelta::Action::EXISTENCE_CONSTRAINT_CREATE: { + const auto &info = md_delta.label_property; + if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintCreation(info.label, info.property, + existence_constraints_str)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::EXISTENCE_CONSTRAINT_DROP: { + const auto &info = md_delta.label_property; + if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintDeletion(info.label, info.property, + existence_constraints_str)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::UNIQUE_CONSTRAINT_CREATE: { + const auto &info = md_delta.label_properties; + if (!disk_storage->PersistUniqueConstraintCreation(info.label, info.properties)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + case MetadataDelta::Action::UNIQUE_CONSTRAINT_DROP: { + const auto &info = md_delta.label_properties; + if (!disk_storage->PersistUniqueConstraintDeletion(info.label, info.properties)) { + return StorageManipulationError{PersistenceError{}}; + } + } break; + } + } + } else if (transaction_.deltas.use().empty() || + (!edge_import_mode_active && + std::all_of(transaction_.deltas.use().begin(), transaction_.deltas.use().end(), [](const Delta &delta) { + return delta.action == Delta::Action::DELETE_DESERIALIZED_OBJECT; + }))) { } else { std::unique_lock engine_guard(storage_->engine_lock_); commit_timestamp_.emplace(disk_storage->CommitTimestamp(desired_commit_timestamp)); @@ -1726,7 +1800,7 @@ utils::BasicResult DiskStorage::DiskAccessor disk_transaction_ = nullptr; if (!commitStatus.ok()) { spdlog::error("rocksdb: Commit failed with status {}", commitStatus.ToString()); - return StorageDataManipulationError{SerializationError{}}; + return StorageManipulationError{SerializationError{}}; } spdlog::trace("rocksdb: Commit successful"); @@ -1871,156 +1945,122 @@ void DiskStorage::DiskAccessor::FinalizeTransaction() { } } -utils::BasicResult DiskStorage::CreateIndex( - LabelId label, const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - auto *disk_label_index = static_cast(indices_.label_index_.get()); - if (!disk_label_index->CreateIndex(label, SerializeVerticesForLabelIndex(label))) { +utils::BasicResult DiskStorage::DiskAccessor::CreateIndex(LabelId label) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_label_index = static_cast(on_disk->indices_.label_index_.get()); + if (!disk_label_index->CreateIndex(label, on_disk->SerializeVerticesForLabelIndex(label))) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - - if (!PersistLabelIndexCreation(label)) { - return StorageIndexDefinitionError{IndexPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_create, label); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelIndices); - return {}; } -utils::BasicResult DiskStorage::CreateIndex( - LabelId label, PropertyId property, const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); +utils::BasicResult DiskStorage::DiskAccessor::CreateIndex(LabelId label, + PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_label_property_index = + static_cast(on_disk->indices_.label_property_index_.get()); if (!disk_label_property_index->CreateIndex(label, property, - SerializeVerticesForLabelPropertyIndex(label, property))) { + on_disk->SerializeVerticesForLabelPropertyIndex(label, property))) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - - if (!PersistLabelPropertyIndexAndExistenceConstraintCreation(label, property, label_property_index_str)) { - return StorageIndexDefinitionError{IndexPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_create, label, property); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelPropertyIndices); - return {}; } -utils::BasicResult DiskStorage::DropIndex( - LabelId label, const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - if (!indices_.label_index_->DropIndex(label)) { +utils::BasicResult DiskStorage::DiskAccessor::DropIndex(LabelId label) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_label_index = static_cast(on_disk->indices_.label_index_.get()); + if (!disk_label_index->DropIndex(label)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - - if (!PersistLabelIndexDeletion(label)) { - return StorageIndexDefinitionError{IndexPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_drop, label); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelIndices); - return {}; } -utils::BasicResult DiskStorage::DropIndex( - LabelId label, PropertyId property, const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - if (!indices_.label_property_index_->DropIndex(label, property)) { +utils::BasicResult DiskStorage::DiskAccessor::DropIndex(LabelId label, + PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_label_property_index = + static_cast(on_disk->indices_.label_property_index_.get()); + if (!disk_label_property_index->DropIndex(label, property)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - - if (!PersistLabelPropertyIndexAndExistenceConstraintDeletion(label, property, label_property_index_str)) { - return StorageIndexDefinitionError{IndexPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_drop, label, property); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelPropertyIndices); - return {}; } -utils::BasicResult DiskStorage::CreateExistenceConstraint( - LabelId label, PropertyId property, const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - if (constraints_.existence_constraints_->ConstraintExists(label, property)) { +utils::BasicResult +DiskStorage::DiskAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create existence constraint requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *existence_constraints = on_disk->constraints_.existence_constraints_.get(); + if (existence_constraints->ConstraintExists(label, property)) { return StorageExistenceConstraintDefinitionError{ConstraintDefinitionError{}}; } - - if (auto check = CheckExistingVerticesBeforeCreatingExistenceConstraint(label, property); check.has_value()) { + if (auto check = on_disk->CheckExistingVerticesBeforeCreatingExistenceConstraint(label, property); + check.has_value()) { return StorageExistenceConstraintDefinitionError{check.value()}; } - - constraints_.existence_constraints_->InsertConstraint(label, property); - - if (!PersistLabelPropertyIndexAndExistenceConstraintCreation(label, property, existence_constraints_str)) { - return StorageExistenceConstraintDefinitionError{ConstraintsPersistenceError{}}; - } - + existence_constraints->InsertConstraint(label, property); + transaction_.md_deltas.emplace_back(MetadataDelta::existence_constraint_create, label, property); return {}; } -utils::BasicResult DiskStorage::DropExistenceConstraint( - LabelId label, PropertyId property, const std::optional /*desired_commit_timestamp*/) { - if (!constraints_.existence_constraints_->DropConstraint(label, property)) { +utils::BasicResult DiskStorage::DiskAccessor::DropExistenceConstraint( + LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Drop existence constraint requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *existence_constraints = on_disk->constraints_.existence_constraints_.get(); + if (!existence_constraints->DropConstraint(label, property)) { return StorageExistenceConstraintDroppingError{ConstraintDefinitionError{}}; } - - if (!PersistLabelPropertyIndexAndExistenceConstraintDeletion(label, property, existence_constraints_str)) { - return StorageExistenceConstraintDroppingError{ConstraintsPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::existence_constraint_drop, label, property); return {}; } utils::BasicResult -DiskStorage::CreateUniqueConstraint(LabelId label, const std::set &properties, - const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - - auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); - +DiskStorage::DiskAccessor::CreateUniqueConstraint(LabelId label, const std::set &properties) { + MG_ASSERT(unique_guard_.owns_lock(), "Create unique constraint requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_unique_constraints = static_cast(on_disk->constraints_.unique_constraints_.get()); if (auto constraint_check = disk_unique_constraints->CheckIfConstraintCanBeCreated(label, properties); constraint_check != UniqueConstraints::CreationStatus::SUCCESS) { return constraint_check; } - - auto check = CheckExistingVerticesBeforeCreatingUniqueConstraint(label, properties); + auto check = on_disk->CheckExistingVerticesBeforeCreatingUniqueConstraint(label, properties); if (check.HasError()) { return StorageUniqueConstraintDefinitionError{check.GetError()}; } - if (!disk_unique_constraints->InsertConstraint(label, properties, check.GetValue())) { return StorageUniqueConstraintDefinitionError{ConstraintDefinitionError{}}; } - - if (!PersistUniqueConstraintCreation(label, properties)) { - return StorageUniqueConstraintDefinitionError{ConstraintsPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::unique_constraint_create, label, properties); return UniqueConstraints::CreationStatus::SUCCESS; } -utils::BasicResult -DiskStorage::DropUniqueConstraint(LabelId label, const std::set &properties, - const std::optional /*desired_commit_timestamp*/) { - std::unique_lock storage_guard(main_lock_); - auto ret = constraints_.unique_constraints_->DropConstraint(label, properties); - if (ret != UniqueConstraints::DeletionStatus::SUCCESS) { +UniqueConstraints::DeletionStatus DiskStorage::DiskAccessor::DropUniqueConstraint( + LabelId label, const std::set &properties) { + MG_ASSERT(unique_guard_.owns_lock(), "Drop unique constraint requires a unique access to the storage!"); + auto *on_disk = static_cast(storage_); + auto *disk_unique_constraints = static_cast(on_disk->constraints_.unique_constraints_.get()); + if (auto ret = disk_unique_constraints->DropConstraint(label, properties); + ret != UniqueConstraints::DeletionStatus::SUCCESS) { return ret; } - - if (!PersistUniqueConstraintDeletion(label, properties)) { - return StorageUniqueConstraintDroppingError{ConstraintsPersistenceError{}}; - } - + transaction_.md_deltas.emplace_back(MetadataDelta::unique_constraint_create, label, properties); return UniqueConstraints::DeletionStatus::SUCCESS; } @@ -2050,4 +2090,32 @@ uint64_t DiskStorage::CommitTimestamp(const std::optional desired_comm return *desired_commit_timestamp; } +std::unique_ptr DiskStorage::Access(std::optional override_isolation_level) { + auto isolation_level = override_isolation_level.value_or(isolation_level_); + if (isolation_level != IsolationLevel::SNAPSHOT_ISOLATION) { + throw utils::NotYetImplemented("Disk storage supports only SNAPSHOT isolation level."); + } + return std::unique_ptr( + new DiskAccessor{Storage::Accessor::shared_access, this, isolation_level, storage_mode_}); +} +std::unique_ptr DiskStorage::UniqueAccess(std::optional override_isolation_level) { + auto isolation_level = override_isolation_level.value_or(isolation_level_); + if (isolation_level != IsolationLevel::SNAPSHOT_ISOLATION) { + throw utils::NotYetImplemented("Disk storage supports only SNAPSHOT isolation level."); + } + return std::unique_ptr( + new DiskAccessor{Storage::Accessor::unique_access, this, isolation_level, storage_mode_}); +} +IndicesInfo DiskStorage::DiskAccessor::ListAllIndices() const { + auto *on_disk = static_cast(storage_); + auto *disk_label_index = static_cast(on_disk->indices_.label_index_.get()); + auto *disk_label_property_index = + static_cast(on_disk->indices_.label_property_index_.get()); + return {disk_label_index->ListIndices(), disk_label_property_index->ListIndices()}; +} +ConstraintsInfo DiskStorage::DiskAccessor::ListAllConstraints() const { + auto *disk_storage = static_cast(storage_); + return {disk_storage->constraints_.existence_constraints_->ListConstraints(), + disk_storage->constraints_.unique_constraints_->ListConstraints()}; +} } // namespace memgraph::storage diff --git a/src/storage/v2/disk/storage.hpp b/src/storage/v2/disk/storage.hpp index 68181a2ec..d2f6615d7 100644 --- a/src/storage/v2/disk/storage.hpp +++ b/src/storage/v2/disk/storage.hpp @@ -44,7 +44,7 @@ class DiskStorage final : public Storage { private: friend class DiskStorage; - explicit DiskAccessor(DiskStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode); + explicit DiskAccessor(auto tag, DiskStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode); /// TODO: const methods? void LoadVerticesToMainMemoryCache(); @@ -145,20 +145,12 @@ class DiskStorage final : public Storage { return {}; } - std::vector ClearLabelIndexStats() override { - throw utils::NotYetImplemented("ClearIndexStats() is not implemented for DiskStorage."); - } - - std::vector> ClearLabelPropertyIndexStats() override { - throw utils::NotYetImplemented("ClearIndexStats() is not implemented for DiskStorage."); - } - - std::vector DeleteLabelIndexStats(std::span /*labels*/) override { + bool DeleteLabelIndexStats(const storage::LabelId & /*labels*/) override { throw utils::NotYetImplemented("DeleteIndexStatsForLabels(labels) is not implemented for DiskStorage."); } std::vector> DeleteLabelPropertyIndexStats( - const std::span /*labels*/) override { + const storage::LabelId & /*labels*/) override { throw utils::NotYetImplemented("DeleteIndexStatsForLabels(labels) is not implemented for DiskStorage."); } @@ -194,18 +186,12 @@ class DiskStorage final : public Storage { return disk_storage->indices_.label_property_index_->IndexExists(label, property); } - IndicesInfo ListAllIndices() const override { - auto *disk_storage = static_cast(storage_); - return disk_storage->ListAllIndices(); - } + IndicesInfo ListAllIndices() const override; - ConstraintsInfo ListAllConstraints() const override { - auto *disk_storage = static_cast(storage_); - return disk_storage->ListAllConstraints(); - } + ConstraintsInfo ListAllConstraints() const override; // NOLINTNEXTLINE(google-default-arguments) - utils::BasicResult Commit( + utils::BasicResult Commit( std::optional desired_commit_timestamp = {}) override; void UpdateObjectsCountOnAbort(); @@ -227,6 +213,26 @@ class DiskStorage final : public Storage { std::optional DeserializeEdge(const rocksdb::Slice &key, const rocksdb::Slice &value, const rocksdb::Slice &ts); + utils::BasicResult CreateIndex(LabelId label) override; + + utils::BasicResult CreateIndex(LabelId label, PropertyId property) override; + + utils::BasicResult DropIndex(LabelId label) override; + + utils::BasicResult DropIndex(LabelId label, PropertyId property) override; + + utils::BasicResult CreateExistenceConstraint( + LabelId label, PropertyId property) override; + + utils::BasicResult DropExistenceConstraint( + LabelId label, PropertyId property) override; + + utils::BasicResult + CreateUniqueConstraint(LabelId label, const std::set &properties) override; + + UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, + const std::set &properties) override; + private: VertexAccessor CreateVertexFromDisk(utils::SkipList::Accessor &accessor, storage::Gid gid, std::vector &&label_ids, PropertyStore &&properties, Delta *delta); @@ -242,20 +248,20 @@ class DiskStorage final : public Storage { /// At the time of calling, the commit_timestamp_ must already exist. /// After this method, the vertex and edge caches are cleared. - [[nodiscard]] utils::BasicResult FlushIndexCache(); + [[nodiscard]] utils::BasicResult FlushIndexCache(); - [[nodiscard]] utils::BasicResult FlushDeletedVertices(); + [[nodiscard]] utils::BasicResult FlushDeletedVertices(); - [[nodiscard]] utils::BasicResult FlushDeletedEdges(); + [[nodiscard]] utils::BasicResult FlushDeletedEdges(); - [[nodiscard]] utils::BasicResult FlushVertices( + [[nodiscard]] utils::BasicResult FlushVertices( const auto &vertex_acc, std::vector> &unique_storage); - [[nodiscard]] utils::BasicResult FlushModifiedEdges(const auto &edge_acc); + [[nodiscard]] utils::BasicResult FlushModifiedEdges(const auto &edge_acc); - [[nodiscard]] utils::BasicResult ClearDanglingVertices(); + [[nodiscard]] utils::BasicResult ClearDanglingVertices(); - [[nodiscard]] utils::BasicResult CheckVertexConstraintsBeforeCommit( + [[nodiscard]] utils::BasicResult CheckVertexConstraintsBeforeCommit( const Vertex &vertex, std::vector> &unique_storage) const; bool WriteVertexToDisk(const Vertex &vertex); @@ -290,42 +296,14 @@ class DiskStorage final : public Storage { std::vector> vertices_to_delete_; rocksdb::Transaction *disk_transaction_; bool scanned_all_vertices_ = false; - }; + }; // Accessor - std::unique_ptr Access(std::optional override_isolation_level) override { - auto isolation_level = override_isolation_level.value_or(isolation_level_); - if (isolation_level != IsolationLevel::SNAPSHOT_ISOLATION) { - throw utils::NotYetImplemented("Disk storage supports only SNAPSHOT isolation level."); - } - return std::unique_ptr(new DiskAccessor{this, isolation_level, storage_mode_}); - } + std::unique_ptr Access(std::optional override_isolation_level) override; + + std::unique_ptr UniqueAccess(std::optional override_isolation_level) override; RocksDBStorage *GetRocksDBStorage() const { return kvstore_.get(); } - utils::BasicResult CreateIndex( - LabelId label, std::optional desired_commit_timestamp) override; - - utils::BasicResult CreateIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - utils::BasicResult DropIndex( - LabelId label, std::optional desired_commit_timestamp) override; - - utils::BasicResult DropIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - utils::BasicResult CreateExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - utils::BasicResult DropExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - utils::BasicResult CreateUniqueConstraint( - LabelId label, const std::set &properties, std::optional desired_commit_timestamp) override; - - utils::BasicResult DropUniqueConstraint( - LabelId label, const std::set &properties, std::optional desired_commit_timestamp) override; - Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode) override; void SetEdgeImportMode(EdgeImportMode edge_import_status); @@ -379,21 +357,23 @@ class DiskStorage final : public Storage { StorageInfo GetInfo() const override; - void FreeMemory(std::unique_lock /*lock*/) override {} + void FreeMemory(std::unique_lock /*lock*/) override {} - void EstablishNewEpoch() override { throw utils::BasicException("Disk storage mode does not support replication."); } + void PrepareForNewEpoch(std::string prev_epoch) override { + throw utils::BasicException("Disk storage mode does not support replication."); + } uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); EdgeImportMode edge_import_status_{EdgeImportMode::INACTIVE}; std::unique_ptr edge_import_mode_cache_{nullptr}; - auto CreateReplicationClient(replication::ReplicationClientConfig const &config) + auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config) -> std::unique_ptr override { throw utils::BasicException("Disk storage mode does not support replication."); } - auto CreateReplicationServer(const replication::ReplicationServerConfig &config) + auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig &config) -> std::unique_ptr override { throw utils::BasicException("Disk storage mode does not support replication."); } diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index d80eac1fe..aa0730e6f 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -24,6 +24,7 @@ #include #include +#include "replication/epoch.hpp" #include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/wal.hpp" @@ -133,8 +134,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("Recreating indices from metadata."); // Recover label indices. spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size()); + auto *mem_label_index = static_cast(indices->label_index_.get()); for (const auto &item : indices_constraints.indices.label) { - auto *mem_label_index = static_cast(indices->label_index_.get()); if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) throw RecoveryFailure("The label index must be created here!"); @@ -142,6 +143,15 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ } spdlog::info("Label indices are recreated."); + spdlog::info("Recreating index statistics from metadata."); + // Recover label indices statistics. + spdlog::info("Recreating {} label index statistics from metadata.", indices_constraints.indices.label_stats.size()); + for (const auto &item : indices_constraints.indices.label_stats) { + mem_label_index->SetIndexStats(item.first, item.second); + spdlog::info("A label index statistics is recreated from metadata."); + } + spdlog::info("Label indices statistics are recreated."); + // Recover label+property indices. spdlog::info("Recreating {} label+property indices from metadata.", indices_constraints.indices.label_property.size()); @@ -152,6 +162,19 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ spdlog::info("A label+property index is recreated from metadata."); } spdlog::info("Label+property indices are recreated."); + + // Recover label+property indices statistics. + spdlog::info("Recreating {} label+property indices statistics from metadata.", + indices_constraints.indices.label_property_stats.size()); + for (const auto &item : indices_constraints.indices.label_property_stats) { + const auto label_id = item.first; + const auto property_id = item.second.first; + const auto &stats = item.second.second; + mem_label_property_index->SetIndexStats({label_id, property_id}, stats); + spdlog::info("A label+property index statistics is recreated from metadata."); + } + spdlog::info("Label+property indices statistics are recreated."); + spdlog::info("Indices are recreated."); spdlog::info("Recreating constraints from metadata."); @@ -188,7 +211,7 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_ std::optional RecoverData(const std::filesystem::path &snapshot_directory, const std::filesystem::path &wal_directory, std::string *uuid, - std::string *epoch_id, + memgraph::replication::ReplicationEpoch &epoch, std::deque> *epoch_history, utils::SkipList *vertices, utils::SkipList *edges, std::atomic *edge_count, NameIdMapper *name_id_mapper, @@ -241,7 +264,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di recovery_info = recovered_snapshot->recovery_info; indices_constraints = std::move(recovered_snapshot->indices_constraints); snapshot_timestamp = recovered_snapshot->snapshot_info.start_timestamp; - *epoch_id = std::move(recovered_snapshot->snapshot_info.epoch_id); + epoch.SetEpoch(std::move(recovered_snapshot->snapshot_info.epoch_id)); if (!utils::DirExists(wal_directory)) { const auto par_exec_info = config.durability.allow_parallel_index_creation @@ -286,7 +309,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di // UUID used for durability is the UUID of the last WAL file. // Same for the epoch id. *uuid = std::move(wal_files.back().uuid); - *epoch_id = std::move(wal_files.back().epoch_id); + epoch.SetEpoch(std::move(wal_files.back().epoch_id)); } auto maybe_wal_files = GetWalFiles(wal_directory, *uuid); @@ -342,7 +365,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di } previous_seq_num = wal_file.seq_num; - if (wal_file.epoch_id != *epoch_id) { + if (wal_file.epoch_id != epoch.id()) { // This way we skip WALs finalized only because of role change. // We can also set the last timestamp to 0 if last loaded timestamp // is nullopt as this can only happen if the WAL file with seq = 0 @@ -350,7 +373,7 @@ std::optional RecoverData(const std::filesystem::path &snapshot_di if (last_loaded_timestamp) { epoch_history->emplace_back(wal_file.epoch_id, *last_loaded_timestamp); } - *epoch_id = std::move(wal_file.epoch_id); + epoch.SetEpoch(std::move(wal_file.epoch_id)); } try { auto info = LoadWal(wal_file.path, &indices_constraints, last_loaded_timestamp, vertices, edges, name_id_mapper, diff --git a/src/storage/v2/durability/durability.hpp b/src/storage/v2/durability/durability.hpp index 99efde6b6..b68e18071 100644 --- a/src/storage/v2/durability/durability.hpp +++ b/src/storage/v2/durability/durability.hpp @@ -18,6 +18,7 @@ #include #include +#include "replication/epoch.hpp" #include "storage/v2/config.hpp" #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/durability/metadata.hpp" @@ -109,7 +110,7 @@ void RecoverIndicesAndConstraints( /// @throw std::bad_alloc std::optional RecoverData(const std::filesystem::path &snapshot_directory, const std::filesystem::path &wal_directory, std::string *uuid, - std::string *epoch_id, + memgraph::replication::ReplicationEpoch &epoch, std::deque> *epoch_history, utils::SkipList *vertices, utils::SkipList *edges, std::atomic *edge_count, NameIdMapper *name_id_mapper, diff --git a/src/storage/v2/durability/marker.hpp b/src/storage/v2/durability/marker.hpp index 5bc3b5359..8f00d435d 100644 --- a/src/storage/v2/durability/marker.hpp +++ b/src/storage/v2/durability/marker.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -56,6 +56,10 @@ enum class Marker : uint8_t { DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e, DELTA_UNIQUE_CONSTRAINT_CREATE = 0x5f, DELTA_UNIQUE_CONSTRAINT_DROP = 0x60, + DELTA_LABEL_INDEX_STATS_SET = 0x61, + DELTA_LABEL_INDEX_STATS_CLEAR = 0x62, + DELTA_LABEL_PROPERTY_INDEX_STATS_SET = 0x63, + DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR = 0x64, VALUE_FALSE = 0x00, VALUE_TRUE = 0xff, @@ -93,6 +97,10 @@ static const Marker kMarkersAll[] = { Marker::DELTA_TRANSACTION_END, Marker::DELTA_LABEL_INDEX_CREATE, Marker::DELTA_LABEL_INDEX_DROP, + Marker::DELTA_LABEL_INDEX_STATS_SET, + Marker::DELTA_LABEL_INDEX_STATS_CLEAR, + Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET, + Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR, Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE, Marker::DELTA_LABEL_PROPERTY_INDEX_DROP, Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE, diff --git a/src/storage/v2/durability/metadata.hpp b/src/storage/v2/durability/metadata.hpp index 3c66f7ec8..1045d4f97 100644 --- a/src/storage/v2/durability/metadata.hpp +++ b/src/storage/v2/durability/metadata.hpp @@ -19,6 +19,8 @@ #include "storage/v2/durability/exceptions.hpp" #include "storage/v2/id_types.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" namespace memgraph::storage::durability { @@ -39,6 +41,8 @@ struct RecoveredIndicesAndConstraints { struct { std::vector label; std::vector> label_property; + std::vector> label_stats; + std::vector>> label_property_stats; } indices; struct { @@ -74,4 +78,19 @@ void RemoveRecoveredIndexConstraint(std::vector *list, TObj obj, const cha } } +// Helper function used to remove indices stats from the recovered +// indices/constraints object. +// @note multiple stats can be pushed one after the other; when removing, remove from the back +// @throw RecoveryFailure +template +void RemoveRecoveredIndexStats(std::vector *list, K label, const char *error_message) { + for (auto it = list->rbegin(); it != list->rend(); ++it) { + if (it->first == label) { + list->erase(std::next(it).base()); // erase using a reverse iterator + return; + } + } + throw RecoveryFailure(error_message); +} + } // namespace memgraph::storage::durability diff --git a/src/storage/v2/durability/paths.hpp b/src/storage/v2/durability/paths.hpp index 557e4daa9..7e3083937 100644 --- a/src/storage/v2/durability/paths.hpp +++ b/src/storage/v2/durability/paths.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -22,7 +22,6 @@ static const std::string kSnapshotDirectory{"snapshots"}; static const std::string kWalDirectory{"wal"}; static const std::string kBackupDirectory{".backup"}; static const std::string kLockFile{".lock"}; -static const std::string kReplicationDirectory{"replication"}; // This is the prefix used for Snapshot and WAL filenames. It is a timestamp // format that equals to: YYYYmmddHHMMSSffffff diff --git a/src/storage/v2/durability/serialization.cpp b/src/storage/v2/durability/serialization.cpp index 61c390da6..6b13d9d00 100644 --- a/src/storage/v2/durability/serialization.cpp +++ b/src/storage/v2/durability/serialization.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -344,6 +344,10 @@ std::optional Decoder::ReadPropertyValue() { case Marker::DELTA_TRANSACTION_END: case Marker::DELTA_LABEL_INDEX_CREATE: case Marker::DELTA_LABEL_INDEX_DROP: + case Marker::DELTA_LABEL_INDEX_STATS_SET: + case Marker::DELTA_LABEL_INDEX_STATS_CLEAR: + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET: + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR: case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: @@ -443,6 +447,10 @@ bool Decoder::SkipPropertyValue() { case Marker::DELTA_TRANSACTION_END: case Marker::DELTA_LABEL_INDEX_CREATE: case Marker::DELTA_LABEL_INDEX_DROP: + case Marker::DELTA_LABEL_INDEX_STATS_SET: + case Marker::DELTA_LABEL_INDEX_STATS_CLEAR: + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET: + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR: case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index 38018fcd5..6e173fb06 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -23,6 +23,10 @@ #include "storage/v2/edge_accessor.hpp" #include "storage/v2/edge_ref.hpp" #include "storage/v2/id_types.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" +#include "storage/v2/inmemory/label_index.hpp" +#include "storage/v2/inmemory/label_property_index.hpp" #include "storage/v2/mvcc.hpp" #include "storage/v2/vertex.hpp" #include "storage/v2/vertex_accessor.hpp" @@ -1077,10 +1081,11 @@ RecoveredSnapshot LoadSnapshotVersion14(const std::filesystem::path &path, utils return {info, ret, std::move(indices_constraints)}; } -RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList *vertices, - utils::SkipList *edges, - std::deque> *epoch_history, - NameIdMapper *name_id_mapper, std::atomic *edge_count, const Config &config) { +RecoveredSnapshot LoadSnapshotVersion15(const std::filesystem::path &path, utils::SkipList *vertices, + utils::SkipList *edges, + std::deque> *epoch_history, + NameIdMapper *name_id_mapper, std::atomic *edge_count, + const Config &config) { RecoveryInfo recovery_info; RecoveredIndicesAndConstraints indices_constraints; @@ -1089,9 +1094,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!"); if (!IsVersionSupported(*version)) throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version)); - if (*version == 14U) { - return LoadSnapshotVersion14(path, vertices, edges, epoch_history, name_id_mapper, edge_count, config.items); - } + if (*version != 15U) throw RecoveryFailure(fmt::format("Expected snapshot version is 15, but got {}", *version)); // Cleanup of loaded data in case of failure. bool success = false; @@ -1364,11 +1367,355 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis return {info, recovery_info, std::move(indices_constraints)}; } +RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList *vertices, + utils::SkipList *edges, + std::deque> *epoch_history, + NameIdMapper *name_id_mapper, std::atomic *edge_count, const Config &config) { + RecoveryInfo recovery_info; + RecoveredIndicesAndConstraints indices_constraints; + + Decoder snapshot; + const auto version = snapshot.Initialize(path, kSnapshotMagic); + if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!"); + + if (!IsVersionSupported(*version)) throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version)); + if (*version == 14U) { + return LoadSnapshotVersion14(path, vertices, edges, epoch_history, name_id_mapper, edge_count, config.items); + } + if (*version == 15U) { + return LoadSnapshotVersion15(path, vertices, edges, epoch_history, name_id_mapper, edge_count, config); + } + + // Cleanup of loaded data in case of failure. + bool success = false; + utils::OnScopeExit cleanup([&] { + if (!success) { + edges->clear(); + vertices->clear(); + epoch_history->clear(); + } + }); + + // Read snapshot info. + const auto info = ReadSnapshotInfo(path); + spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count); + // Check for edges. + bool snapshot_has_edges = info.offset_edges != 0; + + // Recover mapper. + std::unordered_map snapshot_id_map; + { + spdlog::info("Recovering mapper metadata."); + if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!"); + + auto marker = snapshot.ReadMarker(); + if (!marker || *marker != Marker::SECTION_MAPPER) throw RecoveryFailure("Invalid snapshot data!"); + + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + + for (uint64_t i = 0; i < *size; ++i) { + auto id = snapshot.ReadUint(); + if (!id) throw RecoveryFailure("Invalid snapshot data!"); + auto name = snapshot.ReadString(); + if (!name) throw RecoveryFailure("Invalid snapshot data!"); + auto my_id = name_id_mapper->NameToId(*name); + snapshot_id_map.emplace(*id, my_id); + SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id); + } + } + auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) { + auto it = snapshot_id_map.find(snapshot_id); + if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); + return LabelId::FromUint(it->second); + }; + auto get_property_from_id = [&snapshot_id_map](uint64_t snapshot_id) { + auto it = snapshot_id_map.find(snapshot_id); + if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); + return PropertyId::FromUint(it->second); + }; + auto get_edge_type_from_id = [&snapshot_id_map](uint64_t snapshot_id) { + auto it = snapshot_id_map.find(snapshot_id); + if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!"); + return EdgeTypeId::FromUint(it->second); + }; + + // Reset current edge count. + edge_count->store(0, std::memory_order_release); + + { + spdlog::info("Recovering edges."); + // Recover edges. + if (snapshot_has_edges) { + // We don't need to check whether we store properties on edge or not, because `LoadPartialEdges` will always + // iterate over the edges in the snapshot (if they exist) and the current configuration of properties on edge only + // affect what it does: + // 1. If properties are allowed on edges, then it loads the edges. + // 2. If properties are not allowed on edges, then it checks that none of the edges have any properties. + if (!snapshot.SetPosition(info.offset_edge_batches)) { + throw RecoveryFailure("Couldn't read data from snapshot!"); + } + const auto edge_batches = ReadBatchInfos(snapshot); + + RecoverOnMultipleThreads( + config.durability.recovery_thread_count, + [path, edges, items = config.items, &get_property_from_id](const size_t /*batch_index*/, + const BatchInfo &batch) { + LoadPartialEdges(path, *edges, batch.offset, batch.count, items, get_property_from_id); + }, + edge_batches); + } + spdlog::info("Edges are recovered."); + + // Recover vertices (labels and properties). + spdlog::info("Recovering vertices.", info.vertices_count); + uint64_t last_vertex_gid{0}; + + if (!snapshot.SetPosition(info.offset_vertex_batches)) { + throw RecoveryFailure("Couldn't read data from snapshot!"); + } + + const auto vertex_batches = ReadBatchInfos(snapshot); + RecoverOnMultipleThreads( + config.durability.recovery_thread_count, + [path, vertices, &vertex_batches, &get_label_from_id, &get_property_from_id, &last_vertex_gid]( + const size_t batch_index, const BatchInfo &batch) { + const auto last_vertex_gid_in_batch = + LoadPartialVertices(path, *vertices, batch.offset, batch.count, get_label_from_id, get_property_from_id); + if (batch_index == vertex_batches.size() - 1) { + last_vertex_gid = last_vertex_gid_in_batch; + } + }, + vertex_batches); + + spdlog::info("Vertices are recovered."); + + // Recover vertices (in/out edges). + spdlog::info("Recover connectivity."); + recovery_info.vertex_batches.reserve(vertex_batches.size()); + for (const auto batch : vertex_batches) { + recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count)); + } + std::atomic highest_edge_gid{0}; + + RecoverOnMultipleThreads( + config.durability.recovery_thread_count, + [path, vertices, edges, edge_count, items = config.items, snapshot_has_edges, &get_edge_type_from_id, + &highest_edge_gid, &recovery_info](const size_t batch_index, const BatchInfo &batch) { + const auto result = LoadPartialConnectivity(path, *vertices, *edges, batch.offset, batch.count, items, + snapshot_has_edges, get_edge_type_from_id); + edge_count->fetch_add(result.edge_count); + auto known_highest_edge_gid = highest_edge_gid.load(); + while (known_highest_edge_gid < result.highest_edge_id) { + highest_edge_gid.compare_exchange_weak(known_highest_edge_gid, result.highest_edge_id); + } + recovery_info.vertex_batches[batch_index].first = result.first_vertex_gid; + }, + vertex_batches); + + spdlog::info("Connectivity is recovered."); + + // Set initial values for edge/vertex ID generators. + recovery_info.next_edge_id = highest_edge_gid + 1; + recovery_info.next_vertex_id = last_vertex_gid + 1; + } + + // Recover indices. + { + spdlog::info("Recovering metadata of indices."); + if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!"); + + auto marker = snapshot.ReadMarker(); + if (!marker || *marker != Marker::SECTION_INDICES) throw RecoveryFailure("Invalid snapshot data!"); + + // Recover label indices. + { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} label indices.", *size); + for (uint64_t i = 0; i < *size; ++i) { + const auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label), + "The label index already exists!"); + SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label))); + } + spdlog::info("Metadata of label indices are recovered."); + } + + // Recover label indices statistics. + { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} label indices statistics.", *size); + for (uint64_t i = 0; i < *size; ++i) { + const auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + const auto count = snapshot.ReadUint(); + if (!count) throw RecoveryFailure("Invalid snapshot data!"); + const auto avg_degree = snapshot.ReadDouble(); + if (!avg_degree) throw RecoveryFailure("Invalid snapshot data!"); + const auto label_id = get_label_from_id(*label); + indices_constraints.indices.label_stats.emplace_back(label_id, LabelIndexStats{*count, *avg_degree}); + SPDLOG_TRACE("Recovered metadata of label index statistics for :{}", + name_id_mapper->IdToName(snapshot_id_map.at(*label))); + } + spdlog::info("Metadata of label indices are recovered."); + } + + // Recover label+property indices. + { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} label+property indices.", *size); + for (uint64_t i = 0; i < *size; ++i) { + const auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + const auto property = snapshot.ReadUint(); + if (!property) throw RecoveryFailure("Invalid snapshot data!"); + AddRecoveredIndexConstraint(&indices_constraints.indices.label_property, + {get_label_from_id(*label), get_property_from_id(*property)}, + "The label+property index already exists!"); + SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})", + name_id_mapper->IdToName(snapshot_id_map.at(*label)), + name_id_mapper->IdToName(snapshot_id_map.at(*property))); + } + spdlog::info("Metadata of label+property indices are recovered."); + } + + // Recover label+property indices statistics. + { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} label+property indices statistics.", *size); + for (uint64_t i = 0; i < *size; ++i) { + const auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + const auto property = snapshot.ReadUint(); + if (!property) throw RecoveryFailure("Invalid snapshot data!"); + const auto count = snapshot.ReadUint(); + if (!count) throw RecoveryFailure("Invalid snapshot data!"); + const auto distinct_values_count = snapshot.ReadUint(); + if (!distinct_values_count) throw RecoveryFailure("Invalid snapshot data!"); + const auto statistic = snapshot.ReadDouble(); + if (!statistic) throw RecoveryFailure("Invalid snapshot data!"); + const auto avg_group_size = snapshot.ReadDouble(); + if (!avg_group_size) throw RecoveryFailure("Invalid snapshot data!"); + const auto avg_degree = snapshot.ReadDouble(); + if (!avg_degree) throw RecoveryFailure("Invalid snapshot data!"); + const auto label_id = get_label_from_id(*label); + const auto property_id = get_property_from_id(*property); + indices_constraints.indices.label_property_stats.emplace_back( + label_id, std::make_pair(property_id, LabelPropertyIndexStats{*count, *distinct_values_count, *statistic, + *avg_group_size, *avg_degree})); + SPDLOG_TRACE("Recovered metadata of label+property index statistics for :{}({})", + name_id_mapper->IdToName(snapshot_id_map.at(*label)), + name_id_mapper->IdToName(snapshot_id_map.at(*property))); + } + spdlog::info("Metadata of label+property indices are recovered."); + } + + spdlog::info("Metadata of indices are recovered."); + } + + // Recover constraints. + { + spdlog::info("Recovering metadata of constraints."); + if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!"); + + auto marker = snapshot.ReadMarker(); + if (!marker || *marker != Marker::SECTION_CONSTRAINTS) throw RecoveryFailure("Invalid snapshot data!"); + + // Recover existence constraints. + { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} existence constraints.", *size); + for (uint64_t i = 0; i < *size; ++i) { + auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + auto property = snapshot.ReadUint(); + if (!property) throw RecoveryFailure("Invalid snapshot data!"); + AddRecoveredIndexConstraint(&indices_constraints.constraints.existence, + {get_label_from_id(*label), get_property_from_id(*property)}, + "The existence constraint already exists!"); + SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})", + name_id_mapper->IdToName(snapshot_id_map.at(*label)), + name_id_mapper->IdToName(snapshot_id_map.at(*property))); + } + spdlog::info("Metadata of existence constraints are recovered."); + } + + // Recover unique constraints. + // Snapshot version should be checked since unique constraints were + // implemented in later versions of snapshot. + if (*version >= kUniqueConstraintVersion) { + auto size = snapshot.ReadUint(); + if (!size) throw RecoveryFailure("Invalid snapshot data!"); + spdlog::info("Recovering metadata of {} unique constraints.", *size); + for (uint64_t i = 0; i < *size; ++i) { + auto label = snapshot.ReadUint(); + if (!label) throw RecoveryFailure("Invalid snapshot data!"); + auto properties_count = snapshot.ReadUint(); + if (!properties_count) throw RecoveryFailure("Invalid snapshot data!"); + std::set properties; + for (uint64_t j = 0; j < *properties_count; ++j) { + auto property = snapshot.ReadUint(); + if (!property) throw RecoveryFailure("Invalid snapshot data!"); + properties.insert(get_property_from_id(*property)); + } + AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties}, + "The unique constraint already exists!"); + SPDLOG_TRACE("Recovered metadata of unique constraints for :{}", + name_id_mapper->IdToName(snapshot_id_map.at(*label))); + } + spdlog::info("Metadata of unique constraints are recovered."); + } + spdlog::info("Metadata of constraints are recovered."); + } + + spdlog::info("Recovering metadata."); + // Recover epoch history + { + if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!"); + + const auto marker = snapshot.ReadMarker(); + if (!marker || *marker != Marker::SECTION_EPOCH_HISTORY) throw RecoveryFailure("Invalid snapshot data!"); + + const auto history_size = snapshot.ReadUint(); + if (!history_size) { + throw RecoveryFailure("Invalid snapshot data!"); + } + + for (int i = 0; i < *history_size; ++i) { + auto maybe_epoch_id = snapshot.ReadString(); + if (!maybe_epoch_id) { + throw RecoveryFailure("Invalid snapshot data!"); + } + const auto maybe_last_commit_timestamp = snapshot.ReadUint(); + if (!maybe_last_commit_timestamp) { + throw RecoveryFailure("Invalid snapshot data!"); + } + epoch_history->emplace_back(std::move(*maybe_epoch_id), *maybe_last_commit_timestamp); + } + } + + spdlog::info("Metadata recovered."); + // Recover timestamp. + recovery_info.next_timestamp = info.start_timestamp + 1; + + // Set success flag (to disable cleanup). + success = true; + + return {info, recovery_info, std::move(indices_constraints)}; +} + void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory, const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count, utils::SkipList *vertices, utils::SkipList *edges, NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, const Config &config, const std::string &uuid, - const std::string_view epoch_id, const std::deque> &epoch_history, + const memgraph::replication::ReplicationEpoch &epoch, + const std::deque> &epoch_history, utils::FileRetainer *file_retainer) { // Ensure that the storage directory exists. utils::EnsureDirOrDie(snapshot_directory); @@ -1577,6 +1924,31 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps } } + // Write label indices statistics. + { + // NOTE: On-disk does not support snapshots + auto *inmem_index = static_cast(indices->label_index_.get()); + auto label = inmem_index->ListIndices(); + const auto size_pos = snapshot.GetPosition(); + snapshot.WriteUint(0); // Just a place holder + unsigned i = 0; + for (const auto &item : label) { + auto stats = inmem_index->GetIndexStats(item); + if (stats) { + snapshot.WriteUint(item.AsUint()); + snapshot.WriteUint(stats->count); + snapshot.WriteDouble(stats->avg_degree); + ++i; + } + } + if (i != 0) { + const auto last_pos = snapshot.GetPosition(); + snapshot.SetPosition(size_pos); + snapshot.WriteUint(i); // Write real size + snapshot.SetPosition(last_pos); + } + } + // Write label+property indices. { auto label_property = indices->label_property_index_->ListIndices(); @@ -1586,6 +1958,35 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps write_mapping(item.second); } } + + // Write label+property indices statistics. + { + // NOTE: On-disk does not support snapshots + auto *inmem_index = static_cast(indices->label_property_index_.get()); + auto label = inmem_index->ListIndices(); + const auto size_pos = snapshot.GetPosition(); + snapshot.WriteUint(0); // Just a place holder + unsigned i = 0; + for (const auto &item : label) { + auto stats = inmem_index->GetIndexStats(item); + if (stats) { + snapshot.WriteUint(item.first.AsUint()); + snapshot.WriteUint(item.second.AsUint()); + snapshot.WriteUint(stats->count); + snapshot.WriteUint(stats->distinct_values_count); + snapshot.WriteDouble(stats->statistic); + snapshot.WriteDouble(stats->avg_group_size); + snapshot.WriteDouble(stats->avg_degree); + ++i; + } + } + if (i != 0) { + const auto last_pos = snapshot.GetPosition(); + snapshot.SetPosition(size_pos); + snapshot.WriteUint(i); // Write real size + snapshot.SetPosition(last_pos); + } + } } // Write constraints. @@ -1644,7 +2045,7 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps offset_metadata = snapshot.GetPosition(); snapshot.WriteMarker(Marker::SECTION_METADATA); snapshot.WriteString(uuid); - snapshot.WriteString(epoch_id); + snapshot.WriteString(epoch.id()); snapshot.WriteUint(transaction->start_timestamp); snapshot.WriteUint(edges_count); snapshot.WriteUint(vertices_count); diff --git a/src/storage/v2/durability/snapshot.hpp b/src/storage/v2/durability/snapshot.hpp index 2f16088a0..ac4fe5b1f 100644 --- a/src/storage/v2/durability/snapshot.hpp +++ b/src/storage/v2/durability/snapshot.hpp @@ -15,6 +15,7 @@ #include #include +#include "replication/epoch.hpp" #include "storage/v2/config.hpp" #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/durability/metadata.hpp" @@ -71,7 +72,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count, utils::SkipList *vertices, utils::SkipList *edges, NameIdMapper *name_id_mapper, Indices *indices, Constraints *constraints, const Config &config, const std::string &uuid, - std::string_view epoch_id, const std::deque> &epoch_history, + const memgraph::replication::ReplicationEpoch &epoch, + const std::deque> &epoch_history, utils::FileRetainer *file_retainer); } // namespace memgraph::storage::durability diff --git a/src/storage/v2/durability/storage_global_operation.hpp b/src/storage/v2/durability/storage_global_operation.hpp index f130b2a62..a4f1b043a 100644 --- a/src/storage/v2/durability/storage_global_operation.hpp +++ b/src/storage/v2/durability/storage_global_operation.hpp @@ -14,11 +14,15 @@ namespace memgraph::storage::durability { /// Enum used to indicate a global database operation that isn't transactional. -enum class StorageGlobalOperation { +enum class StorageMetadataOperation { LABEL_INDEX_CREATE, LABEL_INDEX_DROP, + LABEL_INDEX_STATS_SET, + LABEL_INDEX_STATS_CLEAR, LABEL_PROPERTY_INDEX_CREATE, LABEL_PROPERTY_INDEX_DROP, + LABEL_PROPERTY_INDEX_STATS_SET, + LABEL_PROPERTY_INDEX_STATS_CLEAR, EXISTENCE_CONSTRAINT_CREATE, EXISTENCE_CONSTRAINT_DROP, UNIQUE_CONSTRAINT_CREATE, diff --git a/src/storage/v2/durability/version.hpp b/src/storage/v2/durability/version.hpp index 55aeff552..25eb30904 100644 --- a/src/storage/v2/durability/version.hpp +++ b/src/storage/v2/durability/version.hpp @@ -20,7 +20,7 @@ namespace memgraph::storage::durability { // The current version of snapshot and WAL encoding / decoding. // IMPORTANT: Please bump this version for every snapshot and/or WAL format // change!!! -const uint64_t kVersion{15}; +const uint64_t kVersion{16}; const uint64_t kOldestSupportedVersion{14}; const uint64_t kUniqueConstraintVersion{13}; diff --git a/src/storage/v2/durability/wal.cpp b/src/storage/v2/durability/wal.cpp index ffecf1869..8c28a6b6d 100644 --- a/src/storage/v2/durability/wal.cpp +++ b/src/storage/v2/durability/wal.cpp @@ -13,9 +13,11 @@ #include "storage/v2/delta.hpp" #include "storage/v2/durability/exceptions.hpp" +#include "storage/v2/durability/metadata.hpp" #include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/version.hpp" #include "storage/v2/edge.hpp" +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/vertex.hpp" #include "utils/file_locker.hpp" #include "utils/logging.hpp" @@ -75,23 +77,31 @@ namespace memgraph::storage::durability { namespace { -Marker OperationToMarker(StorageGlobalOperation operation) { +Marker OperationToMarker(StorageMetadataOperation operation) { switch (operation) { - case StorageGlobalOperation::LABEL_INDEX_CREATE: + case StorageMetadataOperation::LABEL_INDEX_CREATE: return Marker::DELTA_LABEL_INDEX_CREATE; - case StorageGlobalOperation::LABEL_INDEX_DROP: + case StorageMetadataOperation::LABEL_INDEX_DROP: return Marker::DELTA_LABEL_INDEX_DROP; - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: + case StorageMetadataOperation::LABEL_INDEX_STATS_SET: + return Marker::DELTA_LABEL_INDEX_STATS_SET; + case StorageMetadataOperation::LABEL_INDEX_STATS_CLEAR: + return Marker::DELTA_LABEL_INDEX_STATS_CLEAR; + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE: return Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE; - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP: return Marker::DELTA_LABEL_PROPERTY_INDEX_DROP; - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET: + return Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET; + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR: + return Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR; + case StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE: return Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE; - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: + case StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP: return Marker::DELTA_EXISTENCE_CONSTRAINT_DROP; - case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: + case StorageMetadataOperation::UNIQUE_CONSTRAINT_CREATE: return Marker::DELTA_UNIQUE_CONSTRAINT_CREATE; - case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: + case StorageMetadataOperation::UNIQUE_CONSTRAINT_DROP: return Marker::DELTA_UNIQUE_CONSTRAINT_DROP; } } @@ -150,10 +160,18 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) { return WalDeltaData::Type::LABEL_INDEX_CREATE; case Marker::DELTA_LABEL_INDEX_DROP: return WalDeltaData::Type::LABEL_INDEX_DROP; + case Marker::DELTA_LABEL_INDEX_STATS_SET: + return WalDeltaData::Type::LABEL_INDEX_STATS_SET; + case Marker::DELTA_LABEL_INDEX_STATS_CLEAR: + return WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR; case Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: return WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE; case Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: return WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP; + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET: + return WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET; + case Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR: + return WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR; case Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: return WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE; case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: @@ -263,7 +281,11 @@ WalDeltaData ReadSkipWalDeltaData(BaseDecoder *decoder) { case WalDeltaData::Type::TRANSACTION_END: break; case WalDeltaData::Type::LABEL_INDEX_CREATE: - case WalDeltaData::Type::LABEL_INDEX_DROP: { + case WalDeltaData::Type::LABEL_INDEX_DROP: + case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: /* Special case, this clear is done on all label/property + pairs that contain the defined label */ + { if constexpr (read_data) { auto label = decoder->ReadString(); if (!label) throw RecoveryFailure("Invalid WAL data!"); @@ -273,6 +295,18 @@ WalDeltaData ReadSkipWalDeltaData(BaseDecoder *decoder) { } break; } + case WalDeltaData::Type::LABEL_INDEX_STATS_SET: { + if constexpr (read_data) { + auto label = decoder->ReadString(); + if (!label) throw RecoveryFailure("Invalid WAL data!"); + delta.operation_label_stats.label = std::move(*label); + auto stats = decoder->ReadString(); + if (!stats) throw RecoveryFailure("Invalid WAL data!"); + delta.operation_label_stats.stats = std::move(*stats); + } else { + if (!decoder->SkipString() || !decoder->SkipString()) throw RecoveryFailure("Invalid WAL data!"); + } + } break; case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: @@ -289,6 +323,23 @@ WalDeltaData ReadSkipWalDeltaData(BaseDecoder *decoder) { } break; } + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: { + if constexpr (read_data) { + auto label = decoder->ReadString(); + if (!label) throw RecoveryFailure("Invalid WAL data!"); + delta.operation_label_property_stats.label = std::move(*label); + auto property = decoder->ReadString(); + if (!property) throw RecoveryFailure("Invalid WAL data!"); + delta.operation_label_property_stats.property = std::move(*property); + auto stats = decoder->ReadString(); + if (!stats) throw RecoveryFailure("Invalid WAL data!"); + delta.operation_label_property_stats.stats = std::move(*stats); + } else { + if (!decoder->SkipString() || !decoder->SkipString() || !decoder->SkipString()) + throw RecoveryFailure("Invalid WAL data!"); + } + break; + } case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { if constexpr (read_data) { @@ -371,11 +422,11 @@ WalInfo ReadWalInfo(const std::filesystem::path &path) { // Read deltas. info.num_deltas = 0; - auto validate_delta = [&wal]() -> std::optional> { + auto validate_delta = [&wal, version = *version]() -> std::optional> { try { auto timestamp = ReadWalDeltaHeader(&wal); auto type = SkipWalDeltaData(&wal); - return {{timestamp, IsWalDeltaDataTypeTransactionEnd(type)}}; + return {{timestamp, IsWalDeltaDataTypeTransactionEnd(type, version)}}; } catch (const RecoveryFailure &) { return std::nullopt; } @@ -445,14 +496,28 @@ bool operator==(const WalDeltaData &a, const WalDeltaData &b) { case WalDeltaData::Type::LABEL_INDEX_CREATE: case WalDeltaData::Type::LABEL_INDEX_DROP: + case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: /* Special case, label property index stats clear just + passes the label and all label/property pairs with the + label get cleared */ return a.operation_label.label == b.operation_label.label; + case WalDeltaData::Type::LABEL_INDEX_STATS_SET: + return a.operation_label_stats.label == b.operation_label_stats.label && + a.operation_label_stats.stats == b.operation_label_stats.stats; + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: return a.operation_label_property.label == b.operation_label_property.label && a.operation_label_property.property == b.operation_label_property.property; + + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: + return a.operation_label_property_stats.label == b.operation_label_property_stats.label && + a.operation_label_property_stats.property == b.operation_label_property_stats.property && + a.operation_label_property_stats.stats == b.operation_label_property_stats.stats; + case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: return a.operation_label_properties.label == b.operation_label_properties.label && @@ -585,30 +650,50 @@ void EncodeTransactionEnd(BaseEncoder *encoder, uint64_t timestamp) { encoder->WriteMarker(Marker::DELTA_TRANSACTION_END); } -void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageGlobalOperation operation, - LabelId label, const std::set &properties, uint64_t timestamp) { +void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageMetadataOperation operation, + LabelId label, const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, uint64_t timestamp) { encoder->WriteMarker(Marker::SECTION_DELTA); encoder->WriteUint(timestamp); switch (operation) { - case StorageGlobalOperation::LABEL_INDEX_CREATE: - case StorageGlobalOperation::LABEL_INDEX_DROP: { + case StorageMetadataOperation::LABEL_INDEX_CREATE: + case StorageMetadataOperation::LABEL_INDEX_DROP: + case StorageMetadataOperation::LABEL_INDEX_STATS_CLEAR: + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR: /* Special case, this clear is done on all + label/property pairs that contain the defined + label */ + { MG_ASSERT(properties.empty(), "Invalid function call!"); encoder->WriteMarker(OperationToMarker(operation)); encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); break; } - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: - case StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: - case StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: { + case StorageMetadataOperation::LABEL_INDEX_STATS_SET: { + MG_ASSERT(properties.empty(), "Invalid function call!"); + encoder->WriteMarker(OperationToMarker(operation)); + encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); + encoder->WriteString(ToJson(stats)); + break; + } + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE: + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP: + case StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE: + case StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP: { MG_ASSERT(properties.size() == 1, "Invalid function call!"); encoder->WriteMarker(OperationToMarker(operation)); encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); encoder->WriteString(name_id_mapper->IdToName((*properties.begin()).AsUint())); break; } - case StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: - case StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: { + case StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET: { + encoder->WriteMarker(OperationToMarker(operation)); + encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); + encoder->WriteString(name_id_mapper->IdToName((*properties.begin()).AsUint())); + encoder->WriteString(ToJson(property_stats)); + break; + } + case StorageMetadataOperation::UNIQUE_CONSTRAINT_CREATE: + case StorageMetadataOperation::UNIQUE_CONSTRAINT_DROP: { MG_ASSERT(!properties.empty(), "Invalid function call!"); encoder->WriteMarker(OperationToMarker(operation)); encoder->WriteString(name_id_mapper->IdToName(label.AsUint())); @@ -802,6 +887,21 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst "The label index doesn't exist!"); break; } + case WalDeltaData::Type::LABEL_INDEX_STATS_SET: { + auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_stats.label)); + LabelIndexStats stats{}; + if (!FromJson(delta.operation_label_stats.stats, stats)) { + throw RecoveryFailure("Failed to read statistics!"); + } + indices_constraints->indices.label_stats.emplace_back(label_id, stats); + break; + } + case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: { + auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label)); + RemoveRecoveredIndexStats(&indices_constraints->indices.label_stats, label_id, + "The label index stats doesn't exist!"); + break; + } case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: { auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); @@ -816,6 +916,23 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst "The label property index doesn't exist!"); break; } + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: { + auto &info = delta.operation_label_property_stats; + auto label_id = LabelId::FromUint(name_id_mapper->NameToId(info.label)); + auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(info.property)); + LabelPropertyIndexStats stats{}; + if (!FromJson(info.stats, stats)) { + throw RecoveryFailure("Failed to read statistics!"); + } + indices_constraints->indices.label_property_stats.emplace_back(label_id, std::make_pair(property_id, stats)); + break; + } + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: { + auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label.label)); + RemoveRecoveredIndexStats(&indices_constraints->indices.label_property_stats, label_id, + "The label index stats doesn't exist!"); + break; + } case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: { auto label_id = LabelId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.label)); auto property_id = PropertyId::FromUint(name_id_mapper->NameToId(delta.operation_label_property.property)); @@ -964,9 +1081,10 @@ void WalFile::AppendTransactionEnd(uint64_t timestamp) { UpdateStats(timestamp); } -void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label, const std::set &properties, +void WalFile::AppendOperation(StorageMetadataOperation operation, LabelId label, const std::set &properties, + const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t timestamp) { - EncodeOperation(&wal_, name_id_mapper_, operation, label, properties, timestamp); + EncodeOperation(&wal_, name_id_mapper_, operation, label, properties, stats, property_stats, timestamp); UpdateStats(timestamp); } diff --git a/src/storage/v2/durability/wal.hpp b/src/storage/v2/durability/wal.hpp index 757ffc8a2..3f598cb99 100644 --- a/src/storage/v2/durability/wal.hpp +++ b/src/storage/v2/durability/wal.hpp @@ -21,8 +21,11 @@ #include "storage/v2/durability/metadata.hpp" #include "storage/v2/durability/serialization.hpp" #include "storage/v2/durability/storage_global_operation.hpp" +#include "storage/v2/durability/version.hpp" #include "storage/v2/edge.hpp" #include "storage/v2/id_types.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" #include "storage/v2/name_id_mapper.hpp" #include "storage/v2/property_value.hpp" #include "storage/v2/vertex.hpp" @@ -58,8 +61,12 @@ struct WalDeltaData { TRANSACTION_END, LABEL_INDEX_CREATE, LABEL_INDEX_DROP, + LABEL_INDEX_STATS_SET, + LABEL_INDEX_STATS_CLEAR, LABEL_PROPERTY_INDEX_CREATE, LABEL_PROPERTY_INDEX_DROP, + LABEL_PROPERTY_INDEX_STATS_SET, + LABEL_PROPERTY_INDEX_STATS_CLEAR, EXISTENCE_CONSTRAINT_CREATE, EXISTENCE_CONSTRAINT_DROP, UNIQUE_CONSTRAINT_CREATE, @@ -103,12 +110,23 @@ struct WalDeltaData { std::string label; std::set properties; } operation_label_properties; + + struct { + std::string label; + std::string stats; + } operation_label_stats; + + struct { + std::string label; + std::string property; + std::string stats; + } operation_label_property_stats; }; bool operator==(const WalDeltaData &a, const WalDeltaData &b); bool operator!=(const WalDeltaData &a, const WalDeltaData &b); -constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) { +constexpr bool IsWalDeltaDataTypeTransactionEndVersion15(const WalDeltaData::Type type) { switch (type) { // These delta actions are all found inside transactions so they don't // indicate a transaction end. @@ -131,16 +149,28 @@ constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) { // 'transaction'. case WalDeltaData::Type::LABEL_INDEX_CREATE: case WalDeltaData::Type::LABEL_INDEX_DROP: + case WalDeltaData::Type::LABEL_INDEX_STATS_SET: + case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: + case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: - return true; + return true; // TODO: Still true? } } +constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type, const uint64_t version = kVersion) { + if (version < 16U) { + return IsWalDeltaDataTypeTransactionEndVersion15(type); + } + // All deltas are now handled in a transactional scope + return type == WalDeltaData::Type::TRANSACTION_END; +} + /// Function used to read information about the WAL file. /// @throw RecoveryFailure WalInfo ReadWalInfo(const std::filesystem::path &path); @@ -174,8 +204,9 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta void EncodeTransactionEnd(BaseEncoder *encoder, uint64_t timestamp); /// Function used to encode non-transactional operation. -void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageGlobalOperation operation, - LabelId label, const std::set &properties, uint64_t timestamp); +void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper, StorageMetadataOperation operation, + LabelId label, const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, uint64_t timestamp); /// Function used to load the WAL data into the storage. /// @throw RecoveryFailure @@ -204,8 +235,8 @@ class WalFile { void AppendTransactionEnd(uint64_t timestamp); - void AppendOperation(StorageGlobalOperation operation, LabelId label, const std::set &properties, - uint64_t timestamp); + void AppendOperation(StorageMetadataOperation operation, LabelId label, const std::set &properties, + const LabelIndexStats &stats, const LabelPropertyIndexStats &property_stats, uint64_t timestamp); void Sync(); diff --git a/src/storage/v2/indices/label_index_stats.hpp b/src/storage/v2/indices/label_index_stats.hpp new file mode 100644 index 000000000..4321d096d --- /dev/null +++ b/src/storage/v2/indices/label_index_stats.hpp @@ -0,0 +1,35 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include "utils/simple_json.hpp" + +namespace memgraph::storage { + +struct LabelIndexStats { + uint64_t count; + double avg_degree; +}; + +static inline std::string ToJson(const LabelIndexStats &in) { + return fmt::format(R"({{"count":{}, "avg_degree":{}}})", in.count, in.avg_degree); +} + +static inline bool FromJson(const std::string &json, LabelIndexStats &out) { + bool res = true; + res &= utils::GetJsonValue(json, "count", out.count); + res &= utils::GetJsonValue(json, "avg_degree", out.avg_degree); + return res; +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/indices/label_property_index_stats.hpp b/src/storage/v2/indices/label_property_index_stats.hpp new file mode 100644 index 000000000..c1ca14b6d --- /dev/null +++ b/src/storage/v2/indices/label_property_index_stats.hpp @@ -0,0 +1,40 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include "utils/simple_json.hpp" + +namespace memgraph::storage { + +struct LabelPropertyIndexStats { + uint64_t count, distinct_values_count; + double statistic, avg_group_size, avg_degree; +}; + +static inline std::string ToJson(const LabelPropertyIndexStats &in) { + return fmt::format( + R"({{"count":{}, "distinct_values_count":{}, "statistic":{}, "avg_group_size":{} "avg_degree":{}}})", in.count, + in.distinct_values_count, in.statistic, in.avg_group_size, in.avg_degree); +} + +static inline bool FromJson(const std::string &json, LabelPropertyIndexStats &out) { + bool res = true; + res &= utils::GetJsonValue(json, "count", out.count); + res &= utils::GetJsonValue(json, "distinct_values_count", out.distinct_values_count); + res &= utils::GetJsonValue(json, "statistic", out.statistic); + res &= utils::GetJsonValue(json, "avg_group_size", out.avg_group_size); + res &= utils::GetJsonValue(json, "avg_degree", out.avg_degree); + return res; +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/label_index.cpp b/src/storage/v2/inmemory/label_index.cpp index 93e051aa0..374ea243b 100644 --- a/src/storage/v2/inmemory/label_index.cpp +++ b/src/storage/v2/inmemory/label_index.cpp @@ -159,11 +159,13 @@ InMemoryLabelIndex::Iterable InMemoryLabelIndex::Vertices(LabelId label, View vi } void InMemoryLabelIndex::SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats) { - stats_[label] = stats; + auto locked_stats = stats_.Lock(); + locked_stats->insert_or_assign(label, stats); } std::optional InMemoryLabelIndex::GetIndexStats(const storage::LabelId &label) const { - if (auto it = stats_.find(label); it != stats_.end()) { + auto locked_stats = stats_.ReadLock(); + if (auto it = locked_stats->find(label); it != locked_stats->end()) { return it->second; } return {}; @@ -171,25 +173,24 @@ std::optional InMemoryLabelIndex::GetIndexStats(const storage:: std::vector InMemoryLabelIndex::ClearIndexStats() { std::vector deleted_indexes; - deleted_indexes.reserve(stats_.size()); - std::transform(stats_.begin(), stats_.end(), std::back_inserter(deleted_indexes), + auto locked_stats = stats_.Lock(); + deleted_indexes.reserve(locked_stats->size()); + std::transform(locked_stats->begin(), locked_stats->end(), std::back_inserter(deleted_indexes), [](const auto &elem) { return elem.first; }); - stats_.clear(); + locked_stats->clear(); return deleted_indexes; } -std::vector InMemoryLabelIndex::DeleteIndexStats(const storage::LabelId &label) { - std::vector deleted_indexes; - for (auto it = stats_.cbegin(); it != stats_.cend();) { +// stats_ is a map with label as the key, so only one can exist at a time +bool InMemoryLabelIndex::DeleteIndexStats(const storage::LabelId &label) { + auto locked_stats = stats_.Lock(); + for (auto it = locked_stats->cbegin(); it != locked_stats->cend(); ++it) { if (it->first == label) { - deleted_indexes.push_back(it->first); - it = stats_.erase(it); - } else { - ++it; + locked_stats->erase(it); + return true; } } - - return deleted_indexes; + return false; } } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/label_index.hpp b/src/storage/v2/inmemory/label_index.hpp index 1658050c7..fc82a486f 100644 --- a/src/storage/v2/inmemory/label_index.hpp +++ b/src/storage/v2/inmemory/label_index.hpp @@ -13,15 +13,12 @@ #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/indices/label_index.hpp" +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/vertex.hpp" +#include "utils/rw_lock.hpp" namespace memgraph::storage { -struct LabelIndexStats { - uint64_t count; - double avg_degree; -}; - using ParallelizedIndexCreationInfo = std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; @@ -108,11 +105,11 @@ class InMemoryLabelIndex : public storage::LabelIndex { std::vector ClearIndexStats(); - std::vector DeleteIndexStats(const storage::LabelId &label); + bool DeleteIndexStats(const storage::LabelId &label); private: std::map> index_; - std::map stats_; + utils::Synchronized, utils::ReadPrioritizedRWLock> stats_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/label_property_index.cpp b/src/storage/v2/inmemory/label_property_index.cpp index c3ab165fd..505b3bce1 100644 --- a/src/storage/v2/inmemory/label_property_index.cpp +++ b/src/storage/v2/inmemory/label_property_index.cpp @@ -70,7 +70,7 @@ bool InMemoryLabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple()); - indices_by_property_[property].insert({label, &index_.at({label, property})}); + indices_by_property_[property].insert({label, &it->second}); if (!emplaced) { // Index already exists. @@ -386,20 +386,23 @@ uint64_t InMemoryLabelPropertyIndex::ApproximateVertexCount( std::vector> InMemoryLabelPropertyIndex::ClearIndexStats() { std::vector> deleted_indexes; - deleted_indexes.reserve(stats_.size()); - std::transform(stats_.begin(), stats_.end(), std::back_inserter(deleted_indexes), + auto locked_stats = stats_.Lock(); + deleted_indexes.reserve(locked_stats->size()); + std::transform(locked_stats->begin(), locked_stats->end(), std::back_inserter(deleted_indexes), [](const auto &elem) { return elem.first; }); - stats_.clear(); + locked_stats->clear(); return deleted_indexes; } +// stats_ is a map where the key is a pair of label and property, so for one label many pairs can be deleted std::vector> InMemoryLabelPropertyIndex::DeleteIndexStats( const storage::LabelId &label) { std::vector> deleted_indexes; - for (auto it = stats_.cbegin(); it != stats_.cend();) { + auto locked_stats = stats_.Lock(); + for (auto it = locked_stats->cbegin(); it != locked_stats->cend();) { if (it->first.first == label) { deleted_indexes.push_back(it->first); - it = stats_.erase(it); + it = locked_stats->erase(it); } else { ++it; } @@ -409,12 +412,14 @@ std::vector> InMemoryLabelPropertyIndex::DeleteIn void InMemoryLabelPropertyIndex::SetIndexStats(const std::pair &key, const LabelPropertyIndexStats &stats) { - stats_[key] = stats; + auto locked_stats = stats_.Lock(); + locked_stats->insert_or_assign(key, stats); } std::optional InMemoryLabelPropertyIndex::GetIndexStats( const std::pair &key) const { - if (auto it = stats_.find(key); it != stats_.end()) { + auto locked_stats = stats_.ReadLock(); + if (auto it = locked_stats->find(key); it != locked_stats->end()) { return it->second; } return {}; diff --git a/src/storage/v2/inmemory/label_property_index.hpp b/src/storage/v2/inmemory/label_property_index.hpp index 00e2c90a5..517733e5a 100644 --- a/src/storage/v2/inmemory/label_property_index.hpp +++ b/src/storage/v2/inmemory/label_property_index.hpp @@ -13,14 +13,11 @@ #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/indices/label_property_index.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" +#include "utils/rw_lock.hpp" namespace memgraph::storage { -struct LabelPropertyIndexStats { - uint64_t count, distinct_values_count; - double statistic, avg_group_size, avg_degree; -}; - /// TODO: andi. Too many copies, extract at one place using ParallelizedIndexCreationInfo = std::pair> /*vertex_recovery_info*/, uint64_t /*thread_count*/>; @@ -138,7 +135,9 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex { private: std::map, utils::SkipList> index_; std::unordered_map *>> indices_by_property_; - std::map, storage::LabelPropertyIndexStats> stats_; + utils::Synchronized, storage::LabelPropertyIndexStats>, + utils::ReadPrioritizedRWLock> + stats_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.cpp b/src/storage/v2/inmemory/replication/replication_client.cpp index bfd272a9b..48aeafd7d 100644 --- a/src/storage/v2/inmemory/replication/replication_client.cpp +++ b/src/storage/v2/inmemory/replication/replication_client.cpp @@ -103,8 +103,9 @@ uint64_t ReplicateCurrentWal(CurrentWalHandler &stream, durability::WalFile cons ////// ReplicationClient ////// InMemoryReplicationClient::InMemoryReplicationClient(InMemoryStorage *storage, - const replication::ReplicationClientConfig &config) - : ReplicationClient{storage, config} {} + const memgraph::replication::ReplicationClientConfig &config, + const memgraph::replication::ReplicationEpoch *epoch) + : ReplicationClient{storage, config, epoch} {} void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { spdlog::debug("Starting replica recover"); diff --git a/src/storage/v2/inmemory/replication/replication_client.hpp b/src/storage/v2/inmemory/replication/replication_client.hpp index 94f47cb48..e956838e7 100644 --- a/src/storage/v2/inmemory/replication/replication_client.hpp +++ b/src/storage/v2/inmemory/replication/replication_client.hpp @@ -18,7 +18,8 @@ class InMemoryStorage; class InMemoryReplicationClient : public ReplicationClient { public: - InMemoryReplicationClient(InMemoryStorage *storage, const replication::ReplicationClientConfig &config); + InMemoryReplicationClient(InMemoryStorage *storage, const memgraph::replication::ReplicationClientConfig &config, + const memgraph::replication::ReplicationEpoch *epoch); protected: void RecoverReplica(uint64_t replica_commit) override; diff --git a/src/storage/v2/inmemory/replication/replication_server.cpp b/src/storage/v2/inmemory/replication/replication_server.cpp index 0104ed7bc..0f7669d5a 100644 --- a/src/storage/v2/inmemory/replication/replication_server.cpp +++ b/src/storage/v2/inmemory/replication/replication_server.cpp @@ -13,6 +13,7 @@ #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/version.hpp" +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" @@ -33,8 +34,9 @@ std::pair ReadDelta(durability::BaseDecoder } // namespace InMemoryReplicationServer::InMemoryReplicationServer(InMemoryStorage *storage, - const replication::ReplicationServerConfig &config) - : ReplicationServer{config}, storage_(storage) { + const memgraph::replication::ReplicationServerConfig &config, + memgraph::replication::ReplicationEpoch *repl_epoch) + : ReplicationServer{config}, storage_(storage), repl_epoch_{repl_epoch} { rpc_server_.Register([this](auto *req_reader, auto *res_builder) { spdlog::debug("Received HeartbeatRpc"); this->HeartbeatHandler(req_reader, res_builder); @@ -65,8 +67,8 @@ InMemoryReplicationServer::InMemoryReplicationServer(InMemoryStorage *storage, void InMemoryReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { replication::HeartbeatReq req; slk::Load(&req, req_reader); - replication::HeartbeatRes res{true, storage_->replication_state_.last_commit_timestamp_.load(), - storage_->replication_state_.GetEpoch().id}; + replication::HeartbeatRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load(), + std::string{repl_epoch_->id()}}; slk::Save(res, res_builder); } @@ -79,13 +81,14 @@ void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk auto maybe_epoch_id = decoder.ReadString(); MG_ASSERT(maybe_epoch_id, "Invalid replication message"); - if (*maybe_epoch_id != storage_->replication_state_.GetEpoch().id) { - storage_->replication_state_.AppendEpoch(*maybe_epoch_id); + auto &repl_storage_state = storage_->repl_storage_state_; + if (*maybe_epoch_id != repl_epoch_->id()) { + auto prev_epoch = repl_epoch_->SetEpoch(*maybe_epoch_id); + repl_storage_state.AddEpochToHistoryForce(prev_epoch); } if (storage_->wal_file_) { - if (req.seq_num > storage_->wal_file_->SequenceNumber() || - *maybe_epoch_id != storage_->replication_state_.GetEpoch().id) { + if (req.seq_num > storage_->wal_file_->SequenceNumber() || *maybe_epoch_id != repl_epoch_->id()) { storage_->wal_file_->FinalizeWal(); storage_->wal_file_.reset(); storage_->wal_seq_num_ = req.seq_num; @@ -98,23 +101,25 @@ void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk storage_->wal_seq_num_ = req.seq_num; } - if (req.previous_commit_timestamp != storage_->replication_state_.last_commit_timestamp_.load()) { + if (req.previous_commit_timestamp != repl_storage_state.last_commit_timestamp_.load()) { // Empty the stream bool transaction_complete = false; while (!transaction_complete) { SPDLOG_INFO("Skipping delta"); const auto [timestamp, delta] = ReadDelta(&decoder); - transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type); + transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd( + delta.type, durability::kVersion); // TODO: Check if we are always using the latest version when replicating } - replication::AppendDeltasRes res{false, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::AppendDeltasRes res{false, repl_storage_state.last_commit_timestamp_.load()}; slk::Save(res, res_builder); return; } - ReadAndApplyDelta(storage_, &decoder); + ReadAndApplyDelta(storage_, &decoder, + durability::kVersion); // TODO: Check if we are always using the latest version when replicating - replication::AppendDeltasRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::AppendDeltasRes res{true, repl_storage_state.last_commit_timestamp_.load()}; slk::Save(res, res_builder); spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!"); } @@ -144,17 +149,14 @@ void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Bu std::make_unique(&storage_->indices_, storage_->config_); try { spdlog::debug("Loading snapshot"); - auto &epoch = - storage_->replication_state_ - .GetEpoch(); // This needs to be a non-const ref since we are updating it in LoadSnapshot TODO fix auto recovered_snapshot = durability::LoadSnapshot( - *maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->replication_state_.history, + *maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->repl_storage_state_.history, storage_->name_id_mapper_.get(), &storage_->edge_count_, storage_->config_); spdlog::debug("Snapshot loaded successfully"); // If this step is present it should always be the first step of // the recovery so we use the UUID we read from snasphost storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid); - epoch.id = std::move(recovered_snapshot.snapshot_info.epoch_id); + repl_epoch_->SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id)); const auto &recovery_info = recovered_snapshot.recovery_info; storage_->vertex_id_ = recovery_info.next_vertex_id; storage_->edge_id_ = recovery_info.next_edge_id; @@ -168,7 +170,7 @@ void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Bu } storage_guard.unlock(); - replication::SnapshotRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::SnapshotRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()}; slk::Save(res, res_builder); spdlog::trace("Deleting old snapshot files due to snapshot recovery."); @@ -206,10 +208,10 @@ void InMemoryReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Bu utils::EnsureDirOrDie(storage_->wal_directory_); for (auto i = 0; i < wal_file_number; ++i) { - LoadWal(storage_, &decoder); + LoadWal(storage_, *repl_epoch_, &decoder); } - replication::WalFilesRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::WalFilesRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()}; slk::Save(res, res_builder); spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!"); } @@ -222,14 +224,15 @@ void InMemoryReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk:: utils::EnsureDirOrDie(storage_->wal_directory_); - LoadWal(storage_, &decoder); + LoadWal(storage_, *repl_epoch_, &decoder); - replication::CurrentWalRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::CurrentWalRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()}; slk::Save(res, res_builder); spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!"); } -void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::Decoder *decoder) { +void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::replication::ReplicationEpoch &epoch, + replication::Decoder *decoder) { const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory; utils::EnsureDir(temp_wal_directory); auto maybe_wal_path = decoder->ReadFile(temp_wal_directory); @@ -241,8 +244,9 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::D storage->uuid_ = wal_info.uuid; } - if (wal_info.epoch_id != storage->replication_state_.GetEpoch().id) { - storage->replication_state_.AppendEpoch(wal_info.epoch_id); + if (wal_info.epoch_id != epoch.id()) { + auto prev_epoch = epoch.SetEpoch(wal_info.epoch_id); + storage->repl_storage_state_.AddEpochToHistoryForce(prev_epoch); } if (storage->wal_file_) { @@ -264,7 +268,7 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::D wal.SetPosition(wal_info.offset_deltas); for (size_t i = 0; i < wal_info.num_deltas;) { - i += ReadAndApplyDelta(storage, &wal); + i += ReadAndApplyDelta(storage, &wal, *version); } spdlog::debug("Replication from current WAL successful!"); @@ -277,18 +281,27 @@ void InMemoryReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::B replication::TimestampReq req; slk::Load(&req, req_reader); - replication::TimestampRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; + replication::TimestampRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()}; slk::Save(res, res_builder); } -uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder) { +uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder, + const uint64_t version) { auto edge_acc = storage->edges_.access(); auto vertex_acc = storage->vertices_.access(); + constexpr bool kUniqueAccess = true; + std::optional> commit_timestamp_and_accessor; - auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp) { + auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp, + bool unique = !kUniqueAccess) { if (!commit_timestamp_and_accessor) { - auto acc = storage->Access(std::nullopt); + std::unique_ptr acc = nullptr; + if (unique) { + acc = storage->UniqueAccess(std::nullopt); + } else { + acc = storage->Access(std::nullopt); + } auto inmem_acc = std::unique_ptr( static_cast(acc.release())); commit_timestamp_and_accessor.emplace(commit_timestamp, std::move(*inmem_acc)); @@ -299,7 +312,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, }; uint64_t applied_deltas = 0; - auto max_commit_timestamp = storage->replication_state_.last_commit_timestamp_.load(); + auto max_commit_timestamp = storage->repl_storage_state_.last_commit_timestamp_.load(); for (bool transaction_complete = false; !transaction_complete; ++applied_deltas) { const auto [timestamp, delta] = ReadDelta(decoder); @@ -307,7 +320,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, max_commit_timestamp = timestamp; } - transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type); + transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version); if (timestamp < storage->timestamp_) { continue; @@ -466,7 +479,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, case durability::WalDeltaData::Type::TRANSACTION_END: { spdlog::trace(" Transaction end"); if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp) - throw utils::BasicException("Invalid data!"); + throw utils::BasicException("Invalid commit data!"); auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first); if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); commit_timestamp_and_accessor = std::nullopt; @@ -476,25 +489,45 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: { spdlog::trace(" Create label index on :{}", delta.operation_label.label); // Need to send the timestamp - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - if (storage->CreateIndex(storage->NameToLabel(delta.operation_label.label), timestamp).HasError()) + auto *transaction = get_transaction(timestamp, kUniqueAccess); + if (transaction->CreateIndex(storage->NameToLabel(delta.operation_label.label)).HasError()) throw utils::BasicException("Invalid transaction!"); break; } case durability::WalDeltaData::Type::LABEL_INDEX_DROP: { spdlog::trace(" Drop label index on :{}", delta.operation_label.label); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - if (storage->DropIndex(storage->NameToLabel(delta.operation_label.label), timestamp).HasError()) + auto *transaction = get_transaction(timestamp, kUniqueAccess); + if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError()) throw utils::BasicException("Invalid transaction!"); break; } + case durability::WalDeltaData::Type::LABEL_INDEX_STATS_SET: { + spdlog::trace(" Set label index statistics on :{}", delta.operation_label_stats.label); + // Need to send the timestamp + auto *transaction = get_transaction(timestamp); + const auto label = storage->NameToLabel(delta.operation_label_stats.label); + LabelIndexStats stats{}; + if (!FromJson(delta.operation_label_stats.stats, stats)) { + throw utils::BasicException("Failed to read statistics!"); + } + transaction->SetIndexStats(label, stats); + break; + } + case durability::WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: { + const auto &info = delta.operation_label; + spdlog::trace(" Clear label index statistics on :{}", info.label); + // Need to send the timestamp + auto *transaction = get_transaction(timestamp); + transaction->DeleteLabelIndexStats(storage->NameToLabel(info.label)); + break; + } case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: { spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label, delta.operation_label_property.property); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - if (storage + auto *transaction = get_transaction(timestamp, kUniqueAccess); + if (transaction ->CreateIndex(storage->NameToLabel(delta.operation_label_property.label), - storage->NameToProperty(delta.operation_label_property.property), timestamp) + storage->NameToProperty(delta.operation_label_property.property)) .HasError()) throw utils::BasicException("Invalid transaction!"); break; @@ -502,31 +535,53 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: { spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label, delta.operation_label_property.property); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - if (storage + auto *transaction = get_transaction(timestamp, kUniqueAccess); + if (transaction ->DropIndex(storage->NameToLabel(delta.operation_label_property.label), - storage->NameToProperty(delta.operation_label_property.property), timestamp) + storage->NameToProperty(delta.operation_label_property.property)) .HasError()) throw utils::BasicException("Invalid transaction!"); break; } + case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: { + const auto &info = delta.operation_label_property_stats; + spdlog::trace(" Set label-property index statistics on :{}", info.label); + // Need to send the timestamp + auto *transaction = get_transaction(timestamp); + const auto label = storage->NameToLabel(info.label); + const auto property = storage->NameToProperty(info.property); + LabelPropertyIndexStats stats{}; + if (!FromJson(info.stats, stats)) { + throw utils::BasicException("Failed to read statistics!"); + } + transaction->SetIndexStats(label, property, stats); + break; + } + case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: { + const auto &info = delta.operation_label; + spdlog::trace(" Clear label-property index statistics on :{}", info.label); + // Need to send the timestamp + auto *transaction = get_transaction(timestamp); + transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label)); + break; + } case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: { spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label, delta.operation_label_property.property); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - auto ret = storage->CreateExistenceConstraint(storage->NameToLabel(delta.operation_label_property.label), - storage->NameToProperty(delta.operation_label_property.property), - timestamp); + auto *transaction = get_transaction(timestamp, kUniqueAccess); + auto ret = + transaction->CreateExistenceConstraint(storage->NameToLabel(delta.operation_label_property.label), + storage->NameToProperty(delta.operation_label_property.property)); if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); break; } case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label, delta.operation_label_property.property); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); - if (storage + auto *transaction = get_transaction(timestamp, kUniqueAccess); + if (transaction ->DropExistenceConstraint(storage->NameToLabel(delta.operation_label_property.label), - storage->NameToProperty(delta.operation_label_property.property), timestamp) + storage->NameToProperty(delta.operation_label_property.property)) .HasError()) throw utils::BasicException("Invalid transaction!"); break; @@ -535,13 +590,13 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, std::stringstream ss; utils::PrintIterable(ss, delta.operation_label_properties.properties); spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str()); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); std::set properties; for (const auto &prop : delta.operation_label_properties.properties) { properties.emplace(storage->NameToProperty(prop)); } - auto ret = storage->CreateUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), - properties, timestamp); + auto *transaction = get_transaction(timestamp, kUniqueAccess); + auto ret = transaction->CreateUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), + properties); if (!ret.HasValue() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) throw utils::BasicException("Invalid transaction!"); break; @@ -550,23 +605,24 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, std::stringstream ss; utils::PrintIterable(ss, delta.operation_label_properties.properties); spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str()); - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid transaction!"); std::set properties; for (const auto &prop : delta.operation_label_properties.properties) { properties.emplace(storage->NameToProperty(prop)); } - auto ret = storage->DropUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), - properties, timestamp); - if (ret.HasError() || ret.GetValue() != UniqueConstraints::DeletionStatus::SUCCESS) + auto *transaction = get_transaction(timestamp, kUniqueAccess); + auto ret = + transaction->DropUniqueConstraint(storage->NameToLabel(delta.operation_label_properties.label), properties); + if (ret != UniqueConstraints::DeletionStatus::SUCCESS) { throw utils::BasicException("Invalid transaction!"); + } break; } } } - if (commit_timestamp_and_accessor) throw utils::BasicException("Invalid data!"); + if (commit_timestamp_and_accessor) throw utils::BasicException("Did not finish the transaction!"); - storage->replication_state_.last_commit_timestamp_ = max_commit_timestamp; + storage->repl_storage_state_.last_commit_timestamp_ = max_commit_timestamp; spdlog::debug("Applied {} deltas", applied_deltas); return applied_deltas; diff --git a/src/storage/v2/inmemory/replication/replication_server.hpp b/src/storage/v2/inmemory/replication/replication_server.hpp index 1f6f380af..697108a0a 100644 --- a/src/storage/v2/inmemory/replication/replication_server.hpp +++ b/src/storage/v2/inmemory/replication/replication_server.hpp @@ -11,6 +11,7 @@ #pragma once +#include "replication/epoch.hpp" #include "storage/v2/replication/replication_server.hpp" #include "storage/v2/replication/serialization.hpp" @@ -20,7 +21,9 @@ class InMemoryStorage; class InMemoryReplicationServer : public ReplicationServer { public: - explicit InMemoryReplicationServer(InMemoryStorage *storage, const replication::ReplicationServerConfig &config); + explicit InMemoryReplicationServer(InMemoryStorage *storage, + const memgraph::replication::ReplicationServerConfig &config, + memgraph::replication::ReplicationEpoch *repl_epoch); private: // RPC handlers @@ -36,11 +39,14 @@ class InMemoryReplicationServer : public ReplicationServer { void TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder); - static void LoadWal(InMemoryStorage *storage, replication::Decoder *decoder); + static void LoadWal(InMemoryStorage *storage, memgraph::replication::ReplicationEpoch &epoch, + replication::Decoder *decoder); - static uint64_t ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder); + static uint64_t ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder, uint64_t version); InMemoryStorage *storage_; + + memgraph::replication::ReplicationEpoch *repl_epoch_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 55d0b0b47..63acba07a 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -12,11 +12,14 @@ #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" +#include "storage/v2/metadata_delta.hpp" /// REPLICATION /// #include "storage/v2/inmemory/replication/replication_client.hpp" #include "storage/v2/inmemory/replication/replication_server.hpp" #include "storage/v2/inmemory/unique_constraints.hpp" +#include "storage/v2/replication/replication_handler.hpp" +#include "utils/resource_lock.hpp" namespace memgraph::storage { @@ -56,17 +59,18 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) "process!", config_.durability.storage_directory); } + auto &repl_state = repl_state_; if (config_.durability.recover_on_startup) { - auto &epoch = replication_state_.GetEpoch(); - auto info = durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch.id, - &replication_state_.history, &vertices_, &edges_, &edge_count_, + auto &epoch = repl_state.GetEpoch(); + auto info = durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, epoch, + &repl_storage_state_.history, &vertices_, &edges_, &edge_count_, name_id_mapper_.get(), &indices_, &constraints_, config_, &wal_seq_num_); if (info) { vertex_id_ = info->next_vertex_id; edge_id_ = info->next_edge_id; timestamp_ = std::max(timestamp_, info->next_timestamp); if (info->last_commit_timestamp) { - replication_state_.last_commit_timestamp_ = *info->last_commit_timestamp; + repl_storage_state_.last_commit_timestamp_ = *info->last_commit_timestamp; } } } else if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || @@ -100,7 +104,8 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) } if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED) { snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval, [this] { - if (auto maybe_error = this->CreateSnapshot({true}); maybe_error.HasError()) { + auto const &repl_state = repl_state_; + if (auto maybe_error = this->CreateSnapshot(repl_state, {true}); maybe_error.HasError()) { switch (maybe_error.GetError()) { case CreateSnapshotError::DisabledForReplica: spdlog::warn( @@ -129,19 +134,14 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode) if (config_.durability.restore_replication_state_on_startup) { spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash."); - RestoreReplicationRole(); - - if (replication_state_.GetRole() == replication::ReplicationRole::MAIN) { - RestoreReplicas(); - } + ReplicationHandler{repl_state, *this}.RestoreReplication(); } else { spdlog::warn( "Replication configuration will NOT be stored. When the server restarts, replication state will be " "forgotten."); } - if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED && - replication_state_.GetRole() == replication::ReplicationRole::MAIN) { + if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED && repl_state.IsMain()) { spdlog::warn( "The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider " "enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because " @@ -156,8 +156,8 @@ InMemoryStorage::~InMemoryStorage() { gc_runner_.Stop(); } { - // Clear replication data - replication_state_.Reset(); + // Stop replication (Stop all clients or stop the REPLICA server) + repl_storage_state_.Reset(); } if (wal_file_) { wal_file_->FinalizeWal(); @@ -167,7 +167,8 @@ InMemoryStorage::~InMemoryStorage() { snapshot_runner_.Stop(); } if (config_.durability.snapshot_on_exit) { - if (auto maybe_error = this->CreateSnapshot({false}); maybe_error.HasError()) { + auto const &repl_state = repl_state_; + if (auto maybe_error = this->CreateSnapshot(repl_state, {false}); maybe_error.HasError()) { switch (maybe_error.GetError()) { case CreateSnapshotError::DisabledForReplica: spdlog::warn(utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication")); @@ -187,9 +188,9 @@ InMemoryStorage::~InMemoryStorage() { } } -InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level, +InMemoryStorage::InMemoryAccessor::InMemoryAccessor(auto tag, InMemoryStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode) - : Accessor(storage, isolation_level, storage_mode), config_(storage->config_.items) {} + : Accessor(tag, storage, isolation_level, storage_mode), config_(storage->config_.items) {} InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryAccessor &&other) noexcept : Accessor(std::move(other)), config_(other.config_) {} @@ -642,7 +643,7 @@ Result InMemoryStorage::InMemoryAccessor::EdgeSetTo(EdgeAccessor * } // NOLINTNEXTLINE(google-default-arguments) -utils::BasicResult InMemoryStorage::InMemoryAccessor::Commit( +utils::BasicResult InMemoryStorage::InMemoryAccessor::Commit( const std::optional desired_commit_timestamp) { MG_ASSERT(is_transaction_active_, "The transaction is already terminated!"); MG_ASSERT(!transaction_.must_abort, "The transaction can't be committed!"); @@ -651,7 +652,50 @@ utils::BasicResult InMemoryStorage::InMemory auto *mem_storage = static_cast(storage_); - if (transaction_.deltas.use().empty()) { + auto const &replState = mem_storage->repl_state_; + if (!transaction_.md_deltas.empty()) { + // This is usually done by the MVCC, but it does not handle the metadata deltas + transaction_.EnsureCommitTimestampExists(); + + // Save these so we can mark them used in the commit log. + uint64_t start_timestamp = transaction_.start_timestamp; + + std::unique_lock engine_guard(storage_->engine_lock_); + commit_timestamp_.emplace(mem_storage->CommitTimestamp(desired_commit_timestamp)); + + // Write transaction to WAL while holding the engine lock to make sure + // that committed transactions are sorted by the commit timestamp in the + // WAL files. We supply the new commit timestamp to the function so that + // it knows what will be the final commit timestamp. The WAL must be + // written before actually committing the transaction (before setting + // the commit timestamp) so that no other transaction can see the + // modifications before they are written to disk. + // Replica can log only the write transaction received from Main + // so the Wal files are consistent + if (replState.IsMain() || desired_commit_timestamp.has_value()) { + could_replicate_all_sync_replicas = mem_storage->AppendToWalDataDefinition(transaction_, *commit_timestamp_); + // Take committed_transactions lock while holding the engine lock to + // make sure that committed transactions are sorted by the commit + // timestamp in the list. + mem_storage->committed_transactions_.WithLock([&](auto & /*committed_transactions*/) { + // TODO: release lock, and update all deltas to have a local copy + // of the commit timestamp + MG_ASSERT(transaction_.commit_timestamp != nullptr, "Invalid database state!"); + transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); + // Replica can only update the last commit timestamp with + // the commits received from main. + if (replState.IsMain() || desired_commit_timestamp.has_value()) { + // Update the last commit timestamp + mem_storage->repl_storage_state_.last_commit_timestamp_.store(*commit_timestamp_); + } + // Release engine lock because we don't have to hold it anymore + // and emplace back could take a long time. + engine_guard.unlock(); + }); + + mem_storage->commit_log_->MarkFinished(start_timestamp); + } + } else if (transaction_.deltas.use().empty()) { // We don't have to update the commit timestamp here because no one reads // it. mem_storage->commit_log_->MarkFinished(transaction_.start_timestamp); @@ -669,7 +713,7 @@ utils::BasicResult InMemoryStorage::InMemory auto validation_result = storage_->constraints_.existence_constraints_->Validate(*prev.vertex); if (validation_result) { Abort(); - return StorageDataManipulationError{*validation_result}; + return StorageManipulationError{*validation_result}; } } @@ -726,8 +770,7 @@ utils::BasicResult InMemoryStorage::InMemory // modifications before they are written to disk. // Replica can log only the write transaction received from Main // so the Wal files are consistent - if (mem_storage->replication_state_.GetRole() == replication::ReplicationRole::MAIN || - desired_commit_timestamp.has_value()) { + if (replState.IsMain() || desired_commit_timestamp.has_value()) { could_replicate_all_sync_replicas = mem_storage->AppendToWalDataManipulation(transaction_, *commit_timestamp_); } @@ -742,10 +785,9 @@ utils::BasicResult InMemoryStorage::InMemory transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); // Replica can only update the last commit timestamp with // the commits received from main. - if (mem_storage->replication_state_.GetRole() == replication::ReplicationRole::MAIN || - desired_commit_timestamp.has_value()) { + if (replState.IsMain() || desired_commit_timestamp.has_value()) { // Update the last commit timestamp - mem_storage->replication_state_.last_commit_timestamp_.store(*commit_timestamp_); + mem_storage->repl_storage_state_.last_commit_timestamp_.store(*commit_timestamp_); } // Release engine lock because we don't have to hold it anymore // and emplace back could take a long time. @@ -758,14 +800,14 @@ utils::BasicResult InMemoryStorage::InMemory if (unique_constraint_violation) { Abort(); - return StorageDataManipulationError{*unique_constraint_violation}; + return StorageManipulationError{*unique_constraint_violation}; } } is_transaction_active_ = false; if (!could_replicate_all_sync_replicas) { - return StorageDataManipulationError{ReplicationError{}}; + return StorageManipulationError{ReplicationError{}}; } return {}; @@ -951,189 +993,121 @@ void InMemoryStorage::InMemoryAccessor::FinalizeTransaction() { } } -utils::BasicResult InMemoryStorage::CreateIndex( - LabelId label, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - auto *mem_label_index = static_cast(indices_.label_index_.get()); - if (!mem_label_index->CreateIndex(label, vertices_.access(), std::nullopt)) { +utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex(LabelId label) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); + if (!mem_label_index->CreateIndex(label, in_memory->vertices_.access(), std::nullopt)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - const auto success = - AppendToWalDataDefinition(durability::StorageGlobalOperation::LABEL_INDEX_CREATE, label, {}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_create, label); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelIndices); - - if (success) { - return {}; - } - - return StorageIndexDefinitionError{ReplicationError{}}; + return {}; } -utils::BasicResult InMemoryStorage::CreateIndex( - LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - auto *mem_label_property_index = static_cast(indices_.label_property_index_.get()); - if (!mem_label_property_index->CreateIndex(label, property, vertices_.access(), std::nullopt)) { +utils::BasicResult InMemoryStorage::InMemoryAccessor::CreateIndex( + LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_label_property_index = + static_cast(in_memory->indices_.label_property_index_.get()); + if (!mem_label_property_index->CreateIndex(label, property, in_memory->vertices_.access(), std::nullopt)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE, label, - {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_create, label, property); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelPropertyIndices); - - if (success) { - return {}; - } - - return StorageIndexDefinitionError{ReplicationError{}}; + return {}; } -utils::BasicResult InMemoryStorage::DropIndex( - LabelId label, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - if (!indices_.label_index_->DropIndex(label)) { +utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex(LabelId label) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); + if (!mem_label_index->DropIndex(label)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = - AppendToWalDataDefinition(durability::StorageGlobalOperation::LABEL_INDEX_DROP, label, {}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_drop, label); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelIndices); - - if (success) { - return {}; - } - - return StorageIndexDefinitionError{ReplicationError{}}; + return {}; } -utils::BasicResult InMemoryStorage::DropIndex( - LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - if (!indices_.label_property_index_->DropIndex(label, property)) { +utils::BasicResult InMemoryStorage::InMemoryAccessor::DropIndex( + LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_label_property_index = + static_cast(in_memory->indices_.label_property_index_.get()); + if (!mem_label_property_index->DropIndex(label, property)) { return StorageIndexDefinitionError{IndexDefinitionError{}}; } - // For a description why using `timestamp_` is correct, see - // `CreateIndex(LabelId label)`. - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP, label, - {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_drop, label, property); // We don't care if there is a replication error because on main node the change will go through memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelPropertyIndices); - - if (success) { - return {}; - } - - return StorageIndexDefinitionError{ReplicationError{}}; + return {}; } -utils::BasicResult InMemoryStorage::CreateExistenceConstraint( - LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - - if (constraints_.existence_constraints_->ConstraintExists(label, property)) { +utils::BasicResult +InMemoryStorage::InMemoryAccessor::CreateExistenceConstraint(LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); + if (existence_constraints->ConstraintExists(label, property)) { return StorageExistenceConstraintDefinitionError{ConstraintDefinitionError{}}; } - - if (auto violation = ExistenceConstraints::ValidateVerticesOnConstraint(vertices_.access(), label, property); + if (auto violation = + ExistenceConstraints::ValidateVerticesOnConstraint(in_memory->vertices_.access(), label, property); violation.has_value()) { return StorageExistenceConstraintDefinitionError{violation.value()}; } - - constraints_.existence_constraints_->InsertConstraint(label, property); - - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE, label, - {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - - if (success) { - return {}; - } - - return StorageExistenceConstraintDefinitionError{ReplicationError{}}; + existence_constraints->InsertConstraint(label, property); + transaction_.md_deltas.emplace_back(MetadataDelta::existence_constraint_create, label, property); + return {}; } -utils::BasicResult InMemoryStorage::DropExistenceConstraint( - LabelId label, PropertyId property, const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - if (!constraints_.existence_constraints_->DropConstraint(label, property)) { +utils::BasicResult +InMemoryStorage::InMemoryAccessor::DropExistenceConstraint(LabelId label, PropertyId property) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *existence_constraints = in_memory->constraints_.existence_constraints_.get(); + if (!existence_constraints->DropConstraint(label, property)) { return StorageExistenceConstraintDroppingError{ConstraintDefinitionError{}}; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP, label, - {property}, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - - if (success) { - return {}; - } - - return StorageExistenceConstraintDroppingError{ReplicationError{}}; + transaction_.md_deltas.emplace_back(MetadataDelta::existence_constraint_drop, label, property); + return {}; } utils::BasicResult -InMemoryStorage::CreateUniqueConstraint(LabelId label, const std::set &properties, - const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - auto *mem_unique_constraints = static_cast(constraints_.unique_constraints_.get()); - auto ret = mem_unique_constraints->CreateConstraint(label, properties, vertices_.access()); +InMemoryStorage::InMemoryAccessor::CreateUniqueConstraint(LabelId label, const std::set &properties) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_unique_constraints = + static_cast(in_memory->constraints_.unique_constraints_.get()); + auto ret = mem_unique_constraints->CreateConstraint(label, properties, in_memory->vertices_.access()); if (ret.HasError()) { return StorageUniqueConstraintDefinitionError{ret.GetError()}; } if (ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) { return ret.GetValue(); } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE, label, - properties, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - - if (success) { - return UniqueConstraints::CreationStatus::SUCCESS; - } - - return StorageUniqueConstraintDefinitionError{ReplicationError{}}; + transaction_.md_deltas.emplace_back(MetadataDelta::unique_constraint_create, label, properties); + return UniqueConstraints::CreationStatus::SUCCESS; } -utils::BasicResult -InMemoryStorage::DropUniqueConstraint(LabelId label, const std::set &properties, - const std::optional desired_commit_timestamp) { - std::unique_lock storage_guard(main_lock_); - auto ret = constraints_.unique_constraints_->DropConstraint(label, properties); +UniqueConstraints::DeletionStatus InMemoryStorage::InMemoryAccessor::DropUniqueConstraint( + LabelId label, const std::set &properties) { + MG_ASSERT(unique_guard_.owns_lock(), "Create index requires a unique access to the storage!"); + auto *in_memory = static_cast(storage_); + auto *mem_unique_constraints = + static_cast(in_memory->constraints_.unique_constraints_.get()); + auto ret = mem_unique_constraints->DropConstraint(label, properties); if (ret != UniqueConstraints::DeletionStatus::SUCCESS) { return ret; } - const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp); - auto success = AppendToWalDataDefinition(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP, label, - properties, commit_timestamp); - commit_log_->MarkFinished(commit_timestamp); - replication_state_.last_commit_timestamp_ = commit_timestamp; - - if (success) { - return UniqueConstraints::DeletionStatus::SUCCESS; - } - - return StorageUniqueConstraintDroppingError{ReplicationError{}}; + transaction_.md_deltas.emplace_back(MetadataDelta::unique_constraint_drop, label, properties); + return UniqueConstraints::DeletionStatus::SUCCESS; } VerticesIterable InMemoryStorage::InMemoryAccessor::Vertices(LabelId label, View view) { @@ -1181,7 +1155,8 @@ Transaction InMemoryStorage::CreateTransaction(IsolationLevel isolation_level, S // of any query on replica to the last commited transaction // which is timestamp_ as only commit of transaction with writes // can change the value of it. - if (replication_state_.GetRole() == replication::ReplicationRole::REPLICA) { + auto const &replState = repl_state_; + if (replState.IsReplica()) { start_timestamp = timestamp_; } else { start_timestamp = timestamp_++; @@ -1191,7 +1166,7 @@ Transaction InMemoryStorage::CreateTransaction(IsolationLevel isolation_level, S } template -void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) { +void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) { // NOTE: You do not need to consider cleanup of deleted object that occurred in // different storage modes within the same CollectGarbage call. This is because // SetStorageMode will ensure CollectGarbage is called before any new transactions @@ -1501,8 +1476,8 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) } // tell the linker he can find the CollectGarbage definitions here -template void InMemoryStorage::CollectGarbage(std::unique_lock); -template void InMemoryStorage::CollectGarbage(std::unique_lock); +template void InMemoryStorage::CollectGarbage(std::unique_lock); +template void InMemoryStorage::CollectGarbage(std::unique_lock); StorageInfo InMemoryStorage::GetInfo() const { auto vertex_count = vertices_.size(); @@ -1516,12 +1491,12 @@ StorageInfo InMemoryStorage::GetInfo() const { utils::GetDirDiskUsage(config_.durability.storage_directory)}; } -bool InMemoryStorage::InitializeWalFile() { +bool InMemoryStorage::InitializeWalFile(memgraph::replication::ReplicationEpoch &epoch) { if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL) return false; if (!wal_file_) { - wal_file_.emplace(wal_directory_, uuid_, replication_state_.GetEpoch().id, config_.items, name_id_mapper_.get(), - wal_seq_num_++, &file_retainer_); + wal_file_.emplace(wal_directory_, uuid_, epoch.id(), config_.items, name_id_mapper_.get(), wal_seq_num_++, + &file_retainer_); } return true; } @@ -1546,14 +1521,15 @@ void InMemoryStorage::FinalizeWalFile() { } bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction, uint64_t final_commit_timestamp) { - if (!InitializeWalFile()) { + auto &replState = repl_state_; + if (!InitializeWalFile(replState.GetEpoch())) { return true; } // Traverse deltas and append them to the WAL file. // A single transaction will always be contained in a single WAL file. auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire); - replication_state_.InitializeTransaction(wal_file_->SequenceNumber()); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); auto append_deltas = [&](auto callback) { // Helper lambda that traverses the delta chain on order to find the first @@ -1704,7 +1680,7 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction append_deltas([&](const Delta &delta, const auto &parent, uint64_t timestamp) { wal_file_->AppendDelta(delta, parent, timestamp); - replication_state_.AppendDelta(delta, parent, timestamp); + repl_storage_state_.AppendDelta(delta, parent, timestamp); }); // Add a delta that indicates that the transaction is fully written to the WAL @@ -1712,36 +1688,134 @@ bool InMemoryStorage::AppendToWalDataManipulation(const Transaction &transaction wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return replication_state_.FinalizeTransaction(final_commit_timestamp); + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); } -bool InMemoryStorage::AppendToWalDataDefinition(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, - uint64_t final_commit_timestamp) { - if (!InitializeWalFile()) { +bool InMemoryStorage::AppendToWalDataDefinition(const Transaction &transaction, uint64_t final_commit_timestamp) { + auto &replState = repl_state_; + if (!InitializeWalFile(replState.GetEpoch())) { return true; } - wal_file_->AppendOperation(operation, label, properties, final_commit_timestamp); + repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber()); + + for (const auto &md_delta : transaction.md_deltas) { + switch (md_delta.action) { + case MetadataDelta::Action::LABEL_INDEX_CREATE: { + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_INDEX_CREATE, md_delta.label, + final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_CREATE: { + const auto &info = md_delta.label_property; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE, info.label, + {info.property}, final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_INDEX_DROP: { + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_INDEX_DROP, md_delta.label, + final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_DROP: { + const auto &info = md_delta.label_property; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP, info.label, + {info.property}, final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_INDEX_STATS_SET: { + const auto &info = md_delta.label_stats; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_INDEX_STATS_SET, info.label, info.stats, + final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_INDEX_STATS_CLEAR: { + const auto &info = md_delta.label_stats; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_INDEX_STATS_CLEAR, info.label, + final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_STATS_SET: { + const auto &info = md_delta.label_property_stats; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET, info.label, + {info.property}, info.stats, final_commit_timestamp); + } break; + case MetadataDelta::Action::LABEL_PROPERTY_INDEX_STATS_CLEAR: /* Special case we clear all label/property + pairs with the defined label */ + { + const auto &info = md_delta.label_stats; + AppendToWalDataDefinition(durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR, info.label, + final_commit_timestamp); + } break; + case MetadataDelta::Action::EXISTENCE_CONSTRAINT_CREATE: { + const auto &info = md_delta.label_property; + AppendToWalDataDefinition(durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE, info.label, + {info.property}, final_commit_timestamp); + } break; + case MetadataDelta::Action::EXISTENCE_CONSTRAINT_DROP: { + const auto &info = md_delta.label_property; + AppendToWalDataDefinition(durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP, info.label, + {info.property}, final_commit_timestamp); + } break; + case MetadataDelta::Action::UNIQUE_CONSTRAINT_CREATE: { + const auto &info = md_delta.label_properties; + AppendToWalDataDefinition(durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_CREATE, info.label, + info.properties, final_commit_timestamp); + } break; + case MetadataDelta::Action::UNIQUE_CONSTRAINT_DROP: { + const auto &info = md_delta.label_properties; + AppendToWalDataDefinition(durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_DROP, info.label, + info.properties, final_commit_timestamp); + } break; + } + } + + // Add a delta that indicates that the transaction is fully written to the WAL + wal_file_->AppendTransactionEnd(final_commit_timestamp); FinalizeWalFile(); - return replication_state_.AppendOperation(wal_file_->SequenceNumber(), operation, label, properties, - final_commit_timestamp); + + return repl_storage_state_.FinalizeTransaction(final_commit_timestamp); +} + +void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, LabelIndexStats stats, + LabelPropertyIndexStats property_stats, + uint64_t final_commit_timestamp) { + wal_file_->AppendOperation(operation, label, properties, stats, property_stats, final_commit_timestamp); + repl_storage_state_.AppendOperation(operation, label, properties, stats, property_stats, final_commit_timestamp); +} + +void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, + LabelPropertyIndexStats property_stats, + uint64_t final_commit_timestamp) { + return AppendToWalDataDefinition(operation, label, properties, {}, property_stats, final_commit_timestamp); +} + +void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + LabelIndexStats stats, uint64_t final_commit_timestamp) { + return AppendToWalDataDefinition(operation, label, {}, stats, {}, final_commit_timestamp); +} + +void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, + uint64_t final_commit_timestamp) { + return AppendToWalDataDefinition(operation, label, properties, {}, final_commit_timestamp); +} + +void InMemoryStorage::AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + uint64_t final_commit_timestamp) { + return AppendToWalDataDefinition(operation, label, {}, {}, final_commit_timestamp); } utils::BasicResult InMemoryStorage::CreateSnapshot( - std::optional is_periodic) { - if (replication_state_.GetRole() != replication::ReplicationRole::MAIN) { + memgraph::replication::ReplicationState const &replicationState, std::optional is_periodic) { + if (replicationState.IsReplica()) { return CreateSnapshotError::DisabledForReplica; } - auto snapshot_creator = [this]() { + auto const &epoch = replicationState.GetEpoch(); + auto snapshot_creator = [this, &epoch]() { utils::Timer timer; - const auto &epoch = replication_state_.GetEpoch(); auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION, storage_mode_); // Create snapshot. durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_, config_.durability.snapshot_retention_count, &vertices_, &edges_, name_id_mapper_.get(), - &indices_, &constraints_, config_, uuid_, epoch.id, replication_state_.history, + &indices_, &constraints_, config_, uuid_, epoch, repl_storage_state_.history, &file_retainer_); // Finalize snapshot transaction. commit_log_->MarkFinished(transaction.start_timestamp); @@ -1756,7 +1830,7 @@ utils::BasicResult InMemoryStorage::Create auto max_num_tries{10}; while (max_num_tries) { if (should_try_shared) { - std::shared_lock storage_guard(main_lock_); + std::shared_lock storage_guard(main_lock_); if (storage_mode_ == memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL) { snapshot_creator(); return {}; @@ -1778,7 +1852,7 @@ utils::BasicResult InMemoryStorage::Create return CreateSnapshotError::ReachedMaxNumTries; } -void InMemoryStorage::FreeMemory(std::unique_lock main_guard) { +void InMemoryStorage::FreeMemory(std::unique_lock main_guard) { CollectGarbage(std::move(main_guard)); // SkipList is already threadsafe @@ -1797,14 +1871,13 @@ uint64_t InMemoryStorage::CommitTimestamp(const std::optional desired_ return *desired_commit_timestamp; } -void InMemoryStorage::EstablishNewEpoch() { +void InMemoryStorage::PrepareForNewEpoch(std::string prev_epoch) { std::unique_lock engine_guard{engine_lock_}; if (wal_file_) { wal_file_->FinalizeWal(); wal_file_.reset(); } - // TODO: Move out of storage (no need for the lock) <- missing commit_timestamp at a higher level - replication_state_.NewEpoch(); + repl_storage_state_.AddEpochToHistory(std::move(prev_epoch)); } utils::FileRetainer::FileLockerAccessor::ret_type InMemoryStorage::IsPathLocked() { @@ -1832,14 +1905,66 @@ utils::FileRetainer::FileLockerAccessor::ret_type InMemoryStorage::UnlockPath() return true; } -auto InMemoryStorage::CreateReplicationClient(replication::ReplicationClientConfig const &config) +auto InMemoryStorage::CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config) -> std::unique_ptr { - return std::make_unique(this, config); + auto &replState = this->repl_state_; + return std::make_unique(this, config, &replState.GetEpoch()); } std::unique_ptr InMemoryStorage::CreateReplicationServer( - const replication::ReplicationServerConfig &config) { - return std::make_unique(this, config); + const memgraph::replication::ReplicationServerConfig &config) { + auto &replState = this->repl_state_; + return std::make_unique(this, config, &replState.GetEpoch()); +} + +std::unique_ptr InMemoryStorage::Access(std::optional override_isolation_level) { + return std::unique_ptr(new InMemoryAccessor{ + Storage::Accessor::shared_access, this, override_isolation_level.value_or(isolation_level_), storage_mode_}); +} +std::unique_ptr InMemoryStorage::UniqueAccess( + std::optional override_isolation_level) { + return std::unique_ptr(new InMemoryAccessor{ + Storage::Accessor::unique_access, this, override_isolation_level.value_or(isolation_level_), storage_mode_}); +} +IndicesInfo InMemoryStorage::InMemoryAccessor::ListAllIndices() const { + auto *in_memory = static_cast(storage_); + auto *mem_label_index = static_cast(in_memory->indices_.label_index_.get()); + auto *mem_label_property_index = + static_cast(in_memory->indices_.label_property_index_.get()); + return {mem_label_index->ListIndices(), mem_label_property_index->ListIndices()}; +} +ConstraintsInfo InMemoryStorage::InMemoryAccessor::ListAllConstraints() const { + const auto *mem_storage = static_cast(storage_); + return {mem_storage->constraints_.existence_constraints_->ListConstraints(), + mem_storage->constraints_.unique_constraints_->ListConstraints()}; +} + +void InMemoryStorage::InMemoryAccessor::SetIndexStats(const storage::LabelId &label, const LabelIndexStats &stats) { + SetIndexStatsForIndex(static_cast(storage_->indices_.label_index_.get()), label, stats); + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_stats_set, label, stats); +} + +void InMemoryStorage::InMemoryAccessor::SetIndexStats(const storage::LabelId &label, + const storage::PropertyId &property, + const LabelPropertyIndexStats &stats) { + SetIndexStatsForIndex(static_cast(storage_->indices_.label_property_index_.get()), + std::make_pair(label, property), stats); + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_stats_set, label, property, stats); +} + +bool InMemoryStorage::InMemoryAccessor::DeleteLabelIndexStats(const storage::LabelId &label) { + const auto res = + DeleteIndexStatsForIndex(static_cast(storage_->indices_.label_index_.get()), label); + transaction_.md_deltas.emplace_back(MetadataDelta::label_index_stats_clear, label); + return res; +} + +std::vector> InMemoryStorage::InMemoryAccessor::DeleteLabelPropertyIndexStats( + const storage::LabelId &label) { + const auto &res = DeleteIndexStatsForIndex>>( + static_cast(storage_->indices_.label_property_index_.get()), label); + transaction_.md_deltas.emplace_back(MetadataDelta::label_property_index_stats_clear, label); + return res; } } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 136f8748b..8f403768f 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -15,19 +15,20 @@ #include #include #include +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/label_index.hpp" #include "storage/v2/inmemory/label_property_index.hpp" #include "storage/v2/storage.hpp" /// REPLICATION /// -#include "storage/v2/replication/config.hpp" +#include "replication/config.hpp" #include "storage/v2/replication/enums.hpp" -#include "storage/v2/replication/replication.hpp" -#include "storage/v2/replication/replication_persistence_helper.hpp" +#include "storage/v2/replication/replication_storage_state.hpp" #include "storage/v2/replication/rpc.hpp" #include "storage/v2/replication/serialization.hpp" #include "storage/v2/transaction.hpp" #include "utils/memory.hpp" +#include "utils/resource_lock.hpp" #include "utils/synchronized.hpp" namespace memgraph::storage { @@ -64,7 +65,8 @@ class InMemoryStorage final : public Storage { private: friend class InMemoryStorage; - explicit InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode); + explicit InMemoryAccessor(auto tag, InMemoryStorage *storage, IsolationLevel isolation_level, + StorageMode storage_mode); public: InMemoryAccessor(const InMemoryAccessor &) = delete; @@ -159,52 +161,19 @@ class InMemoryStorage final : public Storage { index->SetIndexStats(key, stats); } - void SetIndexStats(const storage::LabelId &label, const LabelIndexStats &stats) override { - SetIndexStatsForIndex(static_cast(storage_->indices_.label_index_.get()), label, stats); - } + void SetIndexStats(const storage::LabelId &label, const LabelIndexStats &stats) override; void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property, - const LabelPropertyIndexStats &stats) override { - SetIndexStatsForIndex(static_cast(storage_->indices_.label_property_index_.get()), - std::make_pair(label, property), stats); - } + const LabelPropertyIndexStats &stats) override; template - std::vector ClearIndexStatsForIndex(TIndex *index) const { - return index->ClearIndexStats(); + TResult DeleteIndexStatsForIndex(TIndex *index, const storage::LabelId &label) { + return index->DeleteIndexStats(label); } - std::vector ClearLabelIndexStats() override { - return ClearIndexStatsForIndex(static_cast(storage_->indices_.label_index_.get())); - } + std::vector> DeleteLabelPropertyIndexStats(const storage::LabelId &label) override; - std::vector> ClearLabelPropertyIndexStats() override { - return ClearIndexStatsForIndex>( - static_cast(storage_->indices_.label_property_index_.get())); - } - - template - std::vector DeleteIndexStatsForIndex(TIndex *index, const std::span labels) { - std::vector deleted_indexes; - - for (const auto &label : labels) { - std::vector loc_results = index->DeleteIndexStats(NameToLabel(label)); - deleted_indexes.insert(deleted_indexes.end(), std::make_move_iterator(loc_results.begin()), - std::make_move_iterator(loc_results.end())); - } - return deleted_indexes; - } - - std::vector> DeleteLabelPropertyIndexStats( - const std::span labels) override { - return DeleteIndexStatsForIndex>( - static_cast(storage_->indices_.label_property_index_.get()), labels); - } - - std::vector DeleteLabelIndexStats(const std::span labels) override { - return DeleteIndexStatsForIndex(static_cast(storage_->indices_.label_index_.get()), - labels); - } + bool DeleteLabelIndexStats(const storage::LabelId &label) override; Result, std::vector>>> DetachDelete( std::vector nodes, std::vector edges, bool detach) override; @@ -228,15 +197,9 @@ class InMemoryStorage final : public Storage { return static_cast(storage_)->indices_.label_property_index_->IndexExists(label, property); } - IndicesInfo ListAllIndices() const override { - const auto *mem_storage = static_cast(storage_); - return mem_storage->ListAllIndices(); - } + IndicesInfo ListAllIndices() const override; - ConstraintsInfo ListAllConstraints() const override { - const auto *mem_storage = static_cast(storage_); - return mem_storage->ListAllConstraints(); - } + ConstraintsInfo ListAllConstraints() const override; /// Returns void if the transaction has been committed. /// Returns `StorageDataManipulationError` if an error occures. Error can be: @@ -245,7 +208,7 @@ class InMemoryStorage final : public Storage { /// case the transaction is automatically aborted. /// @throw std::bad_alloc // NOLINTNEXTLINE(google-default-arguments) - utils::BasicResult Commit( + utils::BasicResult Commit( std::optional desired_commit_timestamp = {}) override; /// @throw std::bad_alloc @@ -253,6 +216,78 @@ class InMemoryStorage final : public Storage { void FinalizeTransaction() override; + /// Create an index. + /// Returns void if the index has been created. + /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: + /// * `IndexDefinitionError`: the index already exists. + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// @throw std::bad_alloc + utils::BasicResult CreateIndex(LabelId label) override; + + /// Create an index. + /// Returns void if the index has been created. + /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `IndexDefinitionError`: the index already exists. + /// @throw std::bad_alloc + utils::BasicResult CreateIndex(LabelId label, PropertyId property) override; + + /// Drop an existing index. + /// Returns void if the index has been dropped. + /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `IndexDefinitionError`: the index does not exist. + utils::BasicResult DropIndex(LabelId label) override; + + /// Drop an existing index. + /// Returns void if the index has been dropped. + /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `IndexDefinitionError`: the index does not exist. + utils::BasicResult DropIndex(LabelId label, PropertyId property) override; + + /// Returns void if the existence constraint has been created. + /// Returns `StorageExistenceConstraintDefinitionError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `ConstraintViolation`: there is already a vertex existing that would break this new constraint. + /// * `ConstraintDefinitionError`: the constraint already exists. + /// @throw std::bad_alloc + /// @throw std::length_error + utils::BasicResult CreateExistenceConstraint( + LabelId label, PropertyId property) override; + + /// Drop an existing existence constraint. + /// Returns void if the existence constraint has been dropped. + /// Returns `StorageExistenceConstraintDroppingError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `ConstraintDefinitionError`: the constraint did not exists. + utils::BasicResult DropExistenceConstraint( + LabelId label, PropertyId property) override; + + /// Create an unique constraint. + /// Returns `StorageUniqueConstraintDefinitionError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// * `ConstraintViolation`: there are already vertices violating the constraint. + /// Returns `UniqueConstraints::CreationStatus` otherwise. Value can be: + /// * `SUCCESS` if the constraint was successfully created, + /// * `ALREADY_EXISTS` if the constraint already existed, + /// * `EMPTY_PROPERTIES` if the property set is empty, or + /// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties. + /// @throw std::bad_alloc + utils::BasicResult + CreateUniqueConstraint(LabelId label, const std::set &properties) override; + + /// Removes an existing unique constraint. + /// Returns `StorageUniqueConstraintDroppingError` if an error occures. Error can be: + /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. + /// Returns `UniqueConstraints::DeletionStatus` otherwise. Value can be: + /// * `SUCCESS` if constraint was successfully removed, + /// * `NOT_FOUND` if the specified constraint was not found, + /// * `EMPTY_PROPERTIES` if the property set is empty, or + /// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties. + UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, + const std::set &properties) override; + protected: // TODO Better naming /// @throw std::bad_alloc @@ -280,101 +315,25 @@ class InMemoryStorage final : public Storage { Transaction &GetTransaction() { return transaction_; } }; - std::unique_ptr Access(std::optional override_isolation_level) override { - return std::unique_ptr( - new InMemoryAccessor{this, override_isolation_level.value_or(isolation_level_), storage_mode_}); - } + std::unique_ptr Access(std::optional override_isolation_level) override; - /// Create an index. - /// Returns void if the index has been created. - /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: - /// * `IndexDefinitionError`: the index already exists. - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// @throw std::bad_alloc - utils::BasicResult CreateIndex( - LabelId label, std::optional desired_commit_timestamp) override; + std::unique_ptr UniqueAccess(std::optional override_isolation_level) override; - /// Create an index. - /// Returns void if the index has been created. - /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `IndexDefinitionError`: the index already exists. - /// @throw std::bad_alloc - utils::BasicResult CreateIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - /// Drop an existing index. - /// Returns void if the index has been dropped. - /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `IndexDefinitionError`: the index does not exist. - utils::BasicResult DropIndex( - LabelId label, std::optional desired_commit_timestamp) override; - - /// Drop an existing index. - /// Returns void if the index has been dropped. - /// Returns `StorageIndexDefinitionError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `IndexDefinitionError`: the index does not exist. - utils::BasicResult DropIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - /// Returns void if the existence constraint has been created. - /// Returns `StorageExistenceConstraintDefinitionError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `ConstraintViolation`: there is already a vertex existing that would break this new constraint. - /// * `ConstraintDefinitionError`: the constraint already exists. - /// @throw std::bad_alloc - /// @throw std::length_error - utils::BasicResult CreateExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - /// Drop an existing existence constraint. - /// Returns void if the existence constraint has been dropped. - /// Returns `StorageExistenceConstraintDroppingError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `ConstraintDefinitionError`: the constraint did not exists. - utils::BasicResult DropExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) override; - - /// Create an unique constraint. - /// Returns `StorageUniqueConstraintDefinitionError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// * `ConstraintViolation`: there are already vertices violating the constraint. - /// Returns `UniqueConstraints::CreationStatus` otherwise. Value can be: - /// * `SUCCESS` if the constraint was successfully created, - /// * `ALREADY_EXISTS` if the constraint already existed, - /// * `EMPTY_PROPERTIES` if the property set is empty, or - /// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties. - /// @throw std::bad_alloc - utils::BasicResult CreateUniqueConstraint( - LabelId label, const std::set &properties, std::optional desired_commit_timestamp) override; - - /// Removes an existing unique constraint. - /// Returns `StorageUniqueConstraintDroppingError` if an error occures. Error can be: - /// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction. - /// Returns `UniqueConstraints::DeletionStatus` otherwise. Value can be: - /// * `SUCCESS` if constraint was successfully removed, - /// * `NOT_FOUND` if the specified constraint was not found, - /// * `EMPTY_PROPERTIES` if the property set is empty, or - /// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties. - utils::BasicResult DropUniqueConstraint( - LabelId label, const std::set &properties, std::optional desired_commit_timestamp) override; - - void FreeMemory(std::unique_lock main_guard) override; + void FreeMemory(std::unique_lock main_guard) override; utils::FileRetainer::FileLockerAccessor::ret_type IsPathLocked(); utils::FileRetainer::FileLockerAccessor::ret_type LockPath(); utils::FileRetainer::FileLockerAccessor::ret_type UnlockPath(); - utils::BasicResult CreateSnapshot(std::optional is_periodic); + utils::BasicResult CreateSnapshot( + memgraph::replication::ReplicationState const &replicationState, std::optional is_periodic); Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode) override; - auto CreateReplicationClient(replication::ReplicationClientConfig const &config) + auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config) -> std::unique_ptr override; - auto CreateReplicationServer(const replication::ReplicationServerConfig &config) + auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig &config) -> std::unique_ptr override; private: @@ -390,22 +349,37 @@ class InMemoryStorage final : public Storage { /// @throw std::system_error /// @throw std::bad_alloc template - void CollectGarbage(std::unique_lock main_guard = {}); + void CollectGarbage(std::unique_lock main_guard = {}); - bool InitializeWalFile(); + bool InitializeWalFile(memgraph::replication::ReplicationEpoch &epoch); void FinalizeWalFile(); StorageInfo GetInfo() const override; - /// Return true in all cases excepted if any sync replicas have not sent confirmation. [[nodiscard]] bool AppendToWalDataManipulation(const Transaction &transaction, uint64_t final_commit_timestamp); /// Return true in all cases excepted if any sync replicas have not sent confirmation. - [[nodiscard]] bool AppendToWalDataDefinition(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t final_commit_timestamp); + [[nodiscard]] bool AppendToWalDataDefinition(const Transaction &transaction, uint64_t final_commit_timestamp); + /// Return true in all cases excepted if any sync replicas have not sent confirmation. + void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + uint64_t final_commit_timestamp); + /// Return true in all cases excepted if any sync replicas have not sent confirmation. + void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, uint64_t final_commit_timestamp); + /// Return true in all cases excepted if any sync replicas have not sent confirmation. + void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, LabelIndexStats stats, + uint64_t final_commit_timestamp); + /// Return true in all cases excepted if any sync replicas have not sent confirmation. + void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, LabelPropertyIndexStats property_stats, + uint64_t final_commit_timestamp); + /// Return true in all cases excepted if any sync replicas have not sent confirmation. + void AppendToWalDataDefinition(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, LabelIndexStats stats, + LabelPropertyIndexStats property_stats, uint64_t final_commit_timestamp); uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); - void EstablishNewEpoch() override; + void PrepareForNewEpoch(std::string prev_epoch) override; // Main object storage utils::SkipList vertices_; diff --git a/src/storage/v2/metadata_delta.hpp b/src/storage/v2/metadata_delta.hpp new file mode 100644 index 000000000..94d806c19 --- /dev/null +++ b/src/storage/v2/metadata_delta.hpp @@ -0,0 +1,157 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "storage/v2/edge_ref.hpp" +#include "storage/v2/id_types.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" +#include "storage/v2/property_value.hpp" +#include "utils/logging.hpp" + +namespace memgraph::storage { + +// NOTE: MetadataDeltas are not supported in a multi command transaction (single query only) +struct MetadataDelta { + enum class Action { + LABEL_INDEX_CREATE, + LABEL_INDEX_DROP, + LABEL_INDEX_STATS_SET, + LABEL_INDEX_STATS_CLEAR, + LABEL_PROPERTY_INDEX_CREATE, + LABEL_PROPERTY_INDEX_DROP, + LABEL_PROPERTY_INDEX_STATS_SET, + LABEL_PROPERTY_INDEX_STATS_CLEAR, + EXISTENCE_CONSTRAINT_CREATE, + EXISTENCE_CONSTRAINT_DROP, + UNIQUE_CONSTRAINT_CREATE, + UNIQUE_CONSTRAINT_DROP, + }; + + static constexpr struct LabelIndexCreate { + } label_index_create; + static constexpr struct LabelIndexDrop { + } label_index_drop; + static constexpr struct LabelIndexStatsSet { + } label_index_stats_set; + static constexpr struct LabelIndexStatsClear { + } label_index_stats_clear; + static constexpr struct LabelPropertyIndexCreate { + } label_property_index_create; + static constexpr struct LabelPropertyIndexDrop { + } label_property_index_drop; + static constexpr struct LabelPropertyIndexStatsSet { + } label_property_index_stats_set; + static constexpr struct LabelPropertyIndexStatsClear { + } label_property_index_stats_clear; + static constexpr struct ExistenceConstraintCreate { + } existence_constraint_create; + static constexpr struct ExistenceConstraintDrop { + } existence_constraint_drop; + static constexpr struct UniqueConstraintCreate { + } unique_constraint_create; + static constexpr struct UniqueConstraintDrop { + } unique_constraint_drop; + + MetadataDelta(LabelIndexCreate /*tag*/, LabelId label) : action(Action::LABEL_INDEX_CREATE), label(label) {} + + MetadataDelta(LabelIndexDrop /*tag*/, LabelId label) : action(Action::LABEL_INDEX_DROP), label(label) {} + + MetadataDelta(LabelIndexStatsSet /*tag*/, LabelId label, LabelIndexStats stats) + : action(Action::LABEL_INDEX_STATS_SET), label_stats{label, stats} {} + + MetadataDelta(LabelIndexStatsClear /*tag*/, LabelId label) : action(Action::LABEL_INDEX_STATS_CLEAR), label{label} {} + + MetadataDelta(LabelPropertyIndexCreate /*tag*/, LabelId label, PropertyId property) + : action(Action::LABEL_PROPERTY_INDEX_CREATE), label_property{label, property} {} + + MetadataDelta(LabelPropertyIndexDrop /*tag*/, LabelId label, PropertyId property) + : action(Action::LABEL_PROPERTY_INDEX_DROP), label_property{label, property} {} + + MetadataDelta(LabelPropertyIndexStatsSet /*tag*/, LabelId label, PropertyId property, LabelPropertyIndexStats stats) + : action(Action::LABEL_PROPERTY_INDEX_STATS_SET), label_property_stats{label, property, stats} {} + + MetadataDelta(LabelPropertyIndexStatsClear /*tag*/, LabelId label) + : action(Action::LABEL_PROPERTY_INDEX_STATS_CLEAR), label{label} {} + + MetadataDelta(ExistenceConstraintCreate /*tag*/, LabelId label, PropertyId property) + : action(Action::EXISTENCE_CONSTRAINT_CREATE), label_property{label, property} {} + + MetadataDelta(ExistenceConstraintDrop /*tag*/, LabelId label, PropertyId property) + : action(Action::EXISTENCE_CONSTRAINT_DROP), label_property{label, property} {} + + MetadataDelta(UniqueConstraintCreate /*tag*/, LabelId label, std::set properties) + : action(Action::UNIQUE_CONSTRAINT_CREATE), label_properties{label, std::move(properties)} {} + + MetadataDelta(UniqueConstraintDrop /*tag*/, LabelId label, std::set properties) + : action(Action::UNIQUE_CONSTRAINT_DROP), label_properties{label, std::move(properties)} {} + + MetadataDelta(const MetadataDelta &) = delete; + MetadataDelta(MetadataDelta &&) = delete; + MetadataDelta &operator=(const MetadataDelta &) = delete; + MetadataDelta &operator=(MetadataDelta &&) = delete; + + ~MetadataDelta() { + switch (action) { + case Action::LABEL_INDEX_CREATE: + case Action::LABEL_INDEX_DROP: + case Action::LABEL_INDEX_STATS_SET: + case Action::LABEL_INDEX_STATS_CLEAR: + case Action::LABEL_PROPERTY_INDEX_CREATE: + case Action::LABEL_PROPERTY_INDEX_DROP: + case Action::LABEL_PROPERTY_INDEX_STATS_SET: + case Action::LABEL_PROPERTY_INDEX_STATS_CLEAR: + case Action::EXISTENCE_CONSTRAINT_CREATE: + case Action::EXISTENCE_CONSTRAINT_DROP: + break; + case Action::UNIQUE_CONSTRAINT_CREATE: + case Action::UNIQUE_CONSTRAINT_DROP: + label_properties.properties.~set(); + break; + } + } + + Action action; + + union { + LabelId label; + + struct { + LabelId label; + PropertyId property; + } label_property; + + struct { + LabelId label; + std::set properties; + } label_properties; + + struct { + LabelId label; + LabelIndexStats stats; + } label_stats; + + struct { + LabelId label; + PropertyId property; + LabelPropertyIndexStats stats; + } label_property_stats; + }; +}; + +static_assert(alignof(MetadataDelta) >= 8, "The Delta should be aligned to at least 8!"); + +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/enums.hpp b/src/storage/v2/replication/enums.hpp index bbe8ffc62..e89a1fdd3 100644 --- a/src/storage/v2/replication/enums.hpp +++ b/src/storage/v2/replication/enums.hpp @@ -13,11 +13,7 @@ #include namespace memgraph::storage::replication { -enum class ReplicationRole : uint8_t { MAIN, REPLICA }; - -enum class ReplicationMode : std::uint8_t { SYNC, ASYNC }; enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID }; -enum class RegistrationMode : std::uint8_t { MUST_BE_INSTANTLY_VALID, CAN_BE_INVALID }; } // namespace memgraph::storage::replication diff --git a/src/storage/v2/replication/global.hpp b/src/storage/v2/replication/global.hpp index 820362740..7892fb990 100644 --- a/src/storage/v2/replication/global.hpp +++ b/src/storage/v2/replication/global.hpp @@ -17,38 +17,13 @@ #include #include "io/network/endpoint.hpp" +#include "replication/config.hpp" #include "storage/v2/replication/enums.hpp" #include "utils/uuid.hpp" // TODO: move to replication namespace and unify namespace memgraph::storage { -// TODO: Should be at MAIN instance level; shouldn't be connected to storage -struct ReplicationEpoch { - ReplicationEpoch() : id(utils::GenerateUUID()) {} - - // UUID to distinguish different main instance runs for replication process - // on SAME storage. - // Multiple instances can have same storage UUID and be MAIN at the same time. - // We cannot compare commit timestamps of those instances if one of them - // becomes the replica of the other so we use epoch_id_ as additional - // discriminating property. - // Example of this: - // We have 2 instances of the same storage, S1 and S2. - // S1 and S2 are MAIN and accept their own commits and write them to the WAL. - // At the moment when S1 commited a transaction with timestamp 20, and S2 - // a different transaction with timestamp 15, we change S2's role to REPLICA - // and register it on S1. - // Without using the epoch_id, we don't know that S1 and S2 have completely - // different transactions, we think that the S2 is behind only by 5 commits. - std::string id; // TODO: Move to replication level - - // Generates a new epoch id, returning the old one - std::string NewEpoch() { return std::exchange(id, utils::GenerateUUID()); } - - std::string SetEpoch(std::string new_epoch) { return std::exchange(id, std::move(new_epoch)); } -}; - struct TimestampInfo { uint64_t current_timestamp_of_replica; uint64_t current_number_of_timestamp_behind_master; @@ -56,7 +31,7 @@ struct TimestampInfo { struct ReplicaInfo { std::string name; - replication::ReplicationMode mode; + memgraph::replication::ReplicationMode mode; io::network::Endpoint endpoint; replication::ReplicaState state; TimestampInfo timestamp_info; diff --git a/src/storage/v2/replication/replication.cpp b/src/storage/v2/replication/replication.cpp deleted file mode 100644 index 59830a9ec..000000000 --- a/src/storage/v2/replication/replication.cpp +++ /dev/null @@ -1,383 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#include "storage/v2/replication/replication.hpp" -#include "storage/v2/constraints/constraints.hpp" -#include "storage/v2/durability/durability.hpp" -#include "storage/v2/durability/snapshot.hpp" -#include "storage/v2/replication/replication_client.hpp" -#include "storage/v2/replication/replication_server.hpp" -#include "storage/v2/storage.hpp" - -namespace memgraph::storage { - -using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; - -namespace { - -std::string RegisterReplicaErrorToString(RegisterReplicaError error) { - using enum RegisterReplicaError; - switch (error) { - case NAME_EXISTS: - return "NAME_EXISTS"; - case END_POINT_EXISTS: - return "END_POINT_EXISTS"; - case CONNECTION_FAILED: - return "CONNECTION_FAILED"; - case COULD_NOT_BE_PERSISTED: - return "COULD_NOT_BE_PERSISTED"; - } -} -} // namespace - -storage::ReplicationState::ReplicationState(bool restore, std::filesystem::path durability_dir) { - if (restore) { - utils::EnsureDirOrDie(durability_dir / durability::kReplicationDirectory); - durability_ = std::make_unique(durability_dir / durability::kReplicationDirectory); - } -} - -void storage::ReplicationState::Reset() { - replication_server_.reset(); - replication_clients_.WithLock([&](auto &clients) { clients.clear(); }); -} - -bool storage::ReplicationState::SetMainReplicationRole(storage::Storage *storage) { - // We don't want to generate new epoch_id and do the - // cleanup if we're already a MAIN - if (GetRole() == replication::ReplicationRole::MAIN) { - return false; - } - - // Main instance does not need replication server - // This should be always called first so we finalize everything - replication_server_.reset(nullptr); - - storage->EstablishNewEpoch(); - - if (ShouldStoreAndRestoreReplicationState()) { - // Only thing that matters here is the role saved as MAIN - auto data = replication::ReplicationStatusToJSON( - replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName, - .ip_address = "", - .port = 0, - .sync_mode = replication::ReplicationMode::SYNC, - .replica_check_frequency = std::chrono::seconds(0), - .ssl = std::nullopt, - .role = replication::ReplicationRole::MAIN}); - - if (!durability_->Put(replication::kReservedReplicationRoleName, data.dump())) { - spdlog::error("Error when saving MAIN replication role in settings."); - return false; - } - } - - SetRole(replication::ReplicationRole::MAIN); - return true; -} - -bool storage::ReplicationState::AppendOperation(const uint64_t seq_num, durability::StorageGlobalOperation operation, - LabelId label, const std::set &properties, - uint64_t final_commit_timestamp) { - bool finalized_on_all_replicas = true; - // TODO Should we return true if not MAIN? - if (GetRole() == replication::ReplicationRole::MAIN) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->StartTransactionReplication(seq_num); - client->IfStreamingTransaction( - [&](auto &stream) { stream.AppendOperation(operation, label, properties, final_commit_timestamp); }); - - const auto finalized = client->FinalizeTransactionReplication(); - if (client->Mode() == replication::ReplicationMode::SYNC) { - finalized_on_all_replicas = finalized && finalized_on_all_replicas; - } - } - }); - } - return finalized_on_all_replicas; -} - -void storage::ReplicationState::InitializeTransaction(uint64_t seq_num) { - if (GetRole() == replication::ReplicationRole::MAIN) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->StartTransactionReplication(seq_num); - } - }); - } -} - -void storage::ReplicationState::AppendDelta(const Delta &delta, const Edge &parent, uint64_t timestamp) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->IfStreamingTransaction([&](auto &stream) { stream.AppendDelta(delta, parent, timestamp); }); - } - }); -} - -void storage::ReplicationState::AppendDelta(const Delta &delta, const Vertex &parent, uint64_t timestamp) { - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->IfStreamingTransaction([&](auto &stream) { stream.AppendDelta(delta, parent, timestamp); }); - } - }); -} - -bool storage::ReplicationState::FinalizeTransaction(uint64_t timestamp) { - bool finalized_on_all_replicas = true; - replication_clients_.WithLock([&](auto &clients) { - for (auto &client : clients) { - client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(timestamp); }); - const auto finalized = client->FinalizeTransactionReplication(); - - if (client->Mode() == replication::ReplicationMode::SYNC) { - finalized_on_all_replicas = finalized && finalized_on_all_replicas; - } - } - }); - return finalized_on_all_replicas; -} - -utils::BasicResult ReplicationState::RegisterReplica( - const replication::RegistrationMode registration_mode, const replication::ReplicationClientConfig &config, - Storage *storage) { - MG_ASSERT(GetRole() == replication::ReplicationRole::MAIN, "Only main instance can register a replica!"); - - auto name_check = [&config](auto &clients) { - auto name_matches = [&name = config.name](const auto &client) { return client->Name() == name; }; - return std::any_of(clients.begin(), clients.end(), name_matches); - }; - - auto desired_endpoint = io::network::Endpoint{config.ip_address, config.port}; - auto endpoint_check = [&](auto &clients) { - auto endpoint_matches = [&](const auto &client) { return client->Endpoint() == desired_endpoint; }; - return std::any_of(clients.begin(), clients.end(), endpoint_matches); - }; - - auto task = [&](auto &clients) -> utils::BasicResult { - if (name_check(clients)) { - return RegisterReplicaError::NAME_EXISTS; - } - - if (endpoint_check(clients)) { - return RegisterReplicaError::END_POINT_EXISTS; - } - - if (!TryPersistReplicaClient(config)) { - return RegisterReplicaError::COULD_NOT_BE_PERSISTED; - } - - auto client = storage->CreateReplicationClient(config); - client->Start(); - - if (client->State() == replication::ReplicaState::INVALID) { - if (replication::RegistrationMode::CAN_BE_INVALID != registration_mode) { - return RegisterReplicaError::CONNECTION_FAILED; - } - - spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", client->Name()); - } - - clients.push_back(std::move(client)); - return {}; - }; - - return replication_clients_.WithLock(task); -} - -bool ReplicationState::TryPersistReplicaClient(const replication::ReplicationClientConfig &config) { - if (!ShouldStoreAndRestoreReplicationState()) return true; - auto data = replication::ReplicationStatusToJSON( - replication::ReplicationStatus{.name = config.name, - .ip_address = config.ip_address, - .port = config.port, - .sync_mode = config.mode, - .replica_check_frequency = config.replica_check_frequency, - .ssl = config.ssl, - .role = replication::ReplicationRole::REPLICA}); - if (durability_->Put(config.name, data.dump())) return true; - spdlog::error("Error when saving replica {} in settings.", config.name); - return false; -} - -bool ReplicationState::SetReplicaRole(const replication::ReplicationServerConfig &config, Storage *storage) { - // We don't want to restart the server if we're already a REPLICA - if (GetRole() == replication::ReplicationRole::REPLICA) { - return false; - } - - replication_server_ = storage->CreateReplicationServer(config); - bool res = replication_server_->Start(); - if (!res) { - spdlog::error("Unable to start the replication server."); - return false; - } - - if (ShouldStoreAndRestoreReplicationState()) { - // Only thing that matters here is the role saved as REPLICA and the listening port - auto data = replication::ReplicationStatusToJSON( - replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName, - .ip_address = config.ip_address, - .port = config.port, - .sync_mode = replication::ReplicationMode::SYNC, - .replica_check_frequency = std::chrono::seconds(0), - .ssl = std::nullopt, - .role = replication::ReplicationRole::REPLICA}); - - if (!durability_->Put(replication::kReservedReplicationRoleName, data.dump())) { - spdlog::error("Error when saving REPLICA replication role in settings."); - return false; - } - } - - SetRole(replication::ReplicationRole::REPLICA); - return true; -} - -bool ReplicationState::UnregisterReplica(std::string_view name) { - MG_ASSERT(GetRole() == replication::ReplicationRole::MAIN, "Only main instance can unregister a replica!"); - if (ShouldStoreAndRestoreReplicationState()) { - if (!durability_->Delete(name)) { - spdlog::error("Error when removing replica {} from settings.", name); - return false; - } - } - - return replication_clients_.WithLock([&](auto &clients) { - return std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); - }); -} - -std::optional ReplicationState::GetReplicaState(const std::string_view name) { - return replication_clients_.WithLock([&](auto &clients) -> std::optional { - const auto client_it = - std::find_if(clients.cbegin(), clients.cend(), [name](auto &client) { return client->Name() == name; }); - if (client_it == clients.cend()) { - return std::nullopt; - } - return (*client_it)->State(); - }); -} - -std::vector ReplicationState::ReplicasInfo() { - return replication_clients_.WithLock([](auto &clients) { - std::vector replica_info; - replica_info.reserve(clients.size()); - std::transform( - clients.begin(), clients.end(), std::back_inserter(replica_info), [](const auto &client) -> ReplicaInfo { - return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo()}; - }); - return replica_info; - }); -} - -void ReplicationState::RestoreReplicationRole(Storage *storage) { - if (!ShouldStoreAndRestoreReplicationState()) { - return; - } - - spdlog::info("Restoring replication role."); - uint16_t port = replication::kDefaultReplicationPort; - - const auto replication_data = durability_->Get(replication::kReservedReplicationRoleName); - if (!replication_data.has_value()) { - spdlog::debug("Cannot find data needed for restore replication role in persisted metadata."); - return; - } - - const auto maybe_replication_status = replication::JSONToReplicationStatus(nlohmann::json::parse(*replication_data)); - if (!maybe_replication_status.has_value()) { - LOG_FATAL("Cannot parse previously saved configuration of replication role {}.", - replication::kReservedReplicationRoleName); - } - - const auto replication_status = *maybe_replication_status; - if (!replication_status.role.has_value()) { - SetRole(replication::ReplicationRole::MAIN); - } else { - SetRole(*replication_status.role); - port = replication_status.port; - } - - if (GetRole() == replication::ReplicationRole::REPLICA) { - replication_server_ = storage->CreateReplicationServer(replication::ReplicationServerConfig{ - .ip_address = replication::kDefaultReplicationServerIp, - .port = port, - }); - bool res = replication_server_->Start(); - if (!res) { - LOG_FATAL("Unable to start the replication server."); - } - } - - spdlog::info("Replication role restored to {}.", - GetRole() == replication::ReplicationRole::MAIN ? "MAIN" : "REPLICA"); -} - -void ReplicationState::RestoreReplicas(Storage *storage) { - if (!ShouldStoreAndRestoreReplicationState()) { - return; - } - spdlog::info("Restoring replicas."); - - for (const auto &[replica_name, replica_data] : *durability_) { - spdlog::info("Restoring replica {}.", replica_name); - - const auto maybe_replica_status = replication::JSONToReplicationStatus(nlohmann::json::parse(replica_data)); - if (!maybe_replica_status.has_value()) { - LOG_FATAL("Cannot parse previously saved configuration of replica {}.", replica_name); - } - - auto replica_status = *maybe_replica_status; - MG_ASSERT(replica_status.name == replica_name, "Expected replica name is '{}', but got '{}'", replica_status.name, - replica_name); - - if (replica_name == replication::kReservedReplicationRoleName) { - continue; - } - - auto ret = RegisterReplica(replication::RegistrationMode::CAN_BE_INVALID, - replication::ReplicationClientConfig{ - .name = replica_status.name, - .mode = replica_status.sync_mode, - .ip_address = replica_status.ip_address, - .port = replica_status.port, - .replica_check_frequency = replica_status.replica_check_frequency, - .ssl = replica_status.ssl, - }, - storage); - - if (ret.HasError()) { - MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError()); - LOG_FATAL("Failure when restoring replica {}: {}.", replica_name, RegisterReplicaErrorToString(ret.GetError())); - } - spdlog::info("Replica {} restored.", replica_name); - } -} - -constexpr uint16_t kEpochHistoryRetention = 1000; - -void ReplicationState::NewEpoch() { - // Generate new epoch id and save the last one to the history. - if (history.size() == kEpochHistoryRetention) { - history.pop_front(); - } - auto prevEpoch = epoch_.NewEpoch(); - history.emplace_back(std::move(prevEpoch), last_commit_timestamp_); -} - -void ReplicationState::AppendEpoch(std::string new_epoch) { - auto prevEpoch = epoch_.SetEpoch(std::move(new_epoch)); - history.emplace_back(std::move(prevEpoch), last_commit_timestamp_); -} - -} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication.hpp b/src/storage/v2/replication/replication.hpp deleted file mode 100644 index afa3f7d06..000000000 --- a/src/storage/v2/replication/replication.hpp +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2023 Memgraph Ltd. -// -// Use of this software is governed by the Business Source License -// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source -// License, and you may not use this file except in compliance with the Business Source License. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0, included in the file -// licenses/APL.txt. - -#pragma once - -#include "kvstore/kvstore.hpp" -#include "storage/v2/delta.hpp" -#include "storage/v2/durability/storage_global_operation.hpp" -#include "storage/v2/transaction.hpp" -#include "utils/result.hpp" - -/// REPLICATION /// -#include "storage/v2/replication/config.hpp" -#include "storage/v2/replication/enums.hpp" -#include "storage/v2/replication/global.hpp" -#include "storage/v2/replication/replication_persistence_helper.hpp" -#include "storage/v2/replication/rpc.hpp" -#include "storage/v2/replication/serialization.hpp" - -// TODO use replication namespace -namespace memgraph::storage { - -class Storage; -class ReplicationServer; -class ReplicationClient; - -enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED }; - -struct ReplicationState { - // TODO: This mirrors the logic in InMemoryConstructor; make it independent - ReplicationState(bool restore, std::filesystem::path durability_dir); - - // Generic API - void Reset(); - // TODO: Just check if server exists -> you are REPLICA - replication::ReplicationRole GetRole() const { return replication_role_.load(); } - - bool SetMainReplicationRole(Storage *storage); // Set the instance to MAIN - // TODO: ReplicationServer/Client uses Storage* for RPC callbacks - bool SetReplicaRole(const replication::ReplicationServerConfig &config, - Storage *storage); // Sets the instance to REPLICA - // Generic restoration - void RestoreReplicationRole(Storage *storage); - - // MAIN actually doing the replication - bool AppendOperation(uint64_t seq_num, durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t final_commit_timestamp); - void InitializeTransaction(uint64_t seq_num); - void AppendDelta(const Delta &delta, const Vertex &parent, uint64_t timestamp); - void AppendDelta(const Delta &delta, const Edge &parent, uint64_t timestamp); - bool FinalizeTransaction(uint64_t timestamp); - - // MAIN connecting to replicas - utils::BasicResult RegisterReplica(const replication::RegistrationMode registration_mode, - const replication::ReplicationClientConfig &config, - Storage *storage); - bool UnregisterReplica(std::string_view name); - - // MAIN reconnecting to replicas - void RestoreReplicas(Storage *storage); - - // MAIN getting info from replicas - // TODO make into const (problem with SpinLock and WithReadLock) - std::optional GetReplicaState(std::string_view name); - std::vector ReplicasInfo(); - - const ReplicationEpoch &GetEpoch() const { return epoch_; } - ReplicationEpoch &GetEpoch() { return epoch_; } - - // Questions: - // - storage durability <- databases/*name*/wal and snapshots (where this for epoch_id) - // - multi-tenant durability <- databases/.durability (there is a list of all active tenants) - // History of the previous epoch ids. - // Each value consists of the epoch id along the last commit belonging to that - // epoch. - std::deque> history; - - // TODO: actually durability - std::atomic last_commit_timestamp_{kTimestampInitialId}; - - void NewEpoch(); - void AppendEpoch(std::string new_epoch); - - private: - bool TryPersistReplicaClient(const replication::ReplicationClientConfig &config); - bool ShouldStoreAndRestoreReplicationState() const { return nullptr != durability_; } - void SetRole(replication::ReplicationRole role) { return replication_role_.store(role); } - - // NOTE: Server is not in MAIN it is in REPLICA - std::unique_ptr replication_server_{nullptr}; - - // We create ReplicationClient using unique_ptr so we can move - // newly created client into the vector. - // We cannot move the client directly because it contains ThreadPool - // which cannot be moved. Also, the move is necessary because - // we don't want to create the client directly inside the vector - // because that would require the lock on the list putting all - // commits (they iterate list of clients) to halt. - // This way we can initialize client in main thread which means - // that we can immediately notify the user if the initialization - // failed. - using ReplicationClientList = utils::Synchronized>, utils::SpinLock>; - ReplicationClientList replication_clients_; - - std::atomic replication_role_{replication::ReplicationRole::MAIN}; - - std::unique_ptr durability_; - - ReplicationEpoch epoch_; -}; - -} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_client.cpp b/src/storage/v2/replication/replication_client.cpp index e4817929f..c2b099ff1 100644 --- a/src/storage/v2/replication/replication_client.cpp +++ b/src/storage/v2/replication/replication_client.cpp @@ -15,28 +15,25 @@ #include #include "storage/v2/durability/durability.hpp" -#include "storage/v2/replication/config.hpp" -#include "storage/v2/replication/enums.hpp" #include "storage/v2/storage.hpp" -#include "storage/v2/transaction.hpp" -#include "utils/file_locker.hpp" -#include "utils/logging.hpp" -#include "utils/message.hpp" namespace memgraph::storage { -static auto CreateClientContext(const replication::ReplicationClientConfig &config) -> communication::ClientContext { +static auto CreateClientContext(const memgraph::replication::ReplicationClientConfig &config) + -> communication::ClientContext { return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file} : communication::ClientContext{}; } -ReplicationClient::ReplicationClient(Storage *storage, replication::ReplicationClientConfig const &config) +ReplicationClient::ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, + const memgraph::replication::ReplicationEpoch *epoch) : name_{config.name}, rpc_context_{CreateClientContext(config)}, rpc_client_{io::network::Endpoint(config.ip_address, config.port), &rpc_context_}, replica_check_frequency_{config.replica_check_frequency}, mode_{config.mode}, - storage_{storage} {} + storage_{storage}, + repl_epoch_{epoch} {} ReplicationClient::~ReplicationClient() { auto endpoint = rpc_client_.Endpoint(); @@ -45,21 +42,19 @@ ReplicationClient::~ReplicationClient() { } uint64_t ReplicationClient::LastCommitTimestamp() const { - return storage_->replication_state_.last_commit_timestamp_.load(); + return storage_->repl_storage_state_.last_commit_timestamp_.load(); } void ReplicationClient::InitializeClient() { uint64_t current_commit_timestamp{kTimestampInitialId}; - const auto &main_epoch = storage_->replication_state_.GetEpoch(); - - auto stream{rpc_client_.Stream(storage_->replication_state_.last_commit_timestamp_, - main_epoch.id)}; + auto stream{rpc_client_.Stream(storage_->repl_storage_state_.last_commit_timestamp_, + std::string{repl_epoch_->id()})}; const auto replica = stream.AwaitResponse(); std::optional branching_point; - if (replica.epoch_id != main_epoch.id && replica.current_commit_timestamp != kTimestampInitialId) { - auto const &history = storage_->replication_state_.history; + if (replica.epoch_id != repl_epoch_->id() && replica.current_commit_timestamp != kTimestampInitialId) { + auto const &history = storage_->repl_storage_state_.history; const auto epoch_info_iter = std::find_if(history.crbegin(), history.crend(), [&](const auto &main_epoch_info) { return main_epoch_info.first == replica.epoch_id; }); @@ -81,8 +76,8 @@ void ReplicationClient::InitializeClient() { current_commit_timestamp = replica.current_commit_timestamp; spdlog::trace("Current timestamp on replica {}: {}", name_, current_commit_timestamp); - spdlog::trace("Current timestamp on main: {}", storage_->replication_state_.last_commit_timestamp_.load()); - if (current_commit_timestamp == storage_->replication_state_.last_commit_timestamp_.load()) { + spdlog::trace("Current timestamp on main: {}", storage_->repl_storage_state_.last_commit_timestamp_.load()); + if (current_commit_timestamp == storage_->repl_storage_state_.last_commit_timestamp_.load()) { spdlog::debug("Replica '{}' up to date", name_); std::unique_lock client_guard{client_lock_}; replica_state_.store(replication::ReplicaState::READY); @@ -109,7 +104,7 @@ TimestampInfo ReplicationClient::GetTimestampInfo() { replica_state_.store(replication::ReplicaState::INVALID); HandleRpcFailure(); } - auto main_time_stamp = storage_->replication_state_.last_commit_timestamp_.load(); + auto main_time_stamp = storage_->repl_storage_state_.last_commit_timestamp_.load(); info.current_timestamp_of_replica = response.current_commit_timestamp; info.current_number_of_timestamp_behind_master = response.current_commit_timestamp - main_time_stamp; } catch (const rpc::RpcFailedException &) { @@ -172,7 +167,7 @@ void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_s MG_ASSERT(!replica_stream_); try { replica_stream_.emplace( - ReplicaStream{this, storage_->replication_state_.last_commit_timestamp_.load(), current_wal_seq_num}); + ReplicaStream{this, storage_->repl_storage_state_.last_commit_timestamp_.load(), current_wal_seq_num}); replica_state_.store(replication::ReplicaState::REPLICATING); } catch (const rpc::RpcFailedException &) { replica_state_.store(replication::ReplicaState::INVALID); @@ -182,8 +177,6 @@ void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_s } } -auto ReplicationClient::GetEpochId() const -> std::string const & { return storage_->replication_state_.GetEpoch().id; } - bool ReplicationClient::FinalizeTransactionReplication() { // We can only check the state because it guarantees to be only // valid during a single transaction replication (if the assumption @@ -217,7 +210,7 @@ bool ReplicationClient::FinalizeTransactionReplication() { return false; }; - if (mode_ == replication::ReplicationMode::ASYNC) { + if (mode_ == memgraph::replication::ReplicationMode::ASYNC) { thread_pool_.AddTask([=] { (void)task(); }); return true; } @@ -289,7 +282,7 @@ ReplicaStream::ReplicaStream(ReplicationClient *self, const uint64_t previous_co stream_(self_->rpc_client_.Stream(previous_commit_timestamp, current_seq_num)) { replication::Encoder encoder{stream_.GetBuilder()}; - encoder.WriteString(self_->GetEpochId()); + encoder.WriteString(self->repl_epoch_->id()); } void ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp) { @@ -308,10 +301,12 @@ void ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) { EncodeTransactionEnd(&encoder, final_commit_timestamp); } -void ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t timestamp) { +void ReplicaStream::AppendOperation(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, uint64_t timestamp) { replication::Encoder encoder(stream_.GetBuilder()); - EncodeOperation(&encoder, self_->GetStorage()->name_id_mapper_.get(), operation, label, properties, timestamp); + EncodeOperation(&encoder, self_->GetStorage()->name_id_mapper_.get(), operation, label, properties, stats, + property_stats, timestamp); } replication::AppendDeltasRes ReplicaStream::Finalize() { return stream_.AwaitResponse(); } diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index fb3d1eb5c..e3c1b2634 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -11,10 +11,13 @@ #pragma once +#include "replication/config.hpp" +#include "replication/epoch.hpp" #include "rpc/client.hpp" #include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/id_types.hpp" -#include "storage/v2/replication/config.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/indices/label_property_index_stats.hpp" #include "storage/v2/replication/enums.hpp" #include "storage/v2/replication/global.hpp" #include "storage/v2/replication/rpc.hpp" @@ -50,8 +53,9 @@ class ReplicaStream { void AppendTransactionEnd(uint64_t final_commit_timestamp); /// @throw rpc::RpcFailedException - void AppendOperation(durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t timestamp); + void AppendOperation(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, uint64_t timestamp); /// @throw rpc::RpcFailedException replication::AppendDeltasRes Finalize(); @@ -66,7 +70,8 @@ class ReplicationClient { friend class ReplicaStream; public: - ReplicationClient(Storage *storage, replication::ReplicationClientConfig const &config); + ReplicationClient(Storage *storage, const memgraph::replication::ReplicationClientConfig &config, + const memgraph::replication::ReplicationEpoch *epoch); ReplicationClient(ReplicationClient const &) = delete; ReplicationClient &operator=(ReplicationClient const &) = delete; @@ -75,7 +80,7 @@ class ReplicationClient { virtual ~ReplicationClient(); - auto Mode() const -> replication::ReplicationMode { return mode_; } + auto Mode() const -> memgraph::replication::ReplicationMode { return mode_; } auto Name() const -> std::string const & { return name_; } auto Endpoint() const -> io::network::Endpoint const & { return rpc_client_.Endpoint(); } auto State() const -> replication::ReplicaState { return replica_state_.load(); } @@ -96,7 +101,6 @@ class ReplicationClient { virtual void RecoverReplica(uint64_t replica_commit) = 0; auto GetStorage() -> Storage * { return storage_; } - auto GetEpochId() const -> std::string const &; auto LastCommitTimestamp() const -> uint64_t; void InitializeClient(); void HandleRpcFailure(); @@ -110,7 +114,7 @@ class ReplicationClient { std::chrono::seconds replica_check_frequency_; std::optional replica_stream_; - replication::ReplicationMode mode_{replication::ReplicationMode::SYNC}; + memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC}; utils::SpinLock client_lock_; // This thread pool is used for background tasks so we don't @@ -131,6 +135,8 @@ class ReplicationClient { utils::Scheduler replica_checker_; Storage *storage_; + + memgraph::replication::ReplicationEpoch const *repl_epoch_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_handler.cpp b/src/storage/v2/replication/replication_handler.cpp new file mode 100644 index 000000000..4ba1edf69 --- /dev/null +++ b/src/storage/v2/replication/replication_handler.cpp @@ -0,0 +1,204 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/replication/replication_handler.hpp" + +#include "replication/state.hpp" +#include "storage/v2/storage.hpp" + +namespace memgraph::storage { + +namespace { + +std::string RegisterReplicaErrorToString(RegisterReplicaError error) { + switch (error) { + using enum RegisterReplicaError; + case NAME_EXISTS: + return "NAME_EXISTS"; + case END_POINT_EXISTS: + return "END_POINT_EXISTS"; + case CONNECTION_FAILED: + return "CONNECTION_FAILED"; + case COULD_NOT_BE_PERSISTED: + return "COULD_NOT_BE_PERSISTED"; + } +} +} // namespace + +bool ReplicationHandler::SetReplicationRoleMain() { + // We don't want to generate new epoch_id and do the + // cleanup if we're already a MAIN + // TODO: under lock + if (repl_state_.IsMain()) { + return false; + } + + // STEP 1) bring down all REPLICA servers + auto current_epoch = std::string(repl_state_.GetEpoch().id()); + { // TODO: foreach storage + // ensure replica server brought down + storage_.repl_storage_state_.replication_server_.reset(nullptr); + // Remember old epoch + storage timestamp association + storage_.repl_storage_state_.AddEpochToHistory(current_epoch); + } + + // STEP 2) Change to MAIN + repl_state_.GetEpoch().NewEpoch(); + if (!repl_state_.TryPersistRoleMain()) { + // TODO: On failure restore old epoch? restore replication servers? + return false; + } + repl_state_.SetRole(memgraph::replication::ReplicationRole::MAIN); + return true; +} +memgraph::utils::BasicResult ReplicationHandler::RegisterReplica( + const RegistrationMode registration_mode, const memgraph::replication::ReplicationClientConfig &config) { + MG_ASSERT(repl_state_.IsMain(), "Only main instance can register a replica!"); + + auto name_check = [&config](auto &clients) { + auto name_matches = [&name = config.name](const auto &client) { return client->Name() == name; }; + return std::any_of(clients.begin(), clients.end(), name_matches); + }; + + auto desired_endpoint = io::network::Endpoint{config.ip_address, config.port}; + auto endpoint_check = [&](auto &clients) { + auto endpoint_matches = [&](const auto &client) { return client->Endpoint() == desired_endpoint; }; + return std::any_of(clients.begin(), clients.end(), endpoint_matches); + }; + + auto task = [&](auto &clients) -> utils::BasicResult { + if (name_check(clients)) { + return RegisterReplicaError::NAME_EXISTS; + } + + if (endpoint_check(clients)) { + return RegisterReplicaError::END_POINT_EXISTS; + } + + using enum RegistrationMode; + if (registration_mode != RESTORE && !repl_state_.TryPersistRegisteredReplica(config)) { + return RegisterReplicaError::COULD_NOT_BE_PERSISTED; + } + + auto client = storage_.CreateReplicationClient(config); + client->Start(); + + if (client->State() == replication::ReplicaState::INVALID) { + if (registration_mode != RESTORE) { + return RegisterReplicaError::CONNECTION_FAILED; + } + + spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.", client->Name()); + } + + clients.push_back(std::move(client)); + return {}; + }; + + return storage_.repl_storage_state_.replication_clients_.WithLock(task); +} +bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) { + // We don't want to restart the server if we're already a REPLICA + if (repl_state_.IsReplica()) { + return false; + } + + std::unique_ptr replication_server = storage_.CreateReplicationServer(config); + bool res = replication_server->Start(); + if (!res) { + spdlog::error("Unable to start the replication server."); + return false; + } + storage_.repl_storage_state_.replication_server_ = std::move(replication_server); + + if (!repl_state_.TryPersistRoleReplica(config)) { + return false; + } + + repl_state_.SetRole(memgraph::replication::ReplicationRole::REPLICA); + return true; +} +auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterReplicaResult { + if (repl_state_.IsReplica()) { + return UnregisterReplicaResult::NOT_MAIN; + } + + if (!repl_state_.TryPersistUnregisterReplica(name)) { + return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED; + } + + auto const n_unregistered = storage_.repl_storage_state_.replication_clients_.WithLock([&](auto &clients) { + return std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); + }); + return (n_unregistered != 0) ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER; +} +void ReplicationHandler::RestoreReplication() { + if (!repl_state_.ShouldPersist()) { + return; + } + + spdlog::info("Restoring replication role."); + + using memgraph::replication::ReplicationState; + + auto replicationData = repl_state_.FetchReplicationData(); + if (replicationData.HasError()) { + switch (replicationData.GetError()) { + using enum ReplicationState::FetchReplicationError; + case NOTHING_FETCHED: { + spdlog::debug("Cannot find data needed for restore replication role in persisted metadata."); + return; + } + case PARSE_ERROR: { + LOG_FATAL("Cannot parse previously saved configuration of replication role."); + return; + } + } + } + + /// MAIN + auto const recover_main = [this](ReplicationState::ReplicationDataMain const &configs) { + storage_.repl_storage_state_.replication_server_.reset(); + repl_state_.SetRole(memgraph::replication::ReplicationRole::MAIN); + for (const auto &config : configs) { + spdlog::info("Replica {} restored for {}.", config.name, storage_.id()); + auto ret = RegisterReplica(RegistrationMode::RESTORE, config); + if (ret.HasError()) { + MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError()); + LOG_FATAL("Failure when restoring replica {}: {}.", config.name, RegisterReplicaErrorToString(ret.GetError())); + } + spdlog::info("Replica {} restored for {}.", config.name, storage_.id()); + } + spdlog::info("Replication role restored to MAIN."); + }; + + /// REPLICA + auto const recover_replica = [this](ReplicationState::ReplicationDataReplica const &config) { + auto replication_server = storage_.CreateReplicationServer(config); + if (!replication_server->Start()) { + LOG_FATAL("Unable to start the replication server."); + } + storage_.repl_storage_state_.replication_server_ = std::move(replication_server); + repl_state_.SetRole(memgraph::replication::ReplicationRole::REPLICA); + spdlog::info("Replication role restored to REPLICA."); + }; + + std::visit( + utils::Overloaded{ + recover_main, + recover_replica, + }, + *replicationData); +} +auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { return repl_state_.GetRole(); } +bool ReplicationHandler::IsMain() const { return repl_state_.IsMain(); } +bool ReplicationHandler::IsReplica() const { return repl_state_.IsReplica(); } +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_handler.hpp b/src/storage/v2/replication/replication_handler.hpp new file mode 100644 index 000000000..797f76b54 --- /dev/null +++ b/src/storage/v2/replication/replication_handler.hpp @@ -0,0 +1,71 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "replication/role.hpp" +#include "utils/result.hpp" + +// BEGIN fwd declares +namespace memgraph::replication { +struct ReplicationState; +struct ReplicationServerConfig; +struct ReplicationClientConfig; +} // namespace memgraph::replication +namespace memgraph::storage { +class Storage; +} +// END fwd declares + +namespace memgraph::storage { + +enum class RegistrationMode : std::uint8_t { MUST_BE_INSTANTLY_VALID, RESTORE }; +enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED }; +enum class UnregisterReplicaResult : uint8_t { + NOT_MAIN, + COULD_NOT_BE_PERSISTED, + CAN_NOT_UNREGISTER, + SUCCESS, +}; + +/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage +/// TODO: extend to do multiple storages +struct ReplicationHandler { + ReplicationHandler(memgraph::replication::ReplicationState &replState, Storage &storage) + : repl_state_(replState), storage_(storage) {} + + // as REPLICA, become MAIN + bool SetReplicationRoleMain(); + + // as MAIN, become REPLICA + bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config); + + // as MAIN, define and connect to REPLICAs + auto RegisterReplica(RegistrationMode registration_mode, const memgraph::replication::ReplicationClientConfig &config) + -> utils::BasicResult; + + // as MAIN, remove a REPLICA connection + auto UnregisterReplica(std::string_view name) -> UnregisterReplicaResult; + + // Generic restoration + // TODO: decouple storage restoration from epoch restoration + void RestoreReplication(); + + // Helper pass-through (TODO: remove) + auto GetRole() const -> memgraph::replication::ReplicationRole; + bool IsMain() const; + bool IsReplica() const; + + private: + memgraph::replication::ReplicationState &repl_state_; + Storage &storage_; +}; +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_server.cpp b/src/storage/v2/replication/replication_server.cpp index f64e2cd16..0ed367d88 100644 --- a/src/storage/v2/replication/replication_server.cpp +++ b/src/storage/v2/replication/replication_server.cpp @@ -11,13 +11,13 @@ #include "replication_server.hpp" #include "io/network/endpoint.hpp" +#include "replication/config.hpp" #include "rpc.hpp" -#include "storage/v2/replication/config.hpp" namespace memgraph::storage { namespace { -auto CreateServerContext(const replication::ReplicationServerConfig &config) -> communication::ServerContext { +auto CreateServerContext(const memgraph::replication::ReplicationServerConfig &config) -> communication::ServerContext { return (config.ssl) ? communication::ServerContext{config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file, config.ssl->verify_peer} : communication::ServerContext{}; @@ -30,7 +30,7 @@ auto CreateServerContext(const replication::ReplicationServerConfig &config) -> constexpr auto kReplictionServerThreads = 1; } // namespace -ReplicationServer::ReplicationServer(const replication::ReplicationServerConfig &config) +ReplicationServer::ReplicationServer(const memgraph::replication::ReplicationServerConfig &config) : rpc_server_context_{CreateServerContext(config)}, rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_, kReplictionServerThreads} { diff --git a/src/storage/v2/replication/replication_server.hpp b/src/storage/v2/replication/replication_server.hpp index ca036fd3a..0d7415396 100644 --- a/src/storage/v2/replication/replication_server.hpp +++ b/src/storage/v2/replication/replication_server.hpp @@ -11,16 +11,16 @@ #pragma once +#include "replication/config.hpp" #include "rpc/server.hpp" #include "slk/streams.hpp" -#include "storage/v2/replication/config.hpp" #include "storage/v2/replication/global.hpp" namespace memgraph::storage { class ReplicationServer { public: - explicit ReplicationServer(const replication::ReplicationServerConfig &config); + explicit ReplicationServer(const memgraph::replication::ReplicationServerConfig &config); ReplicationServer(const ReplicationServer &) = delete; ReplicationServer(ReplicationServer &&) = delete; ReplicationServer &operator=(const ReplicationServer &) = delete; diff --git a/src/storage/v2/replication/replication_storage_state.cpp b/src/storage/v2/replication/replication_storage_state.cpp new file mode 100644 index 000000000..5779bd2ba --- /dev/null +++ b/src/storage/v2/replication/replication_storage_state.cpp @@ -0,0 +1,111 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/replication/replication_storage_state.hpp" + +#include "storage/v2/replication/replication_client.hpp" +#include "storage/v2/replication/replication_server.hpp" + +namespace memgraph::storage { + +void ReplicationStorageState::InitializeTransaction(uint64_t seq_num) { + replication_clients_.WithLock([&](auto &clients) { + for (auto &client : clients) { + client->StartTransactionReplication(seq_num); + } + }); +} + +void ReplicationStorageState::AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp) { + replication_clients_.WithLock([&](auto &clients) { + for (auto &client : clients) { + client->IfStreamingTransaction([&](auto &stream) { stream.AppendDelta(delta, vertex, timestamp); }); + } + }); +} + +void ReplicationStorageState::AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp) { + replication_clients_.WithLock([&](auto &clients) { + for (auto &client : clients) { + client->IfStreamingTransaction([&](auto &stream) { stream.AppendDelta(delta, edge, timestamp); }); + } + }); +} +void ReplicationStorageState::AppendOperation(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, + uint64_t final_commit_timestamp) { + replication_clients_.WithLock([&](auto &clients) { + for (auto &client : clients) { + client->IfStreamingTransaction([&](auto &stream) { + stream.AppendOperation(operation, label, properties, stats, property_stats, final_commit_timestamp); + }); + } + }); +} + +bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp) { + return replication_clients_.WithLock([=](auto &clients) { + bool finalized_on_all_replicas = true; + for (ReplicationClientPtr &client : clients) { + client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(timestamp); }); + const auto finalized = client->FinalizeTransactionReplication(); + + if (client->Mode() == memgraph::replication::ReplicationMode::SYNC) { + finalized_on_all_replicas = finalized && finalized_on_all_replicas; + } + } + return finalized_on_all_replicas; + }); +} + +std::optional ReplicationStorageState::GetReplicaState(std::string_view name) const { + return replication_clients_.WithReadLock([&](auto const &clients) -> std::optional { + auto const name_matches = [=](ReplicationClientPtr const &client) { return client->Name() == name; }; + auto const client_it = std::find_if(clients.cbegin(), clients.cend(), name_matches); + if (client_it == clients.cend()) { + return std::nullopt; + } + return (*client_it)->State(); + }); +} + +std::vector ReplicationStorageState::ReplicasInfo() const { + return replication_clients_.WithReadLock([](auto const &clients) { + std::vector replica_infos; + replica_infos.reserve(clients.size()); + auto const asReplicaInfo = [](ReplicationClientPtr const &client) -> ReplicaInfo { + return {client->Name(), client->Mode(), client->Endpoint(), client->State(), client->GetTimestampInfo()}; + }; + std::transform(clients.begin(), clients.end(), std::back_inserter(replica_infos), asReplicaInfo); + return replica_infos; + }); +} + +void ReplicationStorageState::Reset() { + replication_server_.reset(); + replication_clients_.WithLock([](auto &clients) { clients.clear(); }); +} + +void ReplicationStorageState::AddEpochToHistory(std::string prev_epoch) { + constexpr uint16_t kEpochHistoryRetention = 1000; + // Generate new epoch id and save the last one to the history. + if (history.size() == kEpochHistoryRetention) { + history.pop_front(); + } + history.emplace_back(std::move(prev_epoch), last_commit_timestamp_); +} + +void ReplicationStorageState::AddEpochToHistoryForce(std::string prev_epoch) { + history.emplace_back(std::move(prev_epoch), last_commit_timestamp_); +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/replication_storage_state.hpp b/src/storage/v2/replication/replication_storage_state.hpp new file mode 100644 index 000000000..c3dc7c833 --- /dev/null +++ b/src/storage/v2/replication/replication_storage_state.hpp @@ -0,0 +1,85 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "kvstore/kvstore.hpp" +#include "storage/v2/delta.hpp" +#include "storage/v2/durability/storage_global_operation.hpp" +#include "storage/v2/transaction.hpp" +#include "utils/result.hpp" + +/// REPLICATION /// +#include "replication/config.hpp" +#include "replication/epoch.hpp" +#include "replication/state.hpp" +#include "storage/v2/replication/enums.hpp" +#include "storage/v2/replication/global.hpp" +#include "storage/v2/replication/rpc.hpp" +#include "storage/v2/replication/serialization.hpp" + +namespace memgraph::storage { + +class Storage; +class ReplicationServer; +class ReplicationClient; + +struct ReplicationStorageState { + // Only MAIN can send + void InitializeTransaction(uint64_t seq_num); + void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t timestamp); + void AppendDelta(const Delta &delta, const Edge &edge, uint64_t timestamp); + void AppendOperation(durability::StorageMetadataOperation operation, LabelId label, + const std::set &properties, const LabelIndexStats &stats, + const LabelPropertyIndexStats &property_stats, uint64_t final_commit_timestamp); + bool FinalizeTransaction(uint64_t timestamp); + + // Getters + auto GetReplicaState(std::string_view name) const -> std::optional; + auto ReplicasInfo() const -> std::vector; + + // History + void AddEpochToHistory(std::string prev_epoch); + void AddEpochToHistoryForce(std::string prev_epoch); + + void Reset(); + + // Questions: + // - storage durability <- databases/*name*/wal and snapshots (where this for epoch_id) + // - multi-tenant durability <- databases/.durability (there is a list of all active tenants) + // History of the previous epoch ids. + // Each value consists of the epoch id along the last commit belonging to that + // epoch. + std::deque> history; + std::atomic last_commit_timestamp_{kTimestampInitialId}; + + // We create ReplicationClient using unique_ptr so we can move + // newly created client into the vector. + // We cannot move the client directly because it contains ThreadPool + // which cannot be moved. Also, the move is necessary because + // we don't want to create the client directly inside the vector + // because that would require the lock on the list putting all + // commits (they iterate list of clients) to halt. + // This way we can initialize client in main thread which means + // that we can immediately notify the user if the initialization + // failed. + using ReplicationClientPtr = std::unique_ptr; + using ReplicationClientList = utils::Synchronized, utils::RWSpinLock>; + + // NOTE: Server is not in MAIN it is in REPLICA + std::unique_ptr replication_server_{nullptr}; + + ReplicationClientList replication_clients_; +}; + +} // namespace memgraph::storage diff --git a/src/storage/v2/replication/rpc.hpp b/src/storage/v2/replication/rpc.hpp index f466b1880..d2cf21fb4 100644 --- a/src/storage/v2/replication/rpc.hpp +++ b/src/storage/v2/replication/rpc.hpp @@ -63,7 +63,7 @@ struct HeartbeatReq { static void Save(const HeartbeatReq &self, memgraph::slk::Builder *builder); HeartbeatReq() {} HeartbeatReq(uint64_t main_commit_timestamp, std::string epoch_id) - : main_commit_timestamp(main_commit_timestamp), epoch_id(epoch_id) {} + : main_commit_timestamp(main_commit_timestamp), epoch_id(std::move(epoch_id)) {} uint64_t main_commit_timestamp; std::string epoch_id; diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 27748ebba..11008d0a1 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include #include "absl/container/flat_hash_set.h" #include "spdlog/spdlog.h" @@ -39,8 +40,14 @@ class InMemoryStorage; using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; +auto ReplicationStateHelper(Config const &config) -> std::optional { + if (!config.durability.restore_replication_state_on_startup) return std::nullopt; + return {config.durability.storage_directory}; +} + Storage::Storage(Config config, StorageMode storage_mode) - : name_id_mapper_(std::invoke([config, storage_mode]() -> std::unique_ptr { + : repl_state_(ReplicationStateHelper(config)), + name_id_mapper_(std::invoke([config, storage_mode]() -> std::unique_ptr { if (storage_mode == StorageMode::ON_DISK_TRANSACTIONAL) { return std::make_unique(config.disk.name_id_mapper_directory, config.disk.id_name_mapper_directory); @@ -52,16 +59,28 @@ Storage::Storage(Config config, StorageMode storage_mode) storage_mode_(storage_mode), indices_(config, storage_mode), constraints_(config, storage_mode), - id_(config.name), - replication_state_(config_.durability.restore_replication_state_on_startup, - config_.durability.storage_directory) {} + id_(config.name) {} -Storage::Accessor::Accessor(Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode) +Storage::Accessor::Accessor(SharedAccess /* tag */, Storage *storage, IsolationLevel isolation_level, + StorageMode storage_mode) : storage_(storage), // The lock must be acquired before creating the transaction object to // prevent freshly created transactions from dangling in an active state // during exclusive operations. storage_guard_(storage_->main_lock_), + unique_guard_(storage_->main_lock_, std::defer_lock), + transaction_(storage->CreateTransaction(isolation_level, storage_mode)), + is_transaction_active_(true), + creation_storage_mode_(storage_mode) {} + +Storage::Accessor::Accessor(UniqueAccess /* tag */, Storage *storage, IsolationLevel isolation_level, + StorageMode storage_mode) + : storage_(storage), + // The lock must be acquired before creating the transaction object to + // prevent freshly created transactions from dangling in an active state + // during exclusive operations. + storage_guard_(storage_->main_lock_, std::defer_lock), + unique_guard_(storage_->main_lock_), transaction_(storage->CreateTransaction(isolation_level, storage_mode)), is_transaction_active_(true), creation_storage_mode_(storage_mode) {} @@ -69,6 +88,7 @@ Storage::Accessor::Accessor(Storage *storage, IsolationLevel isolation_level, St Storage::Accessor::Accessor(Accessor &&other) noexcept : storage_(other.storage_), storage_guard_(std::move(other.storage_guard_)), + unique_guard_(std::move(other.unique_guard_)), transaction_(std::move(other.transaction_)), commit_timestamp_(other.commit_timestamp_), is_transaction_active_(other.is_transaction_active_), @@ -78,16 +98,6 @@ Storage::Accessor::Accessor(Accessor &&other) noexcept other.commit_timestamp_.reset(); } -IndicesInfo Storage::ListAllIndices() const { - std::shared_lock storage_guard_(main_lock_); - return {indices_.label_index_->ListIndices(), indices_.label_property_index_->ListIndices()}; -} - -ConstraintsInfo Storage::ListAllConstraints() const { - std::shared_lock storage_guard_(main_lock_); - return {constraints_.existence_constraints_->ListConstraints(), constraints_.unique_constraints_->ListConstraints()}; -} - /// Main lock is taken by the caller. void Storage::SetStorageMode(StorageMode storage_mode) { std::unique_lock main_guard{main_lock_}; diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 608421228..0aeb206e7 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -11,11 +11,15 @@ #pragma once +#include +#include #include +#include #include "io/network/endpoint.hpp" #include "kvstore/kvstore.hpp" #include "query/exceptions.hpp" +#include "replication/config.hpp" #include "storage/v2/all_vertices_iterable.hpp" #include "storage/v2/commit_log.hpp" #include "storage/v2/config.hpp" @@ -24,17 +28,17 @@ #include "storage/v2/edge_accessor.hpp" #include "storage/v2/indices/indices.hpp" #include "storage/v2/mvcc.hpp" -#include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" -#include "storage/v2/replication/replication.hpp" #include "storage/v2/replication/replication_client.hpp" #include "storage/v2/replication/replication_server.hpp" +#include "storage/v2/replication/replication_storage_state.hpp" #include "storage/v2/storage_error.hpp" #include "storage/v2/storage_mode.hpp" #include "storage/v2/transaction.hpp" #include "storage/v2/vertices_iterable.hpp" #include "utils/event_counter.hpp" #include "utils/event_histogram.hpp" +#include "utils/resource_lock.hpp" #include "utils/scheduler.hpp" #include "utils/timer.hpp" #include "utils/uuid.hpp" @@ -94,7 +98,13 @@ class Storage { class Accessor { public: - Accessor(Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode); + static constexpr struct SharedAccess { + } shared_access; + static constexpr struct UniqueAccess { + } unique_access; + + Accessor(SharedAccess /* tag */, Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode); + Accessor(UniqueAccess /* tag */, Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode); Accessor(const Accessor &) = delete; Accessor &operator=(const Accessor &) = delete; Accessor &operator=(Accessor &&other) = delete; @@ -149,14 +159,10 @@ class Storage { virtual void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property, const LabelPropertyIndexStats &stats) = 0; - virtual std::vector> ClearLabelPropertyIndexStats() = 0; - - virtual std::vector ClearLabelIndexStats() = 0; - virtual std::vector> DeleteLabelPropertyIndexStats( - std::span labels) = 0; + const storage::LabelId &label) = 0; - virtual std::vector DeleteLabelIndexStats(std::span labels) = 0; + virtual bool DeleteLabelIndexStats(const storage::LabelId &label) = 0; virtual void PrefetchInEdges(const VertexAccessor &vertex_acc) = 0; @@ -179,7 +185,7 @@ class Storage { virtual ConstraintsInfo ListAllConstraints() const = 0; // NOLINTNEXTLINE(google-default-arguments) - virtual utils::BasicResult Commit( + virtual utils::BasicResult Commit( std::optional desired_commit_timestamp = {}) = 0; virtual void Abort() = 0; @@ -206,9 +212,30 @@ class Storage { const std::string &id() const { return storage_->id(); } + virtual utils::BasicResult CreateIndex(LabelId label) = 0; + + virtual utils::BasicResult CreateIndex(LabelId label, PropertyId property) = 0; + + virtual utils::BasicResult DropIndex(LabelId label) = 0; + + virtual utils::BasicResult DropIndex(LabelId label, PropertyId property) = 0; + + virtual utils::BasicResult CreateExistenceConstraint( + LabelId label, PropertyId property) = 0; + + virtual utils::BasicResult DropExistenceConstraint( + LabelId label, PropertyId property) = 0; + + virtual utils::BasicResult + CreateUniqueConstraint(LabelId label, const std::set &properties) = 0; + + virtual UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, + const std::set &properties) = 0; + protected: Storage *storage_; - std::shared_lock storage_guard_; + std::shared_lock storage_guard_; + std::unique_lock unique_guard_; // TODO: Split the accessor into Shared/Unique Transaction transaction_; std::optional commit_timestamp_; bool is_transaction_active_; @@ -251,58 +278,15 @@ class Storage { StorageMode GetStorageMode() const; - virtual void FreeMemory(std::unique_lock main_guard) = 0; + virtual void FreeMemory(std::unique_lock main_guard) = 0; void FreeMemory() { FreeMemory({}); } virtual std::unique_ptr Access(std::optional override_isolation_level) = 0; std::unique_ptr Access() { return Access(std::optional{}); } - virtual utils::BasicResult CreateIndex( - LabelId label, std::optional desired_commit_timestamp) = 0; - - utils::BasicResult CreateIndex(LabelId label) { - return CreateIndex(label, std::optional{}); - } - - virtual utils::BasicResult CreateIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) = 0; - - utils::BasicResult CreateIndex(LabelId label, PropertyId property) { - return CreateIndex(label, property, std::optional{}); - } - - virtual utils::BasicResult DropIndex( - LabelId label, std::optional desired_commit_timestamp) = 0; - - utils::BasicResult DropIndex(LabelId label) { - return DropIndex(label, std::optional{}); - } - - virtual utils::BasicResult DropIndex( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) = 0; - - utils::BasicResult DropIndex(LabelId label, PropertyId property) { - return DropIndex(label, property, std::optional{}); - } - - IndicesInfo ListAllIndices() const; - - virtual utils::BasicResult CreateExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) = 0; - - virtual utils::BasicResult DropExistenceConstraint( - LabelId label, PropertyId property, std::optional desired_commit_timestamp) = 0; - - virtual utils::BasicResult - CreateUniqueConstraint(LabelId label, const std::set &properties, - std::optional desired_commit_timestamp) = 0; - - virtual utils::BasicResult - DropUniqueConstraint(LabelId label, const std::set &properties, - std::optional desired_commit_timestamp) = 0; - - ConstraintsInfo ListAllConstraints() const; + virtual std::unique_ptr UniqueAccess(std::optional override_isolation_level) = 0; + std::unique_ptr UniqueAccess() { return UniqueAccess(std::optional{}); } enum class SetIsolationLevelError : uint8_t { DisabledForAnalyticalMode }; @@ -313,37 +297,22 @@ class Storage { virtual Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode) = 0; - virtual void EstablishNewEpoch() = 0; + virtual void PrepareForNewEpoch(std::string prev_epoch) = 0; - virtual auto CreateReplicationClient(replication::ReplicationClientConfig const &config) + virtual auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config) -> std::unique_ptr = 0; - virtual auto CreateReplicationServer(const replication::ReplicationServerConfig &config) + virtual auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig &config) -> std::unique_ptr = 0; - /// REPLICATION - bool SetReplicaRole(const replication::ReplicationServerConfig &config) { - return replication_state_.SetReplicaRole(config, this); - } - bool SetMainReplicationRole() { return replication_state_.SetMainReplicationRole(this); } - - /// @pre The instance should have a MAIN role - /// @pre Timeout can only be set for SYNC replication - auto RegisterReplica(const replication::RegistrationMode registration_mode, - const replication::ReplicationClientConfig &config) { - return replication_state_.RegisterReplica(registration_mode, config, this); - } - /// @pre The instance should have a MAIN role - bool UnregisterReplica(const std::string &name) { return replication_state_.UnregisterReplica(name); } - replication::ReplicationRole GetReplicationRole() const { return replication_state_.GetRole(); } - auto ReplicasInfo() { return replication_state_.ReplicasInfo(); } - std::optional GetReplicaState(std::string_view name) { - return replication_state_.GetReplicaState(name); + auto ReplicasInfo() { return repl_storage_state_.ReplicasInfo(); } + auto GetReplicaState(std::string_view name) -> std::optional { + return repl_storage_state_.GetReplicaState(name); } - protected: - void RestoreReplicas() { return replication_state_.RestoreReplicas(this); } - void RestoreReplicationRole() { return replication_state_.RestoreReplicationRole(this); } + // TODO: make non-public + memgraph::replication::ReplicationState repl_state_; + ReplicationStorageState repl_storage_state_; public: // Main storage lock. @@ -351,7 +320,7 @@ class Storage { // creation of new accessors by taking a unique lock. This is used when doing // operations on storage that affect the global state, for example index // creation. - mutable utils::RWLock main_lock_{utils::RWLock::Priority::WRITE}; + mutable utils::ResourceLock main_lock_; // Even though the edge count is already kept in the `edges_` SkipList, the // list is used only when properties are enabled for edges. Because of that we @@ -376,9 +345,6 @@ class Storage { std::atomic vertex_id_{0}; std::atomic edge_id_{0}; const std::string id_; //!< High-level assigned ID - - protected: - ReplicationState replication_state_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/storage_error.hpp b/src/storage/v2/storage_error.hpp index b9ce0fb71..0c17d886b 100644 --- a/src/storage/v2/storage_error.hpp +++ b/src/storage/v2/storage_error.hpp @@ -19,30 +19,27 @@ namespace memgraph::storage { struct ReplicationError {}; +struct PersistenceError {}; // TODO: Generalize and add to InMemory durability as well (currently durability just + // asserts and terminated if failed) -struct IndexPersistenceError {}; +struct IndexDefinitionError {}; struct ConstraintsPersistenceError {}; struct SerializationError {}; inline bool operator==(const SerializationError & /*err1*/, const SerializationError & /*err2*/) { return true; } -using StorageDataManipulationError = std::variant; +using StorageManipulationError = + std::variant; -struct IndexDefinitionError {}; -using StorageIndexDefinitionError = std::variant; +using StorageIndexDefinitionError = IndexDefinitionError; struct ConstraintDefinitionError {}; -using StorageExistenceConstraintDefinitionError = - std::variant; +using StorageExistenceConstraintDefinitionError = std::variant; -using StorageExistenceConstraintDroppingError = - std::variant; +using StorageExistenceConstraintDroppingError = ConstraintDefinitionError; -using StorageUniqueConstraintDefinitionError = - std::variant; - -using StorageUniqueConstraintDroppingError = std::variant; +using StorageUniqueConstraintDefinitionError = std::variant; } // namespace memgraph::storage diff --git a/src/storage/v2/transaction.hpp b/src/storage/v2/transaction.hpp index 8f0132062..f5fdcedb2 100644 --- a/src/storage/v2/transaction.hpp +++ b/src/storage/v2/transaction.hpp @@ -22,6 +22,7 @@ #include "storage/v2/delta.hpp" #include "storage/v2/edge.hpp" #include "storage/v2/isolation_level.hpp" +#include "storage/v2/metadata_delta.hpp" #include "storage/v2/modified_edge.hpp" #include "storage/v2/property_value.hpp" #include "storage/v2/storage_mode.hpp" @@ -44,6 +45,7 @@ struct Transaction { start_timestamp(start_timestamp), command_id(0), deltas(1024UL), + md_deltas(utils::NewDeleteResource()), must_abort(false), isolation_level(isolation_level), storage_mode(storage_mode), @@ -55,6 +57,7 @@ struct Transaction { commit_timestamp(std::move(other.commit_timestamp)), command_id(other.command_id), deltas(std::move(other.deltas)), + md_deltas(std::move(other.md_deltas)), must_abort(other.must_abort), isolation_level(other.isolation_level), storage_mode(other.storage_mode), @@ -93,6 +96,7 @@ struct Transaction { uint64_t command_id; Bond deltas; + utils::pmr::list md_deltas; bool must_abort; IsolationLevel isolation_level; StorageMode storage_mode; diff --git a/src/storage/v2/vertex.hpp b/src/storage/v2/vertex.hpp index d051b95fd..41021b436 100644 --- a/src/storage/v2/vertex.hpp +++ b/src/storage/v2/vertex.hpp @@ -30,7 +30,7 @@ struct Vertex { "Vertex must be created with an initial DELETE_OBJECT delta!"); } - Gid gid; + const Gid gid; std::vector labels; PropertyStore properties; diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index d5144daf6..9b22389b1 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -25,6 +25,7 @@ find_package(gflags REQUIRED) find_package(Threads REQUIRED) add_library(mg-utils STATIC ${utils_src_files}) +add_library(mg::utils ALIAS mg-utils) target_link_libraries(mg-utils PUBLIC Boost::headers fmt::fmt spdlog::spdlog) target_link_libraries(mg-utils PRIVATE librdtsc stdc++fs Threads::Threads gflags json uuid rt) diff --git a/src/utils/resource_lock.hpp b/src/utils/resource_lock.hpp new file mode 100644 index 000000000..7d6be2685 --- /dev/null +++ b/src/utils/resource_lock.hpp @@ -0,0 +1,81 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include +#include +#include + +namespace memgraph::utils { + +/* A type that complies to the + * C++ named requirements `SharedLockable` + * + * Unlike `std::shared_mutex` it can be locked in one thread + * and unlocked in another. + */ +struct ResourceLock { + private: + enum states { UNLOCKED, UNIQUE, SHARED }; + + public: + void lock() { + auto lock = std::unique_lock{mtx}; + // block until available + cv.wait(lock, [this] { return state == UNLOCKED; }); + state = UNIQUE; + } + void lock_shared() { + auto lock = std::unique_lock{mtx}; + // block until available + cv.wait(lock, [this] { return state != UNIQUE; }); + state = SHARED; + ++count; + } + bool try_lock() { + auto lock = std::unique_lock{mtx}; + if (state == UNLOCKED) { + state = UNIQUE; + return true; + } + return false; + } + bool try_lock_shared() { + auto lock = std::unique_lock{mtx}; + if (state != UNIQUE) { + state = SHARED; + ++count; + return true; + } + return false; + } + void unlock() { + auto lock = std::unique_lock{mtx}; + state = UNLOCKED; + cv.notify_all(); // multiple lock_shared maybe waiting + } + void unlock_shared() { + auto lock = std::unique_lock{mtx}; + --count; + if (count == 0) { + state = UNLOCKED; + cv.notify_one(); // should be 0 waiting in lock_shared, only 1 wait in lock can progress + } + } + + private: + std::mutex mtx; + std::condition_variable cv; + states state = UNLOCKED; + uint64_t count = 0; +}; + +} // namespace memgraph::utils diff --git a/src/utils/rw_lock.hpp b/src/utils/rw_lock.hpp index beae536bf..11b222bc4 100644 --- a/src/utils/rw_lock.hpp +++ b/src/utils/rw_lock.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -126,4 +126,9 @@ class WritePrioritizedRWLock final : public RWLock { WritePrioritizedRWLock() : RWLock{Priority::WRITE} {}; }; +class ReadPrioritizedRWLock final : public RWLock { + public: + ReadPrioritizedRWLock() : RWLock{Priority::READ} {}; +}; + } // namespace memgraph::utils diff --git a/src/utils/simple_json.hpp b/src/utils/simple_json.hpp new file mode 100644 index 000000000..58573efaa --- /dev/null +++ b/src/utils/simple_json.hpp @@ -0,0 +1,56 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include + +namespace memgraph::utils { + +static inline std::regex GetRegexKey(const std::string &key) { + return std::regex("\"" + key + "\"\\s*:\\s*\"([^\"]+)\"|\"" + key + "\"\\s*:\\s*([^,\\}]+)"); +} + +// Function to parse JSON string and return a value by key as a string. +static inline bool GetJsonValue(const std::string &jsonString, const std::string &key, std::string &out) { + const auto regex = GetRegexKey(key); + std::smatch match; + + if (std::regex_search(jsonString, match, regex)) { + out = match[1].str(); + return true; + } + return false; +} + +// Overloaded GetJsonValue function for different types. +template +static inline bool GetJsonValue(const std::string &jsonString, const std::string &key, T &out) { + const auto regex = GetRegexKey(key); + std::smatch match; + + if (std::regex_search(jsonString, match, regex)) { + for (size_t i = 1; i < match.size(); ++i) { + const auto &m = match[i].str(); + if (m.empty()) continue; + std::istringstream ss(m); + ss >> out; + return true; + } + } + return false; +} + +}; // namespace memgraph::utils diff --git a/src/utils/typeinfo.hpp b/src/utils/typeinfo.hpp index bad2c3f8e..679c75ff3 100644 --- a/src/utils/typeinfo.hpp +++ b/src/utils/typeinfo.hpp @@ -163,7 +163,8 @@ enum class TypeId : uint64_t { AST_MERGE, AST_UNWIND, AST_AUTH_QUERY, - AST_INFO_QUERY, + AST_DATABASE_INFO_QUERY, + AST_SYSTEM_INFO_QUERY, AST_CONSTRAINT, AST_CONSTRAINT_QUERY, AST_DUMP_QUERY, diff --git a/tests/benchmark/expansion.cpp b/tests/benchmark/expansion.cpp index 9bbb02579..93e1a83c6 100644 --- a/tests/benchmark/expansion.cpp +++ b/tests/benchmark/expansion.cpp @@ -52,7 +52,10 @@ class ExpansionBenchFixture : public benchmark::Fixture { MG_ASSERT(!dba->Commit().HasError()); } - MG_ASSERT(!db_acc->storage()->CreateIndex(label).HasError()); + { + auto unique_acc = db_acc->UniqueAccess(); + MG_ASSERT(!unique_acc->CreateIndex(label).HasError()); + } interpreter.emplace(&*interpreter_context, std::move(db_acc)); } @@ -69,8 +72,8 @@ BENCHMARK_DEFINE_F(ExpansionBenchFixture, Match)(benchmark::State &state) { auto query = "MATCH (s:Starting) return s"; while (state.KeepRunning()) { - ResultStreamFaker results(interpreter->db_acc_->get()->storage()); - interpreter->Prepare(query, {}, nullptr); + ResultStreamFaker results(interpreter->current_db_.db_acc_->get()->storage()); + interpreter->Prepare(query, {}, {}); interpreter->PullAll(&results); } } @@ -84,8 +87,8 @@ BENCHMARK_DEFINE_F(ExpansionBenchFixture, Expand)(benchmark::State &state) { auto query = "MATCH (s:Starting) WITH s MATCH (s)--(d) RETURN count(d)"; while (state.KeepRunning()) { - ResultStreamFaker results(interpreter->db_acc_->get()->storage()); - interpreter->Prepare(query, {}, nullptr); + ResultStreamFaker results(interpreter->current_db_.db_acc_->get()->storage()); + interpreter->Prepare(query, {}, {}); interpreter->PullAll(&results); } } diff --git a/tests/benchmark/query/execution.cpp b/tests/benchmark/query/execution.cpp index 0aa06e637..771f0ac1f 100644 --- a/tests/benchmark/query/execution.cpp +++ b/tests/benchmark/query/execution.cpp @@ -83,7 +83,10 @@ static void AddStarGraph(memgraph::storage::Storage *db, int spoke_count, int de } MG_ASSERT(!dba->Commit().HasError()); } - MG_ASSERT(!db->CreateIndex(db->NameToLabel(kStartLabel)).HasError()); + { + auto unique_acc = db->UniqueAccess(); + MG_ASSERT(!unique_acc->CreateIndex(db->NameToLabel(kStartLabel)).HasError()); + } } static void AddTree(memgraph::storage::Storage *db, int vertex_count) { @@ -105,7 +108,10 @@ static void AddTree(memgraph::storage::Storage *db, int vertex_count) { } MG_ASSERT(!dba->Commit().HasError()); } - MG_ASSERT(!db->CreateIndex(db->NameToLabel(kStartLabel)).HasError()); + { + auto unique_acc = db->UniqueAccess(); + MG_ASSERT(!unique_acc->CreateIndex(db->NameToLabel(kStartLabel)).HasError()); + } } static memgraph::query::CypherQuery *ParseCypherQuery(const std::string &query_string, diff --git a/tests/benchmark/query/planner.cpp b/tests/benchmark/query/planner.cpp index 2beb4fc41..8cc90fe05 100644 --- a/tests/benchmark/query/planner.cpp +++ b/tests/benchmark/query/planner.cpp @@ -94,7 +94,10 @@ static memgraph::query::CypherQuery *AddIndexedMatches(int num_matches, const st static auto CreateIndexedVertices(int index_count, int vertex_count, memgraph::storage::Storage *db) { auto label = db->NameToLabel("label"); auto prop = db->NameToProperty("prop"); - [[maybe_unused]] auto _ = db->CreateIndex(label, prop); + { + auto unique_acc = db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + } auto dba = db->Access(); for (int vi = 0; vi < vertex_count; ++vi) { for (int index = 0; index < index_count; ++index) { diff --git a/tests/concurrent/storage_indices.cpp b/tests/concurrent/storage_indices.cpp index 4757be189..a3c09d2e4 100644 --- a/tests/concurrent/storage_indices.cpp +++ b/tests/concurrent/storage_indices.cpp @@ -30,7 +30,10 @@ TEST(Storage, LabelIndex) { std::unique_ptr store{new memgraph::storage::InMemoryStorage()}; auto label = store->NameToLabel("label"); - ASSERT_FALSE(store->CreateIndex(label).HasError()); + { + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label).HasError()); + } std::vector verifiers; verifiers.reserve(kNumVerifiers); @@ -112,7 +115,10 @@ TEST(Storage, LabelPropertyIndex) { auto label = store->NameToLabel("label"); auto prop = store->NameToProperty("prop"); - ASSERT_FALSE(store->CreateIndex(label, prop).HasError()); + { + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label, prop).HasError()); + } std::vector verifiers; verifiers.reserve(kNumVerifiers); diff --git a/tests/concurrent/storage_unique_constraints.cpp b/tests/concurrent/storage_unique_constraints.cpp index f2f73930e..694249779 100644 --- a/tests/concurrent/storage_unique_constraints.cpp +++ b/tests/concurrent/storage_unique_constraints.cpp @@ -84,9 +84,11 @@ void AddLabel(memgraph::storage::Storage *storage, memgraph::storage::Gid gid, L TEST_F(StorageUniqueConstraints, ChangeProperties) { { - auto res = storage->CreateUniqueConstraint(label, {prop1, prop2, prop3}, {}); + auto unique_acc = storage->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(label, {prop1, prop2, prop3}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { @@ -166,9 +168,11 @@ TEST_F(StorageUniqueConstraints, ChangeProperties) { TEST_F(StorageUniqueConstraints, ChangeLabels) { { - auto res = storage->CreateUniqueConstraint(label, {prop1, prop2, prop3}, {}); + auto unique_acc = storage->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(label, {prop1, prop2, prop3}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } // In the first part of the test, each transaction tries to add the same label diff --git a/tests/e2e/replication/CMakeLists.txt b/tests/e2e/replication/CMakeLists.txt index ad002402c..b054981c8 100644 --- a/tests/e2e/replication/CMakeLists.txt +++ b/tests/e2e/replication/CMakeLists.txt @@ -3,6 +3,9 @@ find_package(gflags REQUIRED) add_executable(memgraph__e2e__replication__constraints constraints.cpp) target_link_libraries(memgraph__e2e__replication__constraints gflags mgclient mg-utils mg-io Threads::Threads) +add_executable(memgraph__e2e__replication__indices indices.cpp) +target_link_libraries(memgraph__e2e__replication__indices gflags mgclient mg-utils mg-io Threads::Threads) + add_executable(memgraph__e2e__replication__read_write_benchmark read_write_benchmark.cpp) target_link_libraries(memgraph__e2e__replication__read_write_benchmark gflags json mgclient mg-utils mg-io Threads::Threads) diff --git a/tests/e2e/replication/indices.cpp b/tests/e2e/replication/indices.cpp new file mode 100644 index 000000000..c0eee23a7 --- /dev/null +++ b/tests/e2e/replication/indices.cpp @@ -0,0 +1,174 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include + +#include +#include + +#include "common.hpp" +#include "utils/logging.hpp" +#include "utils/thread.hpp" +#include "utils/timer.hpp" + +int main(int argc, char **argv) { + google::SetUsageMessage("Memgraph E2E Replication Indices Test"); + gflags::ParseCommandLineFlags(&argc, &argv, true); + memgraph::logging::RedirectToStderr(); + + auto database_endpoints = mg::e2e::replication::ParseDatabaseEndpoints(FLAGS_database_endpoints); + + mg::Client::Init(); + + auto check_index = [](const auto &data, size_t i, const auto &label, const std::string &property = "") { + const auto label_name = (*data)[i][1].ValueString(); + const auto p_type = (*data)[i][2].type(); + const auto property_name = p_type == mg::Value::Type::String ? (*data)[i][2].ValueString() : ""; + if (label_name != label || property_name != property) { + LOG_FATAL("Invalid index."); + } + }; + + auto check_delete_stats = [](const auto &data, size_t i, const auto &label, const std::string &property = "") { + const auto label_name = (*data)[i][0].ValueString(); + const auto p_type = (*data)[i][1].type(); + const auto property_name = p_type == mg::Value::Type::String ? (*data)[i][1].ValueString() : ""; + if (label_name != label || property_name != property) { + LOG_FATAL("Invalid stats."); + } + }; + + { + auto client = mg::e2e::replication::Connect(database_endpoints[0]); + client->Execute("MATCH (n) DETACH DELETE n;"); + client->DiscardAll(); + client->Execute("CREATE INDEX ON :Node;"); + client->DiscardAll(); + client->Execute("CREATE INDEX ON :Node(id);"); + client->DiscardAll(); + client->Execute("CREATE INDEX ON :Node(id2);"); + client->DiscardAll(); + client->Execute("CREATE INDEX ON :Node2;"); + client->DiscardAll(); + + // Sleep a bit so the constraints get replicated. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + for (const auto &database_endpoint : database_endpoints) { + auto client = mg::e2e::replication::Connect(database_endpoint); + client->Execute("SHOW INDEX INFO;"); + if (const auto data = client->FetchAll()) { + if (data->size() != 4) { + LOG_FATAL("Missing indices!"); + } + check_index(data, 0, "Node"); + check_index(data, 1, "Node2"); + check_index(data, 2, "Node", "id"); + check_index(data, 3, "Node", "id2"); + } else { + LOG_FATAL("Unable to get INDEX INFO from {}", database_endpoint); + } + } + spdlog::info("All indices are in-place."); + + for (int i = 0; i < FLAGS_nodes; ++i) { + client->Execute("CREATE (:Node {id:" + std::to_string(i) + "});"); + client->DiscardAll(); + } + mg::e2e::replication::IntGenerator edge_generator("EdgeCreateGenerator", 0, FLAGS_nodes - 1); + for (int i = 0; i < FLAGS_edges; ++i) { + client->Execute("MATCH (n {id:" + std::to_string(edge_generator.Next()) + + "}), (m {id:" + std::to_string(edge_generator.Next()) + "}) CREATE (n)-[:Edge]->(m);"); + client->DiscardAll(); + } + + client->Execute("ANALYZE GRAPH;"); + client->DiscardAll(); + } + + { + for (int i = 1; i < database_endpoints.size(); ++i) { + auto &database_endpoint = database_endpoints[i]; + auto client = mg::e2e::replication::Connect(database_endpoint); + try { + // Hacky way to determine if the statistics were replicated + client->Execute("ANALYZE GRAPH DELETE STATISTICS;"); + const auto data = client->FetchAll(); + if (data->size() != 4) { + LOG_FATAL("Not all statistics were replicated! Failed endpoint {}:{}", database_endpoint.address, + database_endpoint.port); + } + check_delete_stats(data, 0, "Node"); + check_delete_stats(data, 1, "Node2"); + check_delete_stats(data, 2, "Node", "id"); + check_delete_stats(data, 3, "Node", "id2"); + } catch (const std::exception &e) { + LOG_FATAL(e.what()); + break; + } + } + } + + { + auto client = mg::e2e::replication::Connect(database_endpoints[0]); + client->Execute("ANALYZE GRAPH;"); // Resend stats + client->DiscardAll(); + client->Execute("ANALYZE GRAPH DELETE STATISTICS;"); + client->DiscardAll(); + // Sleep a bit so the drop constraints get replicated. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + for (int i = 1; i < database_endpoints.size(); ++i) { + auto &database_endpoint = database_endpoints[i]; + auto client = mg::e2e::replication::Connect(database_endpoints[i]); + client->Execute("ANALYZE GRAPH DELETE STATISTICS;"); + if (const auto data = client->FetchAll()) { + // Hacky way to determine if the statistics were replicated + if (data->size() != 0) { + LOG_FATAL("Not all statistics were replicated! Failed endpoint {}:{}", database_endpoint.address, + database_endpoint.port); + } + } else { + LOG_FATAL("Unable to delete statistics from {}", database_endpoints[i]); + } + } + } + + { + auto client = mg::e2e::replication::Connect(database_endpoints[0]); + client->Execute("DROP INDEX ON :Node;"); + client->DiscardAll(); + client->Execute("DROP INDEX ON :Node(id);"); + client->DiscardAll(); + client->Execute("DROP INDEX ON :Node(id2);"); + client->DiscardAll(); + client->Execute("DROP INDEX ON :Node2;"); + client->DiscardAll(); + + // Sleep a bit so the constraints get replicated. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + for (const auto &database_endpoint : database_endpoints) { + auto client = mg::e2e::replication::Connect(database_endpoint); + client->Execute("SHOW INDEX INFO;"); + if (const auto data = client->FetchAll()) { + if (!data->empty()) { + LOG_FATAL("Undeleted indices!"); + } + } else { + LOG_FATAL("Unable to get INDEX INFO from {}", database_endpoint); + } + } + spdlog::info("All indices have been deleted."); + } + + return 0; +} diff --git a/tests/e2e/replication/show_while_creating_invalid_state.py b/tests/e2e/replication/show_while_creating_invalid_state.py index 96088375d..d85f7e25e 100644 --- a/tests/e2e/replication/show_while_creating_invalid_state.py +++ b/tests/e2e/replication/show_while_creating_invalid_state.py @@ -974,7 +974,7 @@ def test_attempt_to_create_indexes_on_main_when_async_replica_is_down(): def test_attempt_to_create_indexes_on_main_when_sync_replica_is_down(): - # Goal of this test is to check that main cannot create new indexes/constraints if a sync replica is down. + # Goal of this test is to check creation of new indexes/constraints when a sync replica is down. # 0/ Start main and sync replicas. # 1/ Check status of replicas. # 2/ Add some indexes to main and check it is propagated to the sync_replicas. @@ -1052,9 +1052,10 @@ def test_attempt_to_create_indexes_on_main_when_sync_replica_is_down(): # 5/ expected_data = { ("sync_replica1", "127.0.0.1:10001", "sync", 0, 0, "invalid"), - ("sync_replica2", "127.0.0.1:10002", "sync", 2, 0, "ready"), + ("sync_replica2", "127.0.0.1:10002", "sync", 5, 0, "ready"), } res_from_main = interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query(QUERY_TO_CHECK) + assert res_from_main == interactive_mg_runner.MEMGRAPH_INSTANCES["sync_replica2"].query(QUERY_TO_CHECK) actual_data = set(interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query("SHOW REPLICAS;")) assert actual_data == expected_data diff --git a/tests/e2e/replication/workloads.yaml b/tests/e2e/replication/workloads.yaml index d9830ae04..72269d652 100644 --- a/tests/e2e/replication/workloads.yaml +++ b/tests/e2e/replication/workloads.yaml @@ -41,6 +41,11 @@ workloads: args: [] <<: *template_cluster + - name: "Index replication" + binary: "tests/e2e/replication/memgraph__e2e__replication__indices" + args: [] + <<: *template_cluster + - name: "Read-write benchmark" binary: "tests/e2e/replication/memgraph__e2e__replication__read_write_benchmark" args: [] diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index e6a2ec1dc..1aba6a517 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -1,8 +1,6 @@ #!/bin/bash SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$SCRIPT_DIR" -# TODO(gitbuda): Setup mgclient and pymgclient properly. -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib print_help() { echo -e "$0 ["workload name string"]" diff --git a/tests/gql_behave/README.md b/tests/gql_behave/README.md index 4f3a29880..31599881c 100644 --- a/tests/gql_behave/README.md +++ b/tests/gql_behave/README.md @@ -4,7 +4,9 @@ Python script used to run graph query language behavior tests against Memgraph. To run the script please execute: ``` +cd memgraph/tests source ve3/bin/activate +cd gql_behave ./run.py --help ./run.py memgraph_V1 ``` diff --git a/tests/gql_behave/continuous_integration b/tests/gql_behave/continuous_integration index 35efd90a5..655b45bb3 100755 --- a/tests/gql_behave/continuous_integration +++ b/tests/gql_behave/continuous_integration @@ -15,14 +15,14 @@ List of responsibilities: import argparse import atexit import copy -import os -import sys import json +import os import subprocess +import sys import tempfile import time -import yaml +import yaml SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TESTS_DIR = os.path.join(SCRIPT_DIR, "tests") @@ -66,7 +66,7 @@ def generate_status(suite, result_path, required): if passed == total: msg += " 👍" # +1 emoji else: - msg += " ⛔" # no entry emoji + msg += " ⛔" # no entry emoji return (msg, passed, total) @@ -87,7 +87,7 @@ def generate_result_html(data): return ret -class MemgraphRunner(): +class MemgraphRunner: def __init__(self, build_directory): self.build_directory = build_directory self.proc_mg = None @@ -102,8 +102,7 @@ class MemgraphRunner(): self.data_directory = tempfile.TemporaryDirectory() memgraph_binary = os.path.join(self.build_directory, "memgraph") - args_mg = [memgraph_binary, "--storage-properties-on-edges", - "--data-directory", self.data_directory.name] + args_mg = [memgraph_binary, "--storage-properties-on-edges", "--data-directory", self.data_directory.name] self.proc_mg = subprocess.Popen(args_mg + self.args) wait_for_server(7687, 1) assert self.is_running(), "The Memgraph process died!" @@ -135,7 +134,7 @@ def main(): suites = yaml.safe_load(f) # venv used to run the qa engine - venv_python = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3") + venv_python = os.path.join(BASE_DIR, "tests", "ve3", "bin", "python3") # Temporary directory for suite results output_dir = tempfile.TemporaryDirectory() @@ -153,29 +152,36 @@ def main(): mandatory_fails = [] # Run suites + old_storage_mode = "" for suite in suites: print("Starting suite '{}' scenarios.".format(suite["name"])) - memgraph.start() + suite_storage_mode = suite["storage_mode"] + if old_storage_mode != suite_storage_mode: + memgraph.stop() + memgraph.start(["--storage-mode", suite_storage_mode]) + old_storage_mode = suite_storage_mode - suite["stats_file"] = os.path.join(output_dir.name, - suite["name"] + ".json") - cmd = [venv_python, "-u", - os.path.join(SCRIPT_DIR, "run.py"), - "--stats-file", suite["stats_file"], - suite["test_suite"]] + suite["stats_file"] = os.path.join(output_dir.name, suite["name"] + ".json") + cmd = [ + venv_python, + "-u", + os.path.join(SCRIPT_DIR, "run.py"), + "--stats-file", + suite["stats_file"], + suite["test_suite"], + ] # The exit code isn't checked here because the `behave` framework # returns a non-zero exit code when some tests fail. subprocess.run(cmd) - suite_status, suite_passed, suite_total = \ - generate_status(suite["name"], suite["stats_file"], - suite["must_pass"]) + suite_status, suite_passed, suite_total = generate_status( + suite["name"], suite["stats_file"], suite["must_pass"] + ) status_data.append([suite["name"], suite_status]) - result_csv += generate_result_csv(suite["name"], - suite["stats_file"]) + result_csv += generate_result_csv(suite["name"], suite["stats_file"]) if suite["must_pass"] and suite_passed != suite_total: mandatory_fails.append(suite["name"]) @@ -194,12 +200,12 @@ def main(): with open(result_csv_path, "w") as f: f.write(result_csv) - print(f'CSV status is generated in {result_csv_path}') - print(f'HTML status is generated in {result_html_path}') + print(f"CSV status is generated in {result_csv_path}") + print(f"HTML status is generated in {result_html_path}") # Check if tests failed if mandatory_fails != []: - sys.exit(f'Some tests that must pass have failed: {mandatory_fails}') + sys.exit(f"Some tests that must pass have failed: {mandatory_fails}") if __name__ == "__main__": diff --git a/tests/gql_behave/run.py b/tests/gql_behave/run.py index 50035384a..c2112681a 100755 --- a/tests/gql_behave/run.py +++ b/tests/gql_behave/run.py @@ -15,9 +15,10 @@ import argparse import os import sys -from behave.__main__ import main as behave_main -from behave import configuration +import mgclient +from behave import configuration +from behave.__main__ import main as behave_main SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -55,23 +56,16 @@ def main(): add_config("--test-directory") # Arguments that should be passed on to Behave - add_argument("--db-host", default="127.0.0.1", - help="server host (default is 127.0.0.1)") - add_argument("--db-port", default="7687", - help="server port (default is 7687)") - add_argument("--db-user", default="memgraph", - help="server user (default is memgraph)") - add_argument("--db-pass", default="memgraph", - help="server pass (default is memgraph)") - add_argument("--stop", action="store_true", - help="stop testing after first fail") - add_argument("--single-fail", action="store_true", - help="pause after failed scenario") - add_argument("--single-scenario", action="store_true", - help="pause after every scenario") - add_argument("--single-feature", action="store_true", - help="pause after every feature") + add_argument("--db-host", default="127.0.0.1", help="server host (default is 127.0.0.1)") + add_argument("--db-port", default="7687", help="server port (default is 7687)") + add_argument("--db-user", default="memgraph", help="server user (default is memgraph)") + add_argument("--db-pass", default="memgraph", help="server pass (default is memgraph)") + add_argument("--stop", action="store_true", help="stop testing after first fail") + add_argument("--single-fail", action="store_true", help="pause after failed scenario") + add_argument("--single-scenario", action="store_true", help="pause after every scenario") + add_argument("--single-feature", action="store_true", help="pause after every feature") add_argument("--stats-file", default="", help="statistics output file") + add_argument("--storage-mode", default="in_memory", help="Memgraph storage mode") # Parse arguments parsed_args = argp.parse_args() @@ -96,5 +90,5 @@ def main(): return behave_main(behave_args) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/tests/gql_behave/tests/config.yaml b/tests/gql_behave/tests/config.yaml index 8fbda1ed7..4efaaaed6 100644 --- a/tests/gql_behave/tests/config.yaml +++ b/tests/gql_behave/tests/config.yaml @@ -1,15 +1,39 @@ - name: memgraph_V1 test_suite: memgraph_V1 + storage_mode: IN_MEMORY_TRANSACTIONAL must_pass: true - name: openCypher_M09 test_suite: openCypher_M09 + storage_mode: IN_MEMORY_TRANSACTIONAL must_pass: false - name: stackoverflow_answers test_suite: stackoverflow_answers + storage_mode: IN_MEMORY_TRANSACTIONAL must_pass: true - name: unstable test_suite: unstable + storage_mode: IN_MEMORY_TRANSACTIONAL + must_pass: false + +- name: memgraph_V1_on_disk + test_suite: memgraph_V1 + storage_mode: ON_DISK_TRANSACTIONAL + must_pass: true + +- name: openCypher_M09_on_disk + test_suite: openCypher_M09 + storage_mode: ON_DISK_TRANSACTIONAL + must_pass: false + +- name: stackoverflow_answers_on_disk + test_suite: stackoverflow_answers + storage_mode: ON_DISK_TRANSACTIONAL + must_pass: true + +- name: unstable_on_disk + test_suite: unstable + storage_mode: ON_DISK_TRANSACTIONAL must_pass: false diff --git a/tests/gql_behave/tests/memgraph_V1/features/memgraph_exists.feature b/tests/gql_behave/tests/memgraph_V1/features/memgraph_exists.feature index fb24f6270..aef6940ce 100644 --- a/tests/gql_behave/tests/memgraph_V1/features/memgraph_exists.feature +++ b/tests/gql_behave/tests/memgraph_V1/features/memgraph_exists.feature @@ -527,3 +527,15 @@ Feature: WHERE exists MATCH (n:Two) SET n.prop = exists((n)<-[:TYPE]-()) RETURN n.prop; """ Then an error should be raised + + Scenario: Test exists does not work in RETURN clauses + Given an empty graph + And having executed: + """ + CREATE (:One {prop:1})-[:TYPE]->(:Two); + """ + When executing query: + """ + MATCH (n) RETURN exists((n)-[]-()); + """ + Then an error should be raised diff --git a/tests/gql_behave/tests/openCypher_M09/features/OptionalMatchAcceptance.feature b/tests/gql_behave/tests/openCypher_M09/features/OptionalMatchAcceptance.feature index 391afe337..b2d680d72 100644 --- a/tests/gql_behave/tests/openCypher_M09/features/OptionalMatchAcceptance.feature +++ b/tests/gql_behave/tests/openCypher_M09/features/OptionalMatchAcceptance.feature @@ -323,3 +323,16 @@ Feature: OptionalMatchAcceptance | a | b | | [] | [42, 43, 44] | And no side effects + + Scenario: Declaring a path with only one node in OPTIONAL MATCH after MATCH in which that node is already used + When executing query: + """ + MATCH (n1) OPTIONAL MATCH p=(n1) RETURN p; + """ + Then the result should be: + | p | + | <(:Single)> | + | <(:A {prop: 42})> | + | <(:B {prop: 46})> | + | <(:C)> | + And no side effects diff --git a/tests/integration/durability/tests/v16/test_all/create_dataset.cypher b/tests/integration/durability/tests/v16/test_all/create_dataset.cypher new file mode 100644 index 000000000..84ce30605 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_all/create_dataset.cypher @@ -0,0 +1,18 @@ +// --storage-items-per-batch is set to 10 +CREATE INDEX ON :`label2`(`prop2`); +CREATE INDEX ON :`label2`(`prop`); +CREATE INDEX ON :`label`; +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"}); +CREATE (:__mg_vertex__:`label2`:`label` {__mg_id__: 1, `ext`: 2, `prop`: "joj"}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE; +ANALYZE GRAPH; +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_all/expected_snapshot.cypher b/tests/integration/durability/tests/v16/test_all/expected_snapshot.cypher new file mode 100644 index 000000000..bbb171c08 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_all/expected_snapshot.cypher @@ -0,0 +1,16 @@ +CREATE INDEX ON :`label`; +CREATE INDEX ON :`label2`(`prop2`); +CREATE INDEX ON :`label2`(`prop`); +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"}); +CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `ext`: 2, `prop`: "joj"}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE; +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_all/expected_wal.cypher b/tests/integration/durability/tests/v16/test_all/expected_wal.cypher new file mode 100644 index 000000000..2e704f516 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_all/expected_wal.cypher @@ -0,0 +1,16 @@ +CREATE INDEX ON :`label`; +CREATE INDEX ON :`label2`(`prop2`); +CREATE INDEX ON :`label2`(`prop`); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE; +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `prop`: "joj", `ext`: 2}); +CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `prop`: "joj", `ext`: 2}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v); +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_all/snapshot.bin b/tests/integration/durability/tests/v16/test_all/snapshot.bin new file mode 100644 index 000000000..c73cf0888 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_all/snapshot.bin differ diff --git a/tests/integration/durability/tests/v16/test_all/wal.bin b/tests/integration/durability/tests/v16/test_all/wal.bin new file mode 100644 index 000000000..88fc60550 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_all/wal.bin differ diff --git a/tests/integration/durability/tests/v16/test_constraints/create_dataset.cypher b/tests/integration/durability/tests/v16/test_constraints/create_dataset.cypher new file mode 100644 index 000000000..96bb4bac4 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_constraints/create_dataset.cypher @@ -0,0 +1,6 @@ +CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`a`, u.`b` IS UNIQUE; diff --git a/tests/integration/durability/tests/v16/test_constraints/expected_snapshot.cypher b/tests/integration/durability/tests/v16/test_constraints/expected_snapshot.cypher new file mode 100644 index 000000000..fbe2c28ab --- /dev/null +++ b/tests/integration/durability/tests/v16/test_constraints/expected_snapshot.cypher @@ -0,0 +1,6 @@ +CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`b`, u.`a` IS UNIQUE; diff --git a/tests/integration/durability/tests/v16/test_constraints/expected_wal.cypher b/tests/integration/durability/tests/v16/test_constraints/expected_wal.cypher new file mode 100644 index 000000000..9260455ed --- /dev/null +++ b/tests/integration/durability/tests/v16/test_constraints/expected_wal.cypher @@ -0,0 +1,6 @@ +CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`); +CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`); +CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`a`, u.`b` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE; +CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE; diff --git a/tests/integration/durability/tests/v16/test_constraints/snapshot.bin b/tests/integration/durability/tests/v16/test_constraints/snapshot.bin new file mode 100644 index 000000000..ace6448cd Binary files /dev/null and b/tests/integration/durability/tests/v16/test_constraints/snapshot.bin differ diff --git a/tests/integration/durability/tests/v16/test_constraints/wal.bin b/tests/integration/durability/tests/v16/test_constraints/wal.bin new file mode 100644 index 000000000..f2d54e5fd Binary files /dev/null and b/tests/integration/durability/tests/v16/test_constraints/wal.bin differ diff --git a/tests/integration/durability/tests/v16/test_edges/create_dataset.cypher b/tests/integration/durability/tests/v16/test_edges/create_dataset.cypher new file mode 100644 index 000000000..ab3b3af6d --- /dev/null +++ b/tests/integration/durability/tests/v16/test_edges/create_dataset.cypher @@ -0,0 +1,60 @@ +// --storage-items-per-batch is set to 7 +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__ {__mg_id__: 1}); +CREATE (:__mg_vertex__ {__mg_id__: 2}); +CREATE (:__mg_vertex__ {__mg_id__: 3}); +CREATE (:__mg_vertex__ {__mg_id__: 4}); +CREATE (:__mg_vertex__ {__mg_id__: 5}); +CREATE (:__mg_vertex__ {__mg_id__: 6}); +CREATE (:__mg_vertex__ {__mg_id__: 7}); +CREATE (:__mg_vertex__ {__mg_id__: 8}); +CREATE (:__mg_vertex__ {__mg_id__: 9}); +CREATE (:__mg_vertex__ {__mg_id__: 10}); +CREATE (:__mg_vertex__ {__mg_id__: 11}); +CREATE (:__mg_vertex__ {__mg_id__: 12}); +CREATE (:__mg_vertex__ {__mg_id__: 13}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 14}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 15}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +ANALYZE GRAPH; +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_edges/expected_snapshot.cypher b/tests/integration/durability/tests/v16/test_edges/expected_snapshot.cypher new file mode 100644 index 000000000..596753ba5 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_edges/expected_snapshot.cypher @@ -0,0 +1,58 @@ +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__ {__mg_id__: 1}); +CREATE (:__mg_vertex__ {__mg_id__: 2}); +CREATE (:__mg_vertex__ {__mg_id__: 3}); +CREATE (:__mg_vertex__ {__mg_id__: 4}); +CREATE (:__mg_vertex__ {__mg_id__: 5}); +CREATE (:__mg_vertex__ {__mg_id__: 6}); +CREATE (:__mg_vertex__ {__mg_id__: 7}); +CREATE (:__mg_vertex__ {__mg_id__: 8}); +CREATE (:__mg_vertex__ {__mg_id__: 9}); +CREATE (:__mg_vertex__ {__mg_id__: 10}); +CREATE (:__mg_vertex__ {__mg_id__: 11}); +CREATE (:__mg_vertex__ {__mg_id__: 12}); +CREATE (:__mg_vertex__ {__mg_id__: 13}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 14}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 15}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_edges/expected_wal.cypher b/tests/integration/durability/tests/v16/test_edges/expected_wal.cypher new file mode 100644 index 000000000..596753ba5 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_edges/expected_wal.cypher @@ -0,0 +1,58 @@ +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__ {__mg_id__: 1}); +CREATE (:__mg_vertex__ {__mg_id__: 2}); +CREATE (:__mg_vertex__ {__mg_id__: 3}); +CREATE (:__mg_vertex__ {__mg_id__: 4}); +CREATE (:__mg_vertex__ {__mg_id__: 5}); +CREATE (:__mg_vertex__ {__mg_id__: 6}); +CREATE (:__mg_vertex__ {__mg_id__: 7}); +CREATE (:__mg_vertex__ {__mg_id__: 8}); +CREATE (:__mg_vertex__ {__mg_id__: 9}); +CREATE (:__mg_vertex__ {__mg_id__: 10}); +CREATE (:__mg_vertex__ {__mg_id__: 11}); +CREATE (:__mg_vertex__ {__mg_id__: 12}); +CREATE (:__mg_vertex__ {__mg_id__: 13}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 14}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 15}); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v); +MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v); +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_edges/snapshot.bin b/tests/integration/durability/tests/v16/test_edges/snapshot.bin new file mode 100644 index 000000000..4ee0830d2 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_edges/snapshot.bin differ diff --git a/tests/integration/durability/tests/v16/test_edges/wal.bin b/tests/integration/durability/tests/v16/test_edges/wal.bin new file mode 100644 index 000000000..914f49154 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_edges/wal.bin differ diff --git a/tests/integration/durability/tests/v16/test_indices/create_dataset.cypher b/tests/integration/durability/tests/v16/test_indices/create_dataset.cypher new file mode 100644 index 000000000..b4debe97d --- /dev/null +++ b/tests/integration/durability/tests/v16/test_indices/create_dataset.cypher @@ -0,0 +1,5 @@ +CREATE INDEX ON :`label2`; +CREATE INDEX ON :`label2`(`prop2`); +CREATE INDEX ON :`label`(`prop2`); +CREATE INDEX ON :`label`(`prop`); +ANALYZE GRAPH; diff --git a/tests/integration/durability/tests/v16/test_indices/expected_snapshot.cypher b/tests/integration/durability/tests/v16/test_indices/expected_snapshot.cypher new file mode 100644 index 000000000..2df191aa5 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_indices/expected_snapshot.cypher @@ -0,0 +1,4 @@ +CREATE INDEX ON :`label2`; +CREATE INDEX ON :`label`(`prop`); +CREATE INDEX ON :`label`(`prop2`); +CREATE INDEX ON :`label2`(`prop2`); diff --git a/tests/integration/durability/tests/v16/test_indices/expected_wal.cypher b/tests/integration/durability/tests/v16/test_indices/expected_wal.cypher new file mode 100644 index 000000000..43344cbef --- /dev/null +++ b/tests/integration/durability/tests/v16/test_indices/expected_wal.cypher @@ -0,0 +1,4 @@ +CREATE INDEX ON :`label2`; +CREATE INDEX ON :`label2`(`prop2`); +CREATE INDEX ON :`label`(`prop2`); +CREATE INDEX ON :`label`(`prop`); diff --git a/tests/integration/durability/tests/v16/test_indices/snapshot.bin b/tests/integration/durability/tests/v16/test_indices/snapshot.bin new file mode 100644 index 000000000..4c1c25838 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_indices/snapshot.bin differ diff --git a/tests/integration/durability/tests/v16/test_indices/wal.bin b/tests/integration/durability/tests/v16/test_indices/wal.bin new file mode 100644 index 000000000..03122b25a Binary files /dev/null and b/tests/integration/durability/tests/v16/test_indices/wal.bin differ diff --git a/tests/integration/durability/tests/v16/test_vertices/create_dataset.cypher b/tests/integration/durability/tests/v16/test_vertices/create_dataset.cypher new file mode 100644 index 000000000..061df375b --- /dev/null +++ b/tests/integration/durability/tests/v16/test_vertices/create_dataset.cypher @@ -0,0 +1,18 @@ +// --storage-items-per-batch is set to 5 +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141}); +CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop3`: true, `prop2`: -314000000}); +CREATE (:__mg_vertex__:`label3`:`label1`:`label2` {__mg_id__: 7}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop3`: "str", `prop2`: 2, `prop`: 1}); +CREATE (:__mg_vertex__:`label2`:`label1` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]}); +CREATE (:__mg_vertex__:`label3`:`label` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}}); +CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"}); +ANALYZE GRAPH; +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_vertices/expected_snapshot.cypher b/tests/integration/durability/tests/v16/test_vertices/expected_snapshot.cypher new file mode 100644 index 000000000..ecdc1229e --- /dev/null +++ b/tests/integration/durability/tests/v16/test_vertices/expected_snapshot.cypher @@ -0,0 +1,16 @@ +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141}); +CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop3`: true, `prop2`: -314000000}); +CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop3`: "str", `prop2`: 2, `prop`: 1}); +CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]}); +CREATE (:__mg_vertex__:`label`:`label3` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}}); +CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"}); +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_vertices/expected_wal.cypher b/tests/integration/durability/tests/v16/test_vertices/expected_wal.cypher new file mode 100644 index 000000000..d8f758737 --- /dev/null +++ b/tests/integration/durability/tests/v16/test_vertices/expected_wal.cypher @@ -0,0 +1,16 @@ +CREATE INDEX ON :__mg_vertex__(__mg_id__); +CREATE (:__mg_vertex__ {__mg_id__: 0}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true}); +CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141}); +CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop2`: -314000000, `prop3`: true}); +CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop`: 1, `prop2`: 2, `prop3`: "str"}); +CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}}); +CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]}); +CREATE (:__mg_vertex__:`label`:`label3` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}}); +CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"}); +DROP INDEX ON :__mg_vertex__(__mg_id__); +MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__; diff --git a/tests/integration/durability/tests/v16/test_vertices/snapshot.bin b/tests/integration/durability/tests/v16/test_vertices/snapshot.bin new file mode 100644 index 000000000..88aff06b9 Binary files /dev/null and b/tests/integration/durability/tests/v16/test_vertices/snapshot.bin differ diff --git a/tests/integration/durability/tests/v16/test_vertices/wal.bin b/tests/integration/durability/tests/v16/test_vertices/wal.bin new file mode 100644 index 000000000..304db455f Binary files /dev/null and b/tests/integration/durability/tests/v16/test_vertices/wal.bin differ diff --git a/tests/integration/run_time_settings/flag_tester.cpp b/tests/integration/run_time_settings/flag_tester.cpp index 6031b26c8..8a723d1ae 100644 --- a/tests/integration/run_time_settings/flag_tester.cpp +++ b/tests/integration/run_time_settings/flag_tester.cpp @@ -47,7 +47,6 @@ int main(int argc, char **argv) { MG_ASSERT(res.fields[0] == "setting_name", "Expected \"setting_name\" field in the query result."); MG_ASSERT(res.fields[1] == "setting_value", "Expected \"setting_value\" field in the query result."); - unsigned i = 0; for (const auto &record : res.records) { const auto &settings_name = record[0].ValueString(); if (settings_name == FLAGS_field) { diff --git a/tests/manual/single_query.cpp b/tests/manual/single_query.cpp index 5b0e138de..f8063c14e 100644 --- a/tests/manual/single_query.cpp +++ b/tests/manual/single_query.cpp @@ -41,7 +41,7 @@ int main(int argc, char *argv[]) { memgraph::query::Interpreter interpreter{&interpreter_context, db_acc}; ResultStreamFaker stream(db_acc->storage()); - auto [header, _1, qid, _2] = interpreter.Prepare(argv[1], {}, nullptr); + auto [header, _1, qid, _2] = interpreter.Prepare(argv[1], {}, {}); stream.Header(header); auto summary = interpreter.PullAll(&stream); stream.Summary(summary); diff --git a/tests/mgbench/README.md b/tests/mgbench/README.md index 813090f66..f4181cf11 100644 --- a/tests/mgbench/README.md +++ b/tests/mgbench/README.md @@ -342,6 +342,12 @@ Benchgraph is currently a passive benchmark since resource usage and saturation Latest version: https://memgraph.com/benchgraph +### Release v3 (latest) - 2023-02-10 + +- Improvements have been made for the sake of Memgraph internal performance testing. + - https://github.com/memgraph/memgraph/pull/1286 + - https://github.com/memgraph/memgraph/pull/1280 + ### Release v2 (latest) - 2023-25-04 - Benchmark process changes: diff --git a/tests/mgbench/benchmark.py b/tests/mgbench/benchmark.py index 12af6ead7..cd3fb846f 100755 --- a/tests/mgbench/benchmark.py +++ b/tests/mgbench/benchmark.py @@ -19,6 +19,7 @@ import platform import random import sys import time +from copy import deepcopy from typing import Dict, List import helpers @@ -26,54 +27,11 @@ import log import runners import setup from benchmark_context import BenchmarkContext +from benchmark_results import BenchmarkResults +from constants import * from workload_mode import BENCHMARK_MODE_MIXED, BENCHMARK_MODE_REALISTIC from workloads import * -WITH_FINE_GRAINED_AUTHORIZATION = "with_fine_grained_authorization" -WITHOUT_FINE_GRAINED_AUTHORIZATION = "without_fine_grained_authorization" -RUN_CONFIGURATION = "__run_configuration__" -IMPORT = "__import__" -THROUGHPUT = "throughput" -DATABASE = "database" -MEMORY = "memory" -VENDOR = "vendor" -CONDITION = "condition" -NUM_WORKERS_FOR_BENCHMARK = "num_workers_for_benchmark" -SINGLE_THREADED_RUNTIME_SEC = "single_threaded_runtime_sec" -BENCHMARK_MODE = "benchmark_mode" -BENCHMARK_MODE_CONFIG = "benchmark_mode_config" -PLATFORM = "platform" -COUNT = "count" -DURATION = "duration" -NUM_WORKERS = "num_workers" -CPU = "cpu" -MEMORY = "memory" -VENDOR_RUNNER_IMPORT = "import" -VENDOR_RUNNER_AUTHORIZATION = "authorization" -CLIENT = "client" -DATABASE = "database" -CUSTOM_LOAD = "custom_load" -RETRIES = "retries" -DOCKER = "docker" -METADATA = "metadata" -LATENCY_STATS = "latency_stats" -ITERATIONS = "iterations" -USERNAME = "user" -PASSWORD = "test" -DATABASE_CONDITION_HOT = "hot" -DATABASE_CONDITION_VULCANIC = "vulcanic" -WRITE_TYPE_QUERY = "write" -READ_TYPE_QUERY = "read" -UPDATE_TYPE_QUERY = "update" -ANALYTICAL_TYPE_QUERY = "analytical" -QUERY = "query" -CACHE = "cache" -DISK_PREPARATION_RSS = "disk_storage_preparation" -IN_MEMORY_ANALYTICAL_RSS = "in_memory_analytical_preparation" - -IN_MEMORY_TRANSACTIONAL = "IN_MEMORY_TRANSACTIONAL" - - WARMUP_TO_HOT_QUERIES = [ ("CREATE ();", {}), ("CREATE ()-[:TempEdge]->();", {}), @@ -161,7 +119,19 @@ def parse_args(): benchmark_parser.add_argument( "--export-results", default=None, - help="file path into which results should be exported", + help="file path into which results for in_memory_transactional storage mode should be exported", + ) + + benchmark_parser.add_argument( + "--export-results-in-memory-analytical", + default=None, + help="File path into which results for in_memory_analytical storage mode should be exported. If set, benchmarks for analytical mode will be run.", + ) + + benchmark_parser.add_argument( + "--export-results-on-disk-txn", + default=None, + help="File path into which results for on_disk_transactional storage mode should be exported. If set, benchmarks for disk storage will be run.", ) benchmark_parser.add_argument( @@ -220,20 +190,6 @@ def parse_args(): benchmark_parser.add_argument("--customer-workloads", default=None, help="Path to customers workloads") - benchmark_parser.add_argument( - "--disk-storage", - action="store_true", - default=False, - help="If the flag set, benchmarks will be run also for disk storage.", - ) - - benchmark_parser.add_argument( - "--in-memory-analytical", - action="store_true", - default=False, - help="If the flag set, benchmarks will be run also for in_memory_analytical.", - ) - benchmark_parser.add_argument( "--vendor-specific", nargs="*", @@ -279,6 +235,17 @@ def parse_args(): return parser.parse_args() +def sanitize_args(args): + assert args.benchmarks != None, helpers.list_available_workloads() + assert args.num_workers_for_import > 0 + assert args.num_workers_for_benchmark > 0 + assert args.export_results != None, "Pass where will results be saved" + assert args.single_threaded_runtime_sec >= 1, "Low runtime value, consider extending time for more accurate results" + assert ( + args.workload_realistic == None or args.workload_mixed == None + ), "Cannot run both realistic and mixed workload, only one mode run at the time" + + def get_queries(gen, count): random.seed(gen.__name__) ret = [] @@ -287,21 +254,6 @@ def get_queries(gen, count): return ret -def warmup(condition: str, client: runners.BaseRunner, queries: list = None): - if condition == DATABASE_CONDITION_HOT: - log.log("Execute warm-up to match condition: {} ".format(condition)) - client.execute( - queries=WARMUP_TO_HOT_QUERIES, - num_workers=1, - ) - elif condition == DATABASE_CONDITION_VULCANIC: - log.log("Execute warm-up to match condition: {} ".format(condition)) - client.execute(queries=queries) - else: - log.log("No warm-up on condition: {} ".format(condition)) - log.log("Finished warm-up procedure to match database condition: {} ".format(condition)) - - def validate_workload_distribution(percentage_distribution, queries_by_type): percentages_by_type = { WRITE_TYPE_QUERY: percentage_distribution[0], @@ -360,7 +312,13 @@ def prepare_for_workload(benchmark_context, dataset, group, queries): def realistic_workload( - vendor: runners.BaseRunner, client: runners.BaseClient, dataset, group, queries, benchmark_context: BenchmarkContext + vendor: runners.BaseRunner, + client: runners.BaseClient, + dataset, + group, + queries, + benchmark_context: BenchmarkContext, + results, ): log.log("Executing realistic workload...") config_distribution, queries_by_type, _, percentage_distribution, num_of_queries = prepare_for_workload( @@ -407,7 +365,13 @@ def realistic_workload( def mixed_workload( - vendor: runners.BaseRunner, client: runners.BaseClient, dataset, group, queries, benchmark_context: BenchmarkContext + vendor: runners.BaseRunner, + client: runners.BaseClient, + dataset, + group, + queries, + benchmark_context: BenchmarkContext, + results, ): log.log("Executing mixed workload...") ( @@ -467,6 +431,21 @@ def mixed_workload( results.set_value(*results_key, value=ret) +def warmup(condition: str, client: runners.BaseRunner, queries: list = None): + if condition == DATABASE_CONDITION_HOT: + log.log("Execute warm-up to match condition: {} ".format(condition)) + client.execute( + queries=WARMUP_TO_HOT_QUERIES, + num_workers=1, + ) + elif condition == DATABASE_CONDITION_VULCANIC: + log.log("Execute warm-up to match condition: {} ".format(condition)) + client.execute(queries=queries) + else: + log.log("No warm-up on condition: {} ".format(condition)) + log.log("Finished warm-up procedure to match database condition: {} ".format(condition)) + + def get_query_cache_count( vendor: runners.BaseRunner, client: runners.BaseClient, @@ -477,16 +456,12 @@ def get_query_cache_count( query: str, func: str, ): - log.init("Determining query count for benchmark based on --single-threaded-runtime argument") + log.init( + f"Determining query count for benchmark based on --single-threaded-runtime argument = {benchmark_context.single_threaded_runtime_sec}s" + ) config_key = [workload.NAME, workload.get_variant(), group, query] cached_count = config.get_value(*config_key) if cached_count is None: - log.info( - "Determining the number of queries necessary for {} seconds of single-threaded runtime...".format( - benchmark_context.single_threaded_runtime_sec - ) - ) - log.log("Running query to prime the query cache...") vendor.start_db(CACHE) client.execute(queries=queries, num_workers=1) count = 1 @@ -495,18 +470,16 @@ def get_query_cache_count( duration = ret[0][DURATION] should_execute = int(benchmark_context.single_threaded_runtime_sec / (duration / count)) log.log( - "executed_queries={}, total_duration={}, query_duration={}, estimated_count={}".format( + "Executed_queries={}, total_duration={}, query_duration={}, estimated_count={}".format( count, duration, duration / count, should_execute ) ) - # We don't have to execute the next iteration when - # `should_execute` becomes the same order of magnitude as - # `count * 10`. if should_execute / (count * 10) < 10: count = should_execute break else: count = count * 10 + vendor.stop_db(CACHE) if count < benchmark_context.query_count_lower_bound: @@ -529,38 +502,6 @@ def get_query_cache_count( return count -def log_benchmark_summary(results: Dict): - log.init("~" * 45) - log.info("Benchmark finished.") - log.init("~" * 45) - log.log("\n") - log.summary("Benchmark summary") - log.log("-" * 90) - log.summary("{:<20} {:>30} {:>30}".format("Query name", "Throughput", "Peak Memory usage")) - for dataset, variants in results.items(): - if dataset == RUN_CONFIGURATION: - continue - for groups in variants.values(): - for group, queries in groups.items(): - if group == IMPORT: - continue - for query, auth in queries.items(): - for value in auth.values(): - log.log("-" * 90) - log.summary( - "{:<20} {:>26.2f} QPS {:>27.2f} MB".format( - query, value[THROUGHPUT], value[DATABASE][MEMORY] / (1024.0 * 1024.0) - ) - ) - log.log("-" * 90) - - -def log_benchmark_arguments(benchmark_context): - log.init("Executing benchmark with following arguments: ") - for key, value in benchmark_context.__dict__.items(): - log.log("{:<30} : {:<30}".format(str(key), str(value))) - - def check_benchmark_requirements(benchmark_context): if setup.check_requirements(benchmark_context): log.success("Requirements for starting benchmark satisfied!") @@ -577,19 +518,6 @@ def setup_cache_config(benchmark_context, cache): return helpers.RecursiveDict() -def sanitize_args(args): - assert args.benchmarks != None, helpers.list_available_workloads() - assert args.num_workers_for_import > 0 - assert args.num_workers_for_benchmark > 0 - assert args.export_results != None, "Pass where will results be saved" - assert ( - args.single_threaded_runtime_sec >= 10 - ), "Low runtime value, consider extending time for more accurate results" - assert ( - args.workload_realistic == None or args.workload_mixed == None - ), "Cannot run both realistic and mixed workload, only one mode run at the time" - - def save_import_results(workload, results, import_results, rss_usage): log.info("Summarized importing benchmark results:") import_key = [workload.NAME, workload.get_variant(), IMPORT] @@ -597,21 +525,279 @@ def save_import_results(workload, results, import_results, rss_usage): # Display import statistics. for row in import_results: log.success( - "Executed {} queries in {} seconds using {} workers with a total throughput of {} Q/S.".format( - row[COUNT], row[DURATION], row[NUM_WORKERS], row[THROUGHPUT] - ) + f"Executed {row[COUNT]} queries in {row[DURATION]} seconds using {row[NUM_WORKERS]} workers with a total throughput of {row[THROUGHPUT]} Q/S." ) log.success( - "The database used {} seconds of CPU time and peaked at {} MiB of RAM".format( - rss_usage[CPU], rss_usage[MEMORY] / (1024 * 1024) - ) + f"The database used {rss_usage[CPU]} seconds of CPU time and peaked at {rss_usage[MEMORY] / (1024 * 1024)} MiB of RAM" ) + results.set_value(*import_key, value={CLIENT: import_results, DATABASE: rss_usage}) else: results.set_value(*import_key, value={CLIENT: CUSTOM_LOAD, DATABASE: CUSTOM_LOAD}) +def save_to_results(results, ret, workload, group, query, authorization_mode): + results_key = [ + workload.NAME, + workload.get_variant(), + group, + query, + authorization_mode, + ] + results.set_value(*results_key, value=ret) + + +def run_isolated_workload_with_authorization(vendor_runner, client, queries, group, workload, results): + log.init("Running isolated workload with authorization") + + log.info("Running preprocess AUTH queries") + vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) + client.execute(queries=SETUP_AUTH_QUERIES) + client.set_credentials(username=USERNAME, password=PASSWORD) + vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) + + for query, funcname in queries[group]: + log.init("Running query:" + "{}/{}/{}/{}".format(group, query, funcname, WITH_FINE_GRAINED_AUTHORIZATION)) + func = getattr(workload, funcname) + count = get_query_cache_count( + vendor_runner, client, get_queries(func, 1), benchmark_context, workload, group, query, func + ) + + vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) + start_time = time.time() + warmup(condition=benchmark_context.warm_up, client=client, queries=get_queries(func, count)) + + ret = client.execute( + queries=get_queries(func, count), + num_workers=benchmark_context.num_workers_for_benchmark, + )[0] + usage = vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) + time_elapsed = time.time() - start_time + log.info(f"Benchmark execution of query {funcname} finished in {time_elapsed} seconds.") + + ret[DATABASE] = usage + log_metrics_summary(ret, usage) + log_metadata_summary(ret) + log.success("Throughput: {:02f} QPS".format(ret[THROUGHPUT])) + save_to_results(results, ret, workload, group, query, WITH_FINE_GRAINED_AUTHORIZATION) + + vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) + log.info("Running cleanup of auth queries") + ret = client.execute(queries=CLEANUP_AUTH_QUERIES) + vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) + + +def run_isolated_workload_without_authorization(vendor_runner, client, queries, group, workload, results): + log.init("Running isolated workload without authorization") + for query, funcname in queries[group]: + log.init( + "Running query:" + "{}/{}/{}/{}".format(group, query, funcname, WITHOUT_FINE_GRAINED_AUTHORIZATION), + ) + func = getattr(workload, funcname) + count = get_query_cache_count( + vendor_runner, client, get_queries(func, 1), benchmark_context, workload, group, query, func + ) + + # Benchmark run. + sample_query = get_queries(func, 1)[0][0] + log.info("Sample query:{}".format(sample_query)) + log.log( + "Executing benchmark with {} queries that should yield a single-threaded runtime of {} seconds.".format( + count, benchmark_context.single_threaded_runtime_sec + ) + ) + log.log("Queries are executed using {} concurrent clients".format(benchmark_context.num_workers_for_benchmark)) + + start_time = time.time() + rss_db = workload.NAME + workload.get_variant() + "_" + "_" + benchmark_context.mode + "_" + query + vendor_runner.start_db(rss_db) + warmup(condition=benchmark_context.warm_up, client=client, queries=get_queries(func, count)) + log.init("Executing benchmark queries...") + ret = client.execute( + queries=get_queries(func, count), + num_workers=benchmark_context.num_workers_for_benchmark, + time_dependent_execution=benchmark_context.time_dependent_execution, + )[0] + + time_elapsed = time.time() - start_time + + log.info(f"Benchmark execution of query {funcname} finished in {time_elapsed} seconds.") + usage = vendor_runner.stop_db(rss_db) + + ret[DATABASE] = usage + log_output_summary(benchmark_context, ret, usage, funcname, sample_query) + + save_to_results(results, ret, workload, group, query, WITHOUT_FINE_GRAINED_AUTHORIZATION) + + +def setup_indices_and_import_dataset(client, vendor_runner, generated_queries, workload, storage_mode): + vendor_runner.start_db_init(VENDOR_RUNNER_IMPORT) + log.info("Executing database index setup") + start_time = time.time() + + if generated_queries: + client.execute(queries=workload.indexes_generator(), num_workers=1) + log.info("Finished setting up indexes.") + log.info("Started importing dataset") + import_results = client.execute(queries=generated_queries, num_workers=benchmark_context.num_workers_for_import) + else: + log.info("Using workload information for importing dataset and creating indices") + log.info("Preparing workload: " + workload.NAME + "/" + workload.get_variant()) + workload.prepare(cache.cache_directory("datasets", workload.NAME, workload.get_variant())) + imported = workload.custom_import() + if not imported: + client.execute(file_path=workload.get_index(), num_workers=1) + log.info("Finished setting up indexes.") + log.info("Started importing dataset") + if storage_mode == ON_DISK_TRANSACTIONAL: + import_results = client.execute(file_path=workload.get_node_file(), num_workers=1) + import_results = client.execute(file_path=workload.get_edge_file(), num_workers=1) + else: + import_results = client.execute( + file_path=workload.get_file(), num_workers=benchmark_context.num_workers_for_import + ) + else: + log.info("Custom import executed") + + log.info(f"Finished importing dataset in {time.time() - start_time}s") + rss_usage = vendor_runner.stop_db_init(VENDOR_RUNNER_IMPORT) + + return import_results, rss_usage + + +def run_target_workload(benchmark_context, workload, bench_queries, vendor_runner, client, results, storage_mode): + generated_queries = workload.dataset_generator() + import_results, rss_usage = setup_indices_and_import_dataset( + client, vendor_runner, generated_queries, workload, storage_mode + ) + save_import_results(workload, results, import_results, rss_usage) + + for group in sorted(bench_queries.keys()): + log.init(f"\nRunning benchmark in {benchmark_context.mode} workload mode for {group} group") + if benchmark_context.mode == BENCHMARK_MODE_MIXED: + mixed_workload(vendor_runner, client, workload, group, bench_queries, benchmark_context, results) + elif benchmark_context.mode == BENCHMARK_MODE_REALISTIC: + realistic_workload(vendor_runner, client, workload, group, bench_queries, benchmark_context, results) + else: + run_isolated_workload_without_authorization(vendor_runner, client, bench_queries, group, workload, results) + + if benchmark_context.no_authorization: + run_isolated_workload_with_authorization(vendor_runner, client, bench_queries, group, workload, results) + + +# TODO: (andi) Reorder functions in top-down notion in order to improve readibility +def run_target_workloads(benchmark_context, target_workloads, bench_results): + for workload, bench_queries in target_workloads: + log.info(f"Started running {str(workload.NAME)} workload") + + benchmark_context.set_active_workload(workload.NAME) + benchmark_context.set_active_variant(workload.get_variant()) + + if workload.is_disk_workload() and benchmark_context.export_results_on_disk_txn: + run_on_disk_transactional_benchmark(benchmark_context, workload, bench_queries, bench_results.disk_results) + else: + run_in_memory_transactional_benchmark( + benchmark_context, workload, bench_queries, bench_results.in_memory_txn_results + ) + + if benchmark_context.export_results_in_memory_analytical: + run_in_memory_analytical_benchmark( + benchmark_context, workload, bench_queries, bench_results.in_memory_analytical_results + ) + + +def run_on_disk_transactional_benchmark(benchmark_context, workload, bench_queries, disk_results): + log.info(f"Running benchmarks for {ON_DISK_TRANSACTIONAL} storage mode.") + disk_vendor_runner, disk_client = client_runner_factory(benchmark_context) + disk_vendor_runner.start_db(DISK_PREPARATION_RSS) + disk_client.execute(queries=SETUP_DISK_STORAGE) + disk_vendor_runner.stop_db(DISK_PREPARATION_RSS) + run_target_workload( + benchmark_context, workload, bench_queries, disk_vendor_runner, disk_client, disk_results, ON_DISK_TRANSACTIONAL + ) + log.info(f"Finished running benchmarks for {ON_DISK_TRANSACTIONAL} storage mode.") + + +def run_in_memory_analytical_benchmark(benchmark_context, workload, bench_queries, in_memory_analytical_results): + log.info(f"Running benchmarks for {IN_MEMORY_ANALYTICAL} storage mode.") + in_memory_analytical_vendor_runner, in_memory_analytical_client = client_runner_factory(benchmark_context) + in_memory_analytical_vendor_runner.start_db(IN_MEMORY_ANALYTICAL_RSS) + in_memory_analytical_client.execute(queries=SETUP_IN_MEMORY_ANALYTICAL_STORAGE_MODE) + in_memory_analytical_vendor_runner.stop_db(IN_MEMORY_ANALYTICAL_RSS) + run_target_workload( + benchmark_context, + workload, + bench_queries, + in_memory_analytical_vendor_runner, + in_memory_analytical_client, + in_memory_analytical_results, + IN_MEMORY_ANALYTICAL, + ) + log.info(f"Finished running benchmarks for {IN_MEMORY_ANALYTICAL} storage mode.") + + +def run_in_memory_transactional_benchmark(benchmark_context, workload, bench_queries, in_memory_txn_results): + log.info(f"Running benchmarks for {IN_MEMORY_TRANSACTIONAL} storage mode.") + in_memory_txn_vendor_runner, in_memory_txn_client = client_runner_factory(benchmark_context) + run_target_workload( + benchmark_context, + workload, + bench_queries, + in_memory_txn_vendor_runner, + in_memory_txn_client, + in_memory_txn_results, + IN_MEMORY_TRANSACTIONAL, + ) + log.info(f"Finished running benchmarks for {IN_MEMORY_TRANSACTIONAL} storage mode.") + + +def client_runner_factory(benchmark_context): + vendor_runner = runners.BaseRunner.create(benchmark_context=benchmark_context) + vendor_runner.clean_db() + log.log("Database cleaned from any previous data") + client = vendor_runner.fetch_client() + return vendor_runner, client + + +def validate_target_workloads(benchmark_context, target_workloads): + if len(target_workloads) == 0: + log.error("No workloads matched the pattern: " + str(benchmark_context.benchmark_target_workload)) + log.error("Please check the pattern and workload NAME property, query group and query name.") + log.info("Currently available workloads: ") + log.log(helpers.list_available_workloads(benchmark_context.customer_workloads)) + sys.exit(1) + + +def log_benchmark_summary(results: Dict, storage_mode): + log.log("\n") + log.summary(f"Benchmark summary of {storage_mode} mode") + log.log("-" * 120) + log.summary("{:<20} {:>30} {:>30}".format("Query name", "Throughput", "Peak Memory usage")) + for dataset, variants in results.items(): + if dataset == RUN_CONFIGURATION: + continue + for groups in variants.values(): + for group, queries in groups.items(): + if group == IMPORT: + continue + for query, auth in queries.items(): + for value in auth.values(): + log.log("-" * 120) + log.summary( + "{:<20} {:>26.2f} QPS {:>27.2f} MB".format( + query, value[THROUGHPUT], value[DATABASE][MEMORY] / (1024.0 * 1024.0) + ) + ) + log.log("-" * 90) + + +def log_benchmark_arguments(benchmark_context): + log.init("Executing benchmark with following arguments: ") + for key, value in benchmark_context.__dict__.items(): + log.log("{:<30} : {:<30}".format(str(key), str(value))) + + def log_metrics_summary(ret, usage): log.log("Executed {} queries in {} seconds.".format(ret[COUNT], ret[DURATION])) log.log("Queries have been retried {} times".format(ret[RETRIES])) @@ -645,175 +831,6 @@ def log_output_summary(benchmark_context, ret, usage, funcname, sample_query): log.success("Throughput: {:02f} QPS\n\n".format(ret[THROUGHPUT])) -def save_to_results(results, ret, workload, group, query, authorization_mode): - results_key = [ - workload.NAME, - workload.get_variant(), - group, - query, - authorization_mode, - ] - results.set_value(*results_key, value=ret) - - -def run_isolated_workload_with_authorization(vendor_runner, client, queries, group, workload): - log.init("Running isolated workload with authorization") - - log.info("Running preprocess AUTH queries") - vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) - client.execute(queries=SETUP_AUTH_QUERIES) - client.set_credentials(username=USERNAME, password=PASSWORD) - vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) - - for query, funcname in queries[group]: - log.init("Running query:" + "{}/{}/{}/{}".format(group, query, funcname, WITH_FINE_GRAINED_AUTHORIZATION)) - func = getattr(workload, funcname) - count = get_query_cache_count( - vendor_runner, client, get_queries(func, 1), benchmark_context, workload, group, query, func - ) - - vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) - warmup(condition=benchmark_context.warm_up, client=client, queries=get_queries(func, count)) - - ret = client.execute( - queries=get_queries(func, count), - num_workers=benchmark_context.num_workers_for_benchmark, - )[0] - usage = vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) - - ret[DATABASE] = usage - log_metrics_summary(ret, usage) - log_metadata_summary(ret) - log.success("Throughput: {:02f} QPS".format(ret[THROUGHPUT])) - save_to_results(results, ret, workload, group, query, WITH_FINE_GRAINED_AUTHORIZATION) - - vendor_runner.start_db(VENDOR_RUNNER_AUTHORIZATION) - log.info("Running cleanup of auth queries") - ret = client.execute(queries=CLEANUP_AUTH_QUERIES) - vendor_runner.stop_db(VENDOR_RUNNER_AUTHORIZATION) - - -def run_isolated_workload_without_authorization(vendor_runner, client, queries, group, workload): - log.init("Running isolated workload without authorization") - for query, funcname in queries[group]: - log.init( - "Running query:" + "{}/{}/{}/{}".format(group, query, funcname, WITHOUT_FINE_GRAINED_AUTHORIZATION), - ) - func = getattr(workload, funcname) - count = get_query_cache_count( - vendor_runner, client, get_queries(func, 1), benchmark_context, workload, group, query, func - ) - - # Benchmark run. - sample_query = get_queries(func, 1)[0][0] - log.info("Sample query:{}".format(sample_query)) - log.log( - "Executing benchmark with {} queries that should yield a single-threaded runtime of {} seconds.".format( - count, benchmark_context.single_threaded_runtime_sec - ) - ) - log.log("Queries are executed using {} concurrent clients".format(benchmark_context.num_workers_for_benchmark)) - - rss_db = workload.NAME + workload.get_variant() + "_" + "_" + benchmark_context.mode + "_" + query - vendor_runner.start_db(rss_db) - warmup(condition=benchmark_context.warm_up, client=client, queries=get_queries(func, count)) - log.init("Executing benchmark queries...") - ret = client.execute( - queries=get_queries(func, count), - num_workers=benchmark_context.num_workers_for_benchmark, - time_dependent_execution=benchmark_context.time_dependent_execution, - )[0] - - log.info("Benchmark execution finished...") - usage = vendor_runner.stop_db(rss_db) - - ret[DATABASE] = usage - log_output_summary(benchmark_context, ret, usage, funcname, sample_query) - - save_to_results(results, ret, workload, group, query, WITHOUT_FINE_GRAINED_AUTHORIZATION) - - -def setup_indices_and_import_dataset(client, vendor_runner, generated_queries, workload): - vendor_runner.start_db_init(VENDOR_RUNNER_IMPORT) - log.info("Executing database index setup") - - if generated_queries: - client.execute(queries=workload.indexes_generator(), num_workers=1) - log.info("Finished setting up indexes.") - log.info("Started importing dataset") - import_results = client.execute(queries=generated_queries, num_workers=benchmark_context.num_workers_for_import) - else: - log.info("Using workload information for importing dataset and creating indices") - log.info("Preparing workload: " + workload.NAME + "/" + workload.get_variant()) - workload.prepare(cache.cache_directory("datasets", workload.NAME, workload.get_variant())) - imported = workload.custom_import() - if not imported: - client.execute(file_path=workload.get_index(), num_workers=1) - log.info("Finished setting up indexes.") - log.info("Started importing dataset") - import_results = client.execute( - file_path=workload.get_file(), num_workers=benchmark_context.num_workers_for_import - ) - else: - log.info("Custom import executed") - - log.info("Finished importing dataset") - rss_usage = vendor_runner.stop_db_init(VENDOR_RUNNER_IMPORT) - - return import_results, rss_usage - - -def run_target_workload(benchmark_context, workload, bench_queries, vendor_runner, client): - generated_queries = workload.dataset_generator() - import_results, rss_usage = setup_indices_and_import_dataset(client, vendor_runner, generated_queries, workload) - save_import_results(workload, results, import_results, rss_usage) - - for group in sorted(bench_queries.keys()): - log.init(f"\nRunning benchmark in {benchmark_context.mode} workload mode for {group} group") - if benchmark_context.mode == BENCHMARK_MODE_MIXED: - mixed_workload(vendor_runner, client, workload, group, bench_queries, benchmark_context) - elif benchmark_context.mode == BENCHMARK_MODE_REALISTIC: - realistic_workload(vendor_runner, client, workload, group, bench_queries, benchmark_context) - else: - run_isolated_workload_without_authorization(vendor_runner, client, bench_queries, group, workload) - - if benchmark_context.no_authorization: - run_isolated_workload_with_authorization(vendor_runner, client, bench_queries, group, workload) - - -# TODO: (andi) Reorder functions in top-down notion in order to improve readibility -def run_target_workloads(benchmark_context, target_workloads): - for workload, bench_queries in target_workloads: - log.info(f"Started running {str(workload.NAME)} workload") - - benchmark_context.set_active_workload(workload.NAME) - benchmark_context.set_active_variant(workload.get_variant()) - - log.info(f"Running benchmarks for {IN_MEMORY_TRANSACTIONAL} storage mode.") - in_memory_txn_vendor_runner, in_memory_txn_client = client_runner_factory(benchmark_context) - run_target_workload( - benchmark_context, workload, bench_queries, in_memory_txn_vendor_runner, in_memory_txn_client - ) - log.info(f"Finished running benchmarks for {IN_MEMORY_TRANSACTIONAL} storage mode.") - - -def client_runner_factory(benchmark_context): - vendor_runner = runners.BaseRunner.create(benchmark_context=benchmark_context) - vendor_runner.clean_db() - log.log("Database cleaned from any previous data") - client = vendor_runner.fetch_client() - return vendor_runner, client - - -def validate_target_workloads(benchmark_context, target_workloads): - if len(target_workloads) == 0: - log.error("No workloads matched the pattern: " + str(benchmark_context.benchmark_target_workload)) - log.error("Please check the pattern and workload NAME property, query group and query name.") - log.info("Currently available workloads: ") - log.log(helpers.list_available_workloads(benchmark_context.customer_workloads)) - sys.exit(1) - - if __name__ == "__main__": args = parse_args() sanitize_args(args) @@ -833,6 +850,8 @@ if __name__ == "__main__": query_count_lower_bound=args.query_count_lower_bound, no_load_query_counts=args.no_load_query_counts, export_results=args.export_results, + export_results_in_memory_analytical=args.export_results_in_memory_analytical, + export_results_on_disk_txn=args.export_results_on_disk_txn, temporary_directory=temp_dir.absolute(), workload_mixed=args.workload_mixed, workload_realistic=args.workload_realistic, @@ -842,8 +861,6 @@ if __name__ == "__main__": no_authorization=args.no_authorization, customer_workloads=args.customer_workloads, vendor_args=vendor_specific_args, - disk_storage=args.disk_storage, - in_memory_analytical=args.in_memory_analytical, ) log_benchmark_arguments(benchmark_context) @@ -854,7 +871,7 @@ if __name__ == "__main__": log.log("Cache folder in use: " + cache.get_default_cache_directory()) config = setup_cache_config(benchmark_context, cache) - run_config = { + in_memory_txn_run_config = { VENDOR: benchmark_context.vendor_name, CONDITION: benchmark_context.warm_up, NUM_WORKERS_FOR_BENCHMARK: benchmark_context.num_workers_for_benchmark, @@ -862,10 +879,19 @@ if __name__ == "__main__": BENCHMARK_MODE: benchmark_context.mode, BENCHMARK_MODE_CONFIG: benchmark_context.mode_config, PLATFORM: platform.platform(), + STORAGE_MODE: IN_MEMORY_TRANSACTIONAL, } - results = helpers.RecursiveDict() - results.set_value(RUN_CONFIGURATION, value=run_config) + in_memory_analytical_run_config = deepcopy(in_memory_txn_run_config) + in_memory_analytical_run_config[STORAGE_MODE] = IN_MEMORY_ANALYTICAL + + on_disk_transactional_run_config = deepcopy(in_memory_txn_run_config) + on_disk_transactional_run_config[STORAGE_MODE] = ON_DISK_TRANSACTIONAL + + bench_results = BenchmarkResults() + bench_results.in_memory_txn_results.set_value(RUN_CONFIGURATION, value=in_memory_txn_run_config) + bench_results.in_memory_analytical_results.set_value(RUN_CONFIGURATION, value=in_memory_analytical_run_config) + bench_results.disk_results.set_value(RUN_CONFIGURATION, value=on_disk_transactional_run_config) available_workloads = helpers.get_available_workloads(benchmark_context.customer_workloads) @@ -876,13 +902,22 @@ if __name__ == "__main__": ) validate_target_workloads(benchmark_context, target_workloads) - run_target_workloads(benchmark_context, target_workloads) + run_target_workloads(benchmark_context, target_workloads, bench_results) if not benchmark_context.no_save_query_counts: cache.save_config(config) - log_benchmark_summary(results.get_data()) - + log_benchmark_summary(bench_results.in_memory_txn_results.get_data(), IN_MEMORY_TRANSACTIONAL) if benchmark_context.export_results: with open(benchmark_context.export_results, "w") as f: - json.dump(results.get_data(), f) + json.dump(bench_results.in_memory_txn_results.get_data(), f) + + log_benchmark_summary(bench_results.in_memory_analytical_results.get_data(), IN_MEMORY_ANALYTICAL) + if benchmark_context.export_results_in_memory_analytical: + with open(benchmark_context.export_results_in_memory_analytical, "w") as f: + json.dump(bench_results.in_memory_analytical_results.get_data(), f) + + log_benchmark_summary(bench_results.disk_results.get_data(), ON_DISK_TRANSACTIONAL) + if benchmark_context.export_results_on_disk_txn: + with open(benchmark_context.export_results_on_disk_txn, "w") as f: + json.dump(bench_results.disk_results.get_data(), f) diff --git a/tests/mgbench/benchmark_context.py b/tests/mgbench/benchmark_context.py index aa1c098bd..034c2c93a 100644 --- a/tests/mgbench/benchmark_context.py +++ b/tests/mgbench/benchmark_context.py @@ -34,6 +34,8 @@ class BenchmarkContext: no_load_query_counts: bool = False, no_save_query_counts: bool = False, export_results: str = None, + export_results_in_memory_analytical: str = None, + export_results_on_disk_txn: str = None, temporary_directory: str = None, workload_mixed: str = None, # Default mode is isolated, mixed None workload_realistic: str = None, # Default mode is isolated, realistic None @@ -43,8 +45,6 @@ class BenchmarkContext: no_authorization: bool = True, customer_workloads: str = None, vendor_args: dict = {}, - disk_storage: bool = False, - in_memory_analytical: bool = False, ) -> None: self.benchmark_target_workload = benchmark_target_workload self.vendor_binary = vendor_binary @@ -57,6 +57,8 @@ class BenchmarkContext: self.no_load_query_counts = no_load_query_counts self.no_save_query_counts = no_save_query_counts self.export_results = export_results + self.export_results_in_memory_analytical = export_results_in_memory_analytical + self.export_results_on_disk_txn = export_results_on_disk_txn self.temporary_directory = temporary_directory assert ( @@ -81,8 +83,6 @@ class BenchmarkContext: self.vendor_args = vendor_args self.active_workload = None self.active_variant = None - self.disk_storage = disk_storage - self.in_memory_analytical = in_memory_analytical def set_active_workload(self, workload: str) -> None: self.active_workload = workload diff --git a/tests/mgbench/benchmark_results.py b/tests/mgbench/benchmark_results.py new file mode 100644 index 000000000..c82cf2d3d --- /dev/null +++ b/tests/mgbench/benchmark_results.py @@ -0,0 +1,19 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import helpers + + +class BenchmarkResults: + def __init__(self) -> None: + self.in_memory_txn_results = helpers.RecursiveDict() + self.in_memory_analytical_results = helpers.RecursiveDict() + self.disk_results = helpers.RecursiveDict() diff --git a/tests/mgbench/constants.py b/tests/mgbench/constants.py new file mode 100644 index 000000000..211d9a413 --- /dev/null +++ b/tests/mgbench/constants.py @@ -0,0 +1,56 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +WITH_FINE_GRAINED_AUTHORIZATION = "with_fine_grained_authorization" +WITHOUT_FINE_GRAINED_AUTHORIZATION = "without_fine_grained_authorization" +RUN_CONFIGURATION = "__run_configuration__" +IMPORT = "__import__" +THROUGHPUT = "throughput" +DATABASE = "database" +MEMORY = "memory" +VENDOR = "vendor" +CONDITION = "condition" +NUM_WORKERS_FOR_BENCHMARK = "num_workers_for_benchmark" +SINGLE_THREADED_RUNTIME_SEC = "single_threaded_runtime_sec" +BENCHMARK_MODE = "benchmark_mode" +BENCHMARK_MODE_CONFIG = "benchmark_mode_config" +PLATFORM = "platform" +COUNT = "count" +DURATION = "duration" +NUM_WORKERS = "num_workers" +CPU = "cpu" +MEMORY = "memory" +VENDOR_RUNNER_IMPORT = "import" +VENDOR_RUNNER_AUTHORIZATION = "authorization" +CLIENT = "client" +DATABASE = "database" +CUSTOM_LOAD = "custom_load" +RETRIES = "retries" +DOCKER = "docker" +METADATA = "metadata" +LATENCY_STATS = "latency_stats" +ITERATIONS = "iterations" +USERNAME = "user" +PASSWORD = "test" +DATABASE_CONDITION_HOT = "hot" +DATABASE_CONDITION_VULCANIC = "vulcanic" +WRITE_TYPE_QUERY = "write" +READ_TYPE_QUERY = "read" +UPDATE_TYPE_QUERY = "update" +ANALYTICAL_TYPE_QUERY = "analytical" +QUERY = "query" +CACHE = "cache" +DISK_PREPARATION_RSS = "disk_storage_preparation" +IN_MEMORY_ANALYTICAL_RSS = "in_memory_analytical_preparation" +STORAGE_MODE = "storage_mode" +IN_MEMORY_TRANSACTIONAL = "IN_MEMORY_TRANSACTIONAL" +IN_MEMORY_ANALYTICAL = "IN_MEMORY_ANALYTICAL" +ON_DISK_TRANSACTIONAL = "ON_DISK_TRANSACTIONAL" diff --git a/tests/mgbench/helpers.py b/tests/mgbench/helpers.py index b860440f9..11d78fea5 100644 --- a/tests/mgbench/helpers.py +++ b/tests/mgbench/helpers.py @@ -20,10 +20,10 @@ import subprocess import sys from pathlib import Path +import log import workloads from benchmark_context import BenchmarkContext from workloads import * -from workloads import base SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -116,7 +116,7 @@ def get_available_workloads(customer_workloads: str = None) -> dict: if key.startswith("_"): continue base_class = getattr(module, key) - if not inspect.isclass(base_class) or not issubclass(base_class, base.Workload): + if not inspect.isclass(base_class) or not issubclass(base_class, workloads.base.Workload): continue queries = collections.defaultdict(list) for funcname in dir(base_class): @@ -137,7 +137,7 @@ def get_available_workloads(customer_workloads: str = None) -> dict: if key.startswith("_"): continue base_class = getattr(dataset_to_use, key) - if not inspect.isclass(base_class) or not issubclass(base_class, base.Workload): + if not inspect.isclass(base_class) or not issubclass(base_class, workloads.base.Workload): continue queries = collections.defaultdict(list) for funcname in dir(base_class): diff --git a/tests/mgbench/log.py b/tests/mgbench/log.py index 64b648937..8b8d8fad1 100644 --- a/tests/mgbench/log.py +++ b/tests/mgbench/log.py @@ -10,6 +10,23 @@ # licenses/APL.txt. import logging +from typing import Dict + +from constants import ( + COUNT, + CPU, + DATABASE, + DOCKER, + DURATION, + IMPORT, + ITERATIONS, + LATENCY_STATS, + MEMORY, + METADATA, + RETRIES, + RUN_CONFIGURATION, + THROUGHPUT, +) COLOR_GRAY = 0 COLOR_RED = 1 diff --git a/tests/mgbench/runners.py b/tests/mgbench/runners.py index acd868377..e1f52b696 100644 --- a/tests/mgbench/runners.py +++ b/tests/mgbench/runners.py @@ -455,7 +455,6 @@ class Memgraph(BaseRunner): raise Exception("The database process died prematurely!") _wait_for_server_socket(self._bolt_port) ret = self._proc_mg.poll() - assert ret is None, "The database process died prematurely " "({})!".format(ret) def _cleanup(self): if self._proc_mg is None: @@ -479,7 +478,6 @@ class Memgraph(BaseRunner): self._stop_event.set() self.dump_rss(workload) ret, usage = self._cleanup() - assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret) return usage def start_db(self, workload): @@ -495,7 +493,6 @@ class Memgraph(BaseRunner): self._stop_event.set() self.dump_rss(workload) ret, usage = self._cleanup() - assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret) return usage def clean_db(self): @@ -644,7 +641,6 @@ class Neo4j(BaseRunner): self._rss.clear() p.start() - # Start DB self._start() if self._performance_tracking: @@ -657,7 +653,6 @@ class Neo4j(BaseRunner): self.dump_rss(workload) ret, usage = self._cleanup() self.dump_db(path=self._neo4j_dump.parent) - assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret) return usage def start_db(self, workload): @@ -689,7 +684,6 @@ class Neo4j(BaseRunner): self.get_memory_usage("stop_" + workload) self.dump_rss(workload) ret, usage = self._cleanup() - assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret) return usage def dump_db(self, path): diff --git a/tests/mgbench/workloads/base.py b/tests/mgbench/workloads/base.py index 12ebe4002..5264dcba9 100644 --- a/tests/mgbench/workloads/base.py +++ b/tests/mgbench/workloads/base.py @@ -9,8 +9,7 @@ # by the Apache License, Version 2.0, included in the file # licenses/APL.txt. -from abc import ABC, abstractclassmethod -from pathlib import Path +from abc import ABC import helpers from benchmark_context import BenchmarkContext @@ -18,6 +17,10 @@ from benchmark_context import BenchmarkContext # Base dataset class used as a template to create each individual dataset. All # common logic is handled here. +# This dataset is used also for disk storage. In that case, we have two dataset files, one for nodes +# and the second one for edges. Managing index file is handled in the same way. If the workload is used +# for disk storage, when calling the get_file() method, the exception will be raised. The user has to use +# get_node_file() or get_edge_file() method to get the correct file path. class Workload(ABC): # Name of the workload/dataset. NAME = "" @@ -28,9 +31,13 @@ class Workload(ABC): # List of local files that should be used to import the dataset. LOCAL_FILE = None + LOCAL_FILE_NODES = None + LOCAL_FILE_EDGES = None # URLs of remote dataset files that should be used to import the dataset, compressed in gz format. URL_FILE = None + URL_FILE_NODES = None + URL_FILE_EDGES = None # Index files LOCAL_INDEX_FILE = None @@ -49,9 +56,11 @@ class Workload(ABC): generator_prerequisite = "dataset_generator" in cls.__dict__ generator_indexes_prerequisite = "indexes_generator" in cls.__dict__ custom_import_prerequisite = "custom_import" in cls.__dict__ - basic_import_prerequisite = ("LOCAL_FILE" in cls.__dict__ or "URL_FILE" in cls.__dict__) and ( - "LOCAL_INDEX_FILE" in cls.__dict__ or "URL_INDEX_FILE" in cls.__dict__ - ) + basic_import_prerequisite = ( + "LOCAL_FILE" in cls.__dict__ + or "URL_FILE" in cls.__dict__ + or ("URL_FILE_NODES" in cls.__dict__ and "URL_FILE_EDGES" in cls.__dict__) + ) and ("LOCAL_INDEX_FILE" in cls.__dict__ or "URL_INDEX_FILE" in cls.__dict__) if not name_prerequisite: raise ValueError( @@ -87,7 +96,7 @@ class Workload(ABC): return super().__init_subclass__() - def __init__(self, variant: str = None, benchmark_context: BenchmarkContext = None): + def __init__(self, variant: str = None, benchmark_context: BenchmarkContext = None, disk_workload: bool = False): """ Accepts a `variant` variable that indicates which variant of the dataset should be executed @@ -96,63 +105,110 @@ class Workload(ABC): self._variant = variant self._vendor = benchmark_context.vendor_name self._file = None + self._node_file = None + self._edge_file = None self._file_index = None + self.disk_workload: bool = disk_workload + if self.NAME == "": raise ValueError("Give your workload a name, by setting self.NAME") - if variant is None: - variant = self.DEFAULT_VARIANT - if variant not in self.VARIANTS: - raise ValueError("Invalid test variant!") - if (self.LOCAL_FILE and variant not in self.LOCAL_FILE) and (self.URL_FILE and variant not in self.URL_FILE): - raise ValueError("The variant doesn't have a defined URL or LOCAL file path!") - if variant not in self.SIZES: - raise ValueError("The variant doesn't have a defined dataset " "size!") + self._validate_variant_argument() + self._validate_vendor_argument() - if (self.LOCAL_INDEX_FILE and self._vendor not in self.LOCAL_INDEX_FILE) and ( - self.URL_INDEX_FILE and self._vendor not in self.URL_INDEX_FILE - ): - raise ValueError("Vendor does not have INDEX for dataset!") + self._set_local_files() + self._set_url_files() - if self.LOCAL_FILE is not None: - self._local_file = self.LOCAL_FILE.get(variant, None) - else: - self._local_file = None - - if self.URL_FILE is not None: - self._url_file = self.URL_FILE.get(variant, None) - else: - self._url_file = None - - if self.LOCAL_INDEX_FILE is not None: - self._local_index = self.LOCAL_INDEX_FILE.get(self._vendor, None) - else: - self._local_index = None - - if self.URL_INDEX_FILE is not None: - self._url_index = self.URL_INDEX_FILE.get(self._vendor, None) - else: - self._url_index = None + self._set_local_index_file() + self._set_url_index_file() self._size = self.SIZES[variant] if "vertices" in self._size or "edges" in self._size: self._num_vertices = self._size["vertices"] self._num_edges = self._size["edges"] + def _validate_variant_argument(self) -> None: + if self._variant is None: + variant = self.DEFAULT_VARIANT + + if self._variant not in self.VARIANTS: + raise ValueError("Invalid test variant!") + + if self._variant not in self.SIZES: + raise ValueError("The variant doesn't have a defined dataset " "size!") + + if not self.disk_workload: + if (self.LOCAL_FILE and self._variant not in self.LOCAL_FILE) and ( + self.URL_FILE and self._variant not in self.URL_FILE + ): + raise ValueError("The variant doesn't have a defined URL or LOCAL file path!") + else: + if (self.LOCAL_FILE_NODES and self._variant not in self.LOCAL_FILE_NODES) and ( + self.URL_FILE_NODES and self._variant not in self.URL_FILE_NODES + ): + raise ValueError("The variant doesn't have a defined URL or LOCAL file path for nodes!") + if (self.LOCAL_FILE_EDGES and self._variant not in self.LOCAL_FILE_EDGES) and ( + self.URL_FILE_EDGES and self._variant not in self.URL_FILE_EDGES + ): + raise ValueError("The variant doesn't have a defined URL or LOCAL file path for edges!") + + def _validate_vendor_argument(self) -> None: + if (self.LOCAL_INDEX_FILE and self._vendor not in self.LOCAL_INDEX_FILE) and ( + self.URL_INDEX_FILE and self._vendor not in self.URL_INDEX_FILE + ): + raise ValueError("Vendor does not have INDEX for dataset!") + + def _set_local_files(self) -> None: + if not self.disk_workload: + if self.LOCAL_FILE is not None: + self._local_file = self.LOCAL_FILE.get(self._variant, None) + else: + self._local_file = None + else: + if self.LOCAL_FILE_NODES is not None: + self._local_file_nodes = self.LOCAL_FILE_NODES.get(self._variant, None) + else: + self._local_file_nodes = None + + if self.LOCAL_FILE_EDGES is not None: + self._local_file_edges = self.LOCAL_FILE_EDGES.get(self._variant, None) + else: + self._local_file_edges = None + + def _set_url_files(self) -> None: + if not self.disk_workload: + if self.URL_FILE is not None: + self._url_file = self.URL_FILE.get(self._variant, None) + else: + self._url_file = None + else: + if self.URL_FILE_NODES is not None: + self._url_file_nodes = self.URL_FILE_NODES.get(self._variant, None) + else: + self._url_file_nodes = None + if self.URL_FILE_EDGES is not None: + self._url_file_edges = self.URL_FILE_EDGES.get(self._variant, None) + else: + self._url_file_edges = None + + def _set_local_index_file(self) -> None: + if self.LOCAL_INDEX_FILE is not None: + self._local_index = self.LOCAL_INDEX_FILE.get(self._vendor, None) + else: + self._local_index = None + + def _set_url_index_file(self) -> None: + if self.URL_INDEX_FILE is not None: + self._url_index = self.URL_INDEX_FILE.get(self._vendor, None) + else: + self._url_index = None + def prepare(self, directory): - if self._local_file is not None: - print("Using local dataset file:", self._local_file) - self._file = self._local_file - elif self._url_file is not None: - cached_input, exists = directory.get_file("dataset.cypher") - if not exists: - print("Downloading dataset file:", self._url_file) - downloaded_file = helpers.download_file(self._url_file, directory.get_path()) - print("Unpacking and caching file:", downloaded_file) - helpers.unpack_gz_and_move_file(downloaded_file, cached_input) - print("Using cached dataset file:", cached_input) - self._file = cached_input + if not self.disk_workload: + self._prepare_dataset_for_in_memory_workload(directory) + else: + self._prepare_dataset_for_on_disk_workload(directory) if self._local_index is not None: print("Using local index file:", self._local_index) @@ -167,6 +223,55 @@ class Workload(ABC): print("Using cached index file:", cached_index) self._file_index = cached_index + def _prepare_dataset_for_on_disk_workload(self, directory): + self._prepare_nodes_for_on_disk_workload(directory) + self._prepare_edges_for_on_disk_workload(directory) + + def _prepare_nodes_for_on_disk_workload(self, directory): + if self._local_file_nodes is not None: + print("Using local dataset file for nodes:", self._local_file_nodes) + self._node_file = self._local_file_nodes + elif self._url_file_nodes is not None: + cached_input, exists = directory.get_file("dataset_nodes.cypher") + if not exists: + print("Downloading dataset file for nodes:", self._url_file_nodes) + downloaded_file = helpers.download_file(self._url_file_nodes, directory.get_path()) + print("Unpacking and caching file for nodes:", downloaded_file) + helpers.unpack_gz_and_move_file(downloaded_file, cached_input) + print("Using cached dataset file for nodes:", cached_input) + self._node_file = cached_input + + def _prepare_edges_for_on_disk_workload(self, directory): + if self._local_file_edges is not None: + print("Using local dataset file for edges:", self._local_file_edges) + self._edge_file = self._local_file_edges + elif self._url_file_edges is not None: + cached_input, exists = directory.get_file("dataset_edges.cypher") + if not exists: + print("Downloading dataset file for edges:", self._url_file_edges) + downloaded_file = helpers.download_file(self._url_file_edges, directory.get_path()) + print("Unpacking and caching file for edges:", downloaded_file) + helpers.unpack_gz_and_move_file(downloaded_file, cached_input) + print("Using cached dataset file for edges:", cached_input) + self._edge_file = cached_input + + def _prepare_dataset_for_in_memory_workload(self, directory): + if self._local_file is not None: + print("Using local dataset file:", self._local_file) + self._file = self._local_file + elif self._url_file is not None: + cached_input, exists = directory.get_file("dataset.cypher") + if not exists: + print("Downloading dataset file:", self._url_file) + downloaded_file = helpers.download_file(self._url_file, directory.get_path()) + print("Unpacking and caching file:", downloaded_file) + helpers.unpack_gz_and_move_file(downloaded_file, cached_input) + print("Using cached dataset file:", cached_input) + self._file = cached_input + + def is_disk_workload(self): + return self.disk_workload + def get_variant(self): """Returns the current variant of the dataset.""" return self._variant @@ -179,8 +284,25 @@ class Workload(ABC): """ Returns path to the file that contains dataset creation queries. """ + if self.disk_workload: + raise Exception("get_file method should not be called for disk storage") + return self._file + def get_node_file(self): + """Returns path to the file that contains dataset creation queries for nodes.""" + if not self.disk_workload: + raise Exception("get_node_file method should be called only for disk storage") + + return self._node_file + + def get_edge_file(self): + """Returns path to the file that contains dataset creation queries for edges.""" + if not self.disk_workload: + raise Exception("get_edge_file method should be called only for disk storage") + + return self._edge_file + def get_size(self): """Returns number of vertices/edges for the current variant.""" return self._size diff --git a/tests/mgbench/workloads/disk_pokec.py b/tests/mgbench/workloads/disk_pokec.py new file mode 100644 index 000000000..f19110a0c --- /dev/null +++ b/tests/mgbench/workloads/disk_pokec.py @@ -0,0 +1,449 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import random + +from benchmark_context import BenchmarkContext +from workloads.base import Workload +from workloads.importers.disk_importer_pokec import ImporterPokec + + +class Pokec(Workload): + NAME = "pokec_disk" + VARIANTS = ["small", "medium", "large"] + DEFAULT_VARIANT = "small" + FILE = None + + URL_FILE_NODES = { + "small": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_small_import_nodes.cypher", + "medium": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_medium_import_nodes.cypher", + "large": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_large_nodes.setup.cypher.gz", + } + + URL_FILE_EDGES = { + "small": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_small_import_edges.cypher", + "medium": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_medium_import_edges.cypher", + "large": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/pokec_large_edges.setup.cypher.gz", + } + + SIZES = { + "small": {"vertices": 10000, "edges": 121716}, + "medium": {"vertices": 100000, "edges": 1768515}, + "large": {"vertices": 1632803, "edges": 30622564}, + } + + URL_INDEX_FILE = { + "memgraph": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/memgraph.cypher", + "neo4j": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec_disk/benchmark/neo4j.cypher", + } + + PROPERTIES_ON_EDGES = False + + def __init__(self, variant: str = None, benchmark_context: BenchmarkContext = None): + super().__init__(variant, benchmark_context=benchmark_context, disk_workload=True) + + def custom_import(self) -> bool: + importer = ImporterPokec( + benchmark_context=self.benchmark_context, + dataset_name=self.NAME, + index_file=self._file_index, + dataset_nodes_file=self._node_file, + dataset_edges_file=self._edge_file, + variant=self._variant, + ) + return importer.execute_import() + + # Helpers used to generate the queries + def _get_random_vertex(self): + # All vertices in the Pokec dataset have an ID in the range + # [1, _num_vertices]. + return random.randint(1, self._num_vertices) + + def _get_random_from_to(self): + vertex_from = self._get_random_vertex() + vertex_to = vertex_from + while vertex_to == vertex_from: + vertex_to = self._get_random_vertex() + return (vertex_from, vertex_to) + + # Arango benchmarks + + # OK + def benchmark__arango__single_vertex_read(self): + return ("MATCH (n:User {id : $id}) RETURN n", {"id": self._get_random_vertex()}) + + # OK + def benchmark__arango__single_vertex_write(self): + return ( + "CREATE (n:UserTemp {id : $id}) RETURN n", + {"id": random.randint(1, self._num_vertices * 10)}, + ) + + # OK + def benchmark__arango__single_edge_write(self): + vertex_from, vertex_to = self._get_random_from_to() + return ( + "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " "CREATE (n)-[e:Temp]->(m) RETURN e", + {"from": vertex_from, "to": vertex_to}, + ) + + # OK + def benchmark__arango__aggregate(self): + return ("MATCH (n:User) RETURN n.age, COUNT(*)", {}) + + # OK + def benchmark__arango__aggregate_with_distinct(self): + return ("MATCH (n:User) RETURN COUNT(DISTINCT n.age)", {}) + + # OK + def benchmark__arango__aggregate_with_filter(self): + return ("MATCH (n:User) WHERE n.age >= 18 RETURN n.age, COUNT(*)", {}) + + # NOT OK + # def benchmark__arango__expansion_1(self): + # return ( + # "MATCH (s:User {id: $id})-->(n:User) " "RETURN n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_1_with_filter(self): + # return ( + # "MATCH (s:User {id: $id})-->(n:User) " "WHERE n.age >= 18 " "RETURN n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_2(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_2_with_filter(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_3(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_3_with_filter(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_4(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__expansion_4_with_filter(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__neighbours_2(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__neighbours_2_with_filter(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__neighbours_2_with_data(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id, n", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__neighbours_2_with_data_and_filter(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id, n", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__arango__shortest_path(self): + # vertex_from, vertex_to = self._get_random_from_to() + # return ( + # "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " + # "MATCH p=(n)-[*bfs..15]->(m) " + # "RETURN extract(n in nodes(p) | n.id) AS path", + # {"from": vertex_from, "to": vertex_to}, + # ) + + # NOT OK + # def benchmark__arango__shortest_path_with_filter(self): + # vertex_from, vertex_to = self._get_random_from_to() + # return ( + # "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " + # "MATCH p=(n)-[*bfs..15 (e, n | n.age >= 18)]->(m) " + # "RETURN extract(n in nodes(p) | n.id) AS path", + # {"from": vertex_from, "to": vertex_to}, + # ) + + # OK + def benchmark__arango__allshortest_paths(self): + vertex_from, vertex_to = self._get_random_from_to() + return ( + "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " + "MATCH p=(n)-[*allshortest 2 (r, n | 1) total_weight]->(m) " + "RETURN extract(n in nodes(p) | n.id) AS path", + {"from": vertex_from, "to": vertex_to}, + ) + + # Our benchmark queries + + # OK + def benchmark__create__edge(self): + vertex_from, vertex_to = self._get_random_from_to() + return ( + "MATCH (a:User {id: $from}), (b:User {id: $to}) " "CREATE (a)-[:TempEdge]->(b)", + {"from": vertex_from, "to": vertex_to}, + ) + + # OK + def benchmark__create__pattern(self): + return ("CREATE ()-[:TempEdge]->()", {}) + + # OK + def benchmark__create__vertex(self): + return ("CREATE ()", {}) + + # OK + def benchmark__create__vertex_big(self): + return ( + "CREATE (:L1:L2:L3:L4:L5:L6:L7 {p1: true, p2: 42, " + 'p3: "Here is some text that is not extremely short", ' + 'p4:"Short text", p5: 234.434, p6: 11.11, p7: false})', + {}, + ) + + # OK + def benchmark__aggregation__count(self): + return ("MATCH (n) RETURN count(n), count(n.age)", {}) + + # OK + def benchmark__aggregation__min_max_avg(self): + return ("MATCH (n) RETURN min(n.age), max(n.age), avg(n.age)", {}) + + # NOT OK + # def benchmark__match__pattern_cycle(self): + # return ( + # "MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) " "RETURN e1, m, e2", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__match__pattern_long(self): + # return ( + # "MATCH (n1:User {id: $id})-[e1]->(n2)-[e2]->" "(n3)-[e3]->(n4)<-[e4]-(n5) " "RETURN n5 LIMIT 1", + # {"id": self._get_random_vertex()}, + # ) + + # OK + def benchmark__match__pattern_short(self): + return ( + "MATCH (n:User {id: $id})-[e]->(m) " "RETURN m LIMIT 1", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__match__vertex_on_label_property(self): + return ( + "MATCH (n:User) WITH n WHERE n.id = $id RETURN n", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__match__vertex_on_label_property_index(self): + return ("MATCH (n:User {id: $id}) RETURN n", {"id": self._get_random_vertex()}) + + # OK + def benchmark__match__vertex_on_property(self): + return ("MATCH (n:User {id: $id}) RETURN n", {"id": self._get_random_vertex()}) + + # OK + def benchmark__update__vertex_on_property(self): + return ( + "MATCH (n {id: $id}) SET n.property = -1", + {"id": self._get_random_vertex()}, + ) + + # Basic benchmark queries + + # OK + def benchmark__basic__aggregate_aggregate(self): + return ("MATCH (n:User) RETURN n.age, COUNT(*)", {}) + + # OK + def benchmark__basic__aggregate_count_aggregate(self): + return ("MATCH (n) RETURN count(n), count(n.age)", {}) + + # OK + def benchmark__basic__aggregate_with_filter_aggregate(self): + return ("MATCH (n:User) WHERE n.age >= 18 RETURN n.age, COUNT(*)", {}) + + # OK + def benchmark__basic__min_max_avg_aggregate(self): + return ("MATCH (n) RETURN min(n.age), max(n.age), avg(n.age)", {}) + + # NOT OK + # def benchmark__basic__expansion_1_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->(n:User) " "RETURN n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_1_with_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->(n:User) " "WHERE n.age >= 18 " "RETURN n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_2_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_2_with_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_3_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_3_with_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_4_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__expansion_4_with_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__neighbours_2_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__neighbours_2_with_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__neighbours_2_with_data_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id, n", + # {"id": self._get_random_vertex()}, + # ) + + # NOT OK + # def benchmark__basic__neighbours_2_with_data_and_filter_analytical(self): + # return ( + # "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id, n", + # {"id": self._get_random_vertex()}, + # ) + + # OK + def benchmark__basic__pattern_cycle_analytical(self): + return ( + "MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) " "RETURN e1, m, e2", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__basic__pattern_long_analytical(self): + return ( + "MATCH (n1:User {id: $id})-[e1]->(n2)-[e2]->" "(n3)-[e3]->(n4)<-[e4]-(n5) " "RETURN n5 LIMIT 1", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__basic__pattern_short_analytical(self): + return ( + "MATCH (n:User {id: $id})-[e]->(m) " "RETURN m LIMIT 1", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__basic__single_vertex_property_update_update(self): + return ( + "MATCH (n:User {id: $id}) SET n.property = -1", + {"id": self._get_random_vertex()}, + ) + + # OK + def benchmark__basic__single_vertex_read_read(self): + return ("MATCH (n:User {id : $id}) RETURN n", {"id": self._get_random_vertex()}) + + # OK + def benchmark__basic__single_vertex_write_write(self): + return ( + "CREATE (n:UserTemp {id : $id}) RETURN n", + {"id": random.randint(1, self._num_vertices * 10)}, + ) + + # OK + def benchmark__basic__single_edge_write_write(self): + vertex_from, vertex_to = self._get_random_from_to() + return ( + "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " "CREATE (n)-[e:Temp]->(m) RETURN e", + {"from": vertex_from, "to": vertex_to}, + ) diff --git a/tests/mgbench/workloads/importers/disk_importer_pokec.py b/tests/mgbench/workloads/importers/disk_importer_pokec.py new file mode 100644 index 000000000..560d7da9e --- /dev/null +++ b/tests/mgbench/workloads/importers/disk_importer_pokec.py @@ -0,0 +1,62 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +from pathlib import Path + +import log +from benchmark_context import BenchmarkContext +from constants import * +from runners import BaseRunner + + +class ImporterPokec: + def __init__( + self, + benchmark_context: BenchmarkContext, + dataset_name: str, + variant: str, + index_file: str, + dataset_nodes_file: str, + dataset_edges_file: str, + ) -> None: + self._benchmark_context = benchmark_context + self._dataset_name = dataset_name + self._variant = variant + self._index_file = index_file + self._dataset_nodes_file = dataset_nodes_file + self._dataset_edges_file = dataset_edges_file + + def execute_import(self): + if self._benchmark_context.vendor_name == "neo4j": + neo4j_dump = Path() / ".cache" / "datasets" / self._dataset_name / self._variant / "neo4j.dump" + vendor_runner = BaseRunner.create( + benchmark_context=self._benchmark_context, + ) + vendor_runner.clean_db() + if neo4j_dump.exists(): + log.log("Loading database from existing dump...") + vendor_runner.load_db_from_dump(path=neo4j_dump.parent) + else: + client = vendor_runner.fetch_client() + vendor_runner.start_db_init(VENDOR_RUNNER_IMPORT) + log.log("Executing database index setup") + client.execute(file_path=self._index_file, num_workers=1) + log.log("Started importing dataset") + client.execute( + file_path=self._dataset_nodes_file, num_workers=self._benchmark_context.num_workers_for_import + ) + client.execute( + file_path=self._dataset_edges_file, num_workers=self._benchmark_context.num_workers_for_import + ) + vendor_runner.stop_db_init(VENDOR_RUNNER_IMPORT) + return True + else: + return False diff --git a/tests/setup.sh b/tests/setup.sh index a70020778..7cab86db6 100755 --- a/tests/setup.sh +++ b/tests/setup.sh @@ -46,6 +46,7 @@ CFLAGS="-std=c99" python3 setup.py build CFLAGS="-std=c99" python3 setup.py install popd > /dev/null + deactivate "$DIR"/e2e/graphql/setup.sh @@ -58,7 +59,7 @@ if [ $# == 1 ]; then set +u OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH source $toolchain - NEW_LD_LIBRARY_PATH=$LD_LIBRARY_PATH + NEW_LD_LIBRARY_PATH=$LD_LIBRARY_PATH:"../libs/mgclient/lib" deactivate set -u diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 64f95b880..29ef0b611 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -293,6 +293,9 @@ target_link_libraries(${test_prefix}utils_temporal mg-utils) add_unit_test(utils_java_string_formatter.cpp) target_link_libraries(${test_prefix}utils_java_string_formatter mg-utils) +add_unit_test(utils_resource_lock.cpp) +target_link_libraries(${test_prefix}utils_resource_lock mg-utils) + # Test mg-storage-v2 add_unit_test(commit_log_v2.cpp) target_link_libraries(${test_prefix}commit_log_v2 gflags mg-utils mg-storage-v2) @@ -304,7 +307,7 @@ add_unit_test(storage_v2.cpp) target_link_libraries(${test_prefix}storage_v2 mg-storage-v2 storage_test_utils) add_unit_test(storage_v2_constraints.cpp) -target_link_libraries(${test_prefix}storage_v2_constraints mg-storage-v2) +target_link_libraries(${test_prefix}storage_v2_constraints mg-storage-v2 mg-dbms) add_unit_test(storage_v2_decoder_encoder.cpp) target_link_libraries(${test_prefix}storage_v2_decoder_encoder mg-storage-v2) @@ -407,3 +410,9 @@ if(MG_ENTERPRISE) add_unit_test_with_custom_main(dbms_handler.cpp) target_link_libraries(${test_prefix}dbms_handler mg-query mg-auth mg-glue mg-dbms) endif() + + +# Test distributed +add_unit_test(distributed_lamport_clock.cpp) +target_link_libraries(${test_prefix}distributed_lamport_clock mg-distributed) +target_include_directories(${test_prefix}distributed_lamport_clock PRIVATE ${CMAKE_SOURCE_DIR}/include) diff --git a/tests/unit/bolt_session.cpp b/tests/unit/bolt_session.cpp index 684a7e2e2..f0f3ae14c 100644 --- a/tests/unit/bolt_session.cpp +++ b/tests/unit/bolt_session.cpp @@ -117,7 +117,7 @@ class TestSession final : public Session { std::optional GetServerNameForInit() override { return std::nullopt; } void Configure(const std::map &) override {} - std::string GetDatabaseName() const override { return ""; } + std::string GetCurrentDB() const override { return ""; } void TestHook_ShouldAbort() { should_abort_ = true; } diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index dab14c874..acbd1fe41 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -2597,23 +2597,23 @@ TEST_P(CypherMainVisitorTest, TestProfileAuthQuery) { TEST_P(CypherMainVisitorTest, TestShowStorageInfo) { auto &ast_generator = *GetParam(); - auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW STORAGE INFO")); + auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW STORAGE INFO")); ASSERT_TRUE(query); - EXPECT_EQ(query->info_type_, InfoQuery::InfoType::STORAGE); + EXPECT_EQ(query->info_type_, SystemInfoQuery::InfoType::STORAGE); } TEST_P(CypherMainVisitorTest, TestShowIndexInfo) { auto &ast_generator = *GetParam(); - auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW INDEX INFO")); + auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW INDEX INFO")); ASSERT_TRUE(query); - EXPECT_EQ(query->info_type_, InfoQuery::InfoType::INDEX); + EXPECT_EQ(query->info_type_, DatabaseInfoQuery::InfoType::INDEX); } TEST_P(CypherMainVisitorTest, TestShowConstraintInfo) { auto &ast_generator = *GetParam(); - auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW CONSTRAINT INFO")); + auto *query = dynamic_cast(ast_generator.ParseQuery("SHOW CONSTRAINT INFO")); ASSERT_TRUE(query); - EXPECT_EQ(query->info_type_, InfoQuery::InfoType::CONSTRAINT); + EXPECT_EQ(query->info_type_, DatabaseInfoQuery::InfoType::CONSTRAINT); } TEST_P(CypherMainVisitorTest, CreateConstraintSyntaxError) { diff --git a/tests/unit/distributed_lamport_clock.cpp b/tests/unit/distributed_lamport_clock.cpp new file mode 100644 index 000000000..44f36160d --- /dev/null +++ b/tests/unit/distributed_lamport_clock.cpp @@ -0,0 +1,50 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#include "distributed/lamport_clock.hpp" + +#include + +using namespace memgraph::distributed; + +// Define a test fixture for the LamportClock class +class LamportClockTest : public testing::Test { + public: + LamportClock clock1; + LamportClock clock2; + + void SetUp() override { + // Setup code, if any + } + + void TearDown() override { + // Tear down code, if any + } +}; + +TEST_F(LamportClockTest, GetTimestampInternal) { + auto ts1_1 = clock1.get_timestamp(internal); + auto ts1_2 = clock1.get_timestamp(internal); + ASSERT_GT(ts1_2, ts1_1); // internal always increments +} + +TEST_F(LamportClockTest, GetTimestampSend) { + auto ts1_1 = clock1.get_timestamp(internal); + auto ts1_2 = clock1.get_timestamp(send); + ASSERT_GT(ts1_2, ts1_1); // send always increments +} + +TEST_F(LamportClockTest, GetTimestampReceive) { + auto ts1_1 = clock1.get_timestamp(send); + auto ts2_1 = clock2.get_timestamp(internal); + auto ts2_2 = clock2.get_timestamp(receive, ts1_1); + ASSERT_GE(ts2_1, ts1_1); // receive is at least send +1 + ASSERT_GE(ts2_2, ts2_1); // receive is at least internal +1 +} diff --git a/tests/unit/interpreter_faker.hpp b/tests/unit/interpreter_faker.hpp index a303007ec..5823c6a87 100644 --- a/tests/unit/interpreter_faker.hpp +++ b/tests/unit/interpreter_faker.hpp @@ -21,8 +21,8 @@ struct InterpreterFaker { } auto Prepare(const std::string &query, const std::map ¶ms = {}) { - ResultStreamFaker stream(interpreter.db_acc_->get()->storage()); - const auto [header, _1, qid, _2] = interpreter.Prepare(query, params, nullptr); + ResultStreamFaker stream(interpreter.current_db_.db_acc_->get()->storage()); + const auto [header, _1, qid, _2] = interpreter.Prepare(query, params, {}); stream.Header(header); return std::make_pair(std::move(stream), qid); } diff --git a/tests/unit/query_cost_estimator.cpp b/tests/unit/query_cost_estimator.cpp index f089e6cba..a2c7b4a48 100644 --- a/tests/unit/query_cost_estimator.cpp +++ b/tests/unit/query_cost_estimator.cpp @@ -50,8 +50,16 @@ class QueryCostEstimator : public ::testing::Test { int symbol_count = 0; void SetUp() { - ASSERT_FALSE(db->CreateIndex(label).HasError()); - ASSERT_FALSE(db->CreateIndex(label, property).HasError()); + { + auto unique_acc = db->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = db->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label, property).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } storage_dba.emplace(db->Access()); dba.emplace(storage_dba->get()); } diff --git a/tests/unit/query_dump.cpp b/tests/unit/query_dump.cpp index de7530274..b35a3cf3a 100644 --- a/tests/unit/query_dump.cpp +++ b/tests/unit/query_dump.cpp @@ -213,7 +213,7 @@ auto Execute(memgraph::query::InterpreterContext *context, memgraph::dbms::Datab memgraph::query::Interpreter interpreter(context, db); ResultStreamFaker stream(db->storage()); - auto [header, _1, qid, _2] = interpreter.Prepare(query, {}, nullptr); + auto [header, _1, qid, _2] = interpreter.Prepare(query, {}, {}); stream.Header(header); auto summary = interpreter.PullAll(&stream); stream.Summary(summary); @@ -573,14 +573,22 @@ TYPED_TEST(DumpTest, IndicesKeys) { CreateVertex(dba.get(), {"Label1", "Label 2"}, {{"p", memgraph::storage::PropertyValue(1)}}, false); ASSERT_FALSE(dba->Commit().HasError()); } - ASSERT_FALSE( - this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("Label1"), this->db->storage()->NameToProperty("prop")) - .HasError()); - ASSERT_FALSE( - this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("Label 2"), this->db->storage()->NameToProperty("prop `")) - .HasError()); + + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE( + unique_acc->CreateIndex(this->db->storage()->NameToLabel("Label1"), this->db->storage()->NameToProperty("prop")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE( + unique_acc + ->CreateIndex(this->db->storage()->NameToLabel("Label 2"), this->db->storage()->NameToProperty("prop `")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } { ResultStreamFaker stream(this->db->storage()); @@ -604,9 +612,11 @@ TYPED_TEST(DumpTest, ExistenceConstraints) { ASSERT_FALSE(dba->Commit().HasError()); } { - auto res = this->db->storage()->CreateExistenceConstraint(this->db->storage()->NameToLabel("L`abel 1"), - this->db->storage()->NameToProperty("prop"), {}); + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->db->storage()->NameToLabel("L`abel 1"), + this->db->storage()->NameToProperty("prop")); ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { @@ -635,11 +645,13 @@ TYPED_TEST(DumpTest, UniqueConstraints) { ASSERT_FALSE(dba->Commit().HasError()); } { - auto res = this->db->storage()->CreateUniqueConstraint( + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint( this->db->storage()->NameToLabel("Label"), - {this->db->storage()->NameToProperty("prop"), this->db->storage()->NameToProperty("prop2")}, {}); + {this->db->storage()->NameToProperty("prop"), this->db->storage()->NameToProperty("prop2")}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { @@ -757,23 +769,36 @@ TYPED_TEST(DumpTest, CheckStateSimpleGraph) { ASSERT_FALSE(dba->Commit().HasError()); } { - auto ret = this->db->storage()->CreateExistenceConstraint(this->db->storage()->NameToLabel("Person"), - this->db->storage()->NameToProperty("name"), {}); + auto unique_acc = this->db->UniqueAccess(); + auto ret = unique_acc->CreateExistenceConstraint(this->db->storage()->NameToLabel("Person"), + this->db->storage()->NameToProperty("name")); ASSERT_FALSE(ret.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { - auto ret = this->db->storage()->CreateUniqueConstraint(this->db->storage()->NameToLabel("Person"), - {this->db->storage()->NameToProperty("name")}, {}); + auto unique_acc = this->db->UniqueAccess(); + auto ret = unique_acc->CreateUniqueConstraint(this->db->storage()->NameToLabel("Person"), + {this->db->storage()->NameToProperty("name")}); ASSERT_TRUE(ret.HasValue()); ASSERT_EQ(ret.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE( + unique_acc->CreateIndex(this->db->storage()->NameToLabel("Person"), this->db->storage()->NameToProperty("id")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE(unique_acc + ->CreateIndex(this->db->storage()->NameToLabel("Person"), + this->db->storage()->NameToProperty("unexisting_property")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } - ASSERT_FALSE(this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("Person"), this->db->storage()->NameToProperty("id")) - .HasError()); - ASSERT_FALSE(this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("Person"), - this->db->storage()->NameToProperty("unexisting_property")) - .HasError()); const auto &db_initial_state = GetState(this->db->storage()); memgraph::storage::Config config{}; @@ -850,9 +875,9 @@ class StatefulInterpreter { : context_(context), interpreter_(context_, db) {} auto Execute(const std::string &query) { - ResultStreamFaker stream(interpreter_.db_acc_->get()->storage()); + ResultStreamFaker stream(interpreter_.current_db_.db_acc_->get()->storage()); - auto [header, _1, qid, _2] = interpreter_.Prepare(query, {}, nullptr); + auto [header, _1, qid, _2] = interpreter_.Prepare(query, {}, {}); stream.Header(header); auto summary = interpreter_.PullAll(&stream); stream.Summary(summary); @@ -931,39 +956,55 @@ TYPED_TEST(DumpTest, ExecuteDumpDatabaseInMulticommandTransaction) { TYPED_TEST(DumpTest, MultiplePartialPulls) { { // Create indices - ASSERT_FALSE( - this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("PERSON"), this->db->storage()->NameToProperty("name")) - .HasError()); - ASSERT_FALSE( - this->db->storage() - ->CreateIndex(this->db->storage()->NameToLabel("PERSON"), this->db->storage()->NameToProperty("surname")) - .HasError()); + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE( + unique_acc + ->CreateIndex(this->db->storage()->NameToLabel("PERSON"), this->db->storage()->NameToProperty("name")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE( + unique_acc + ->CreateIndex(this->db->storage()->NameToLabel("PERSON"), this->db->storage()->NameToProperty("surname")) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } // Create existence constraints { - auto res = this->db->storage()->CreateExistenceConstraint(this->db->storage()->NameToLabel("PERSON"), - this->db->storage()->NameToProperty("name"), {}); + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->db->storage()->NameToLabel("PERSON"), + this->db->storage()->NameToProperty("name")); ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { - auto res = this->db->storage()->CreateExistenceConstraint(this->db->storage()->NameToLabel("PERSON"), - this->db->storage()->NameToProperty("surname"), {}); + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->db->storage()->NameToLabel("PERSON"), + this->db->storage()->NameToProperty("surname")); ASSERT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } // Create unique constraints { - auto res = this->db->storage()->CreateUniqueConstraint(this->db->storage()->NameToLabel("PERSON"), - {this->db->storage()->NameToProperty("name")}, {}); + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->db->storage()->NameToLabel("PERSON"), + {this->db->storage()->NameToProperty("name")}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { - auto res = this->db->storage()->CreateUniqueConstraint(this->db->storage()->NameToLabel("PERSON"), - {this->db->storage()->NameToProperty("surname")}, {}); + auto unique_acc = this->db->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->db->storage()->NameToLabel("PERSON"), + {this->db->storage()->NameToProperty("surname")}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), memgraph::storage::UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } auto dba = this->db->storage()->Access(); diff --git a/tests/unit/query_plan_edge_cases.cpp b/tests/unit/query_plan_edge_cases.cpp index a81b517b5..f37848d91 100644 --- a/tests/unit/query_plan_edge_cases.cpp +++ b/tests/unit/query_plan_edge_cases.cpp @@ -86,7 +86,7 @@ class QueryExecution : public testing::Test { auto Execute(const std::string &query) { ResultStreamFaker stream(this->db_acc_->get()->storage()); - auto [header, _1, qid, _2] = interpreter_->Prepare(query, {}, nullptr); + auto [header, _1, qid, _2] = interpreter_->Prepare(query, {}, {}); stream.Header(header); auto summary = interpreter_->PullAll(&stream); stream.Summary(summary); diff --git a/tests/unit/query_plan_match_filter_return.cpp b/tests/unit/query_plan_match_filter_return.cpp index ff4e4eeb9..6b856704d 100644 --- a/tests/unit/query_plan_match_filter_return.cpp +++ b/tests/unit/query_plan_match_filter_return.cpp @@ -3081,7 +3081,11 @@ TYPED_TEST(QueryPlan, Distinct) { TYPED_TEST(QueryPlan, ScanAllByLabel) { auto label = this->db->NameToLabel("label"); - [[maybe_unused]] auto _ = this->db->CreateIndex(label); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); // Add a vertex with a label and one without. @@ -3138,7 +3142,12 @@ TYPED_TEST(QueryPlan, ScanAllByLabelProperty) { } ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3244,7 +3253,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyEqualityNoError) { ASSERT_TRUE(string_vertex.SetProperty(prop, memgraph::storage::PropertyValue("string")).HasValue()); ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3279,7 +3292,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyValueError) { } ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3308,7 +3325,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyRangeError) { } ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3360,7 +3381,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyEqualNull) { ASSERT_TRUE(vertex_with_prop.SetProperty(prop, memgraph::storage::PropertyValue(42)).HasValue()); ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3393,7 +3418,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyRangeNull) { ASSERT_TRUE(vertex_with_prop.SetProperty(prop, memgraph::storage::PropertyValue(42)).HasValue()); ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3422,7 +3451,11 @@ TYPED_TEST(QueryPlan, ScanAllByLabelPropertyNoValueInIndexContinuation) { ASSERT_TRUE(v.SetProperty(prop, memgraph::storage::PropertyValue(2)).HasValue()); ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } auto storage_dba = this->db->Access(); memgraph::query::DbAccessor dba(storage_dba.get()); @@ -3463,7 +3496,11 @@ TYPED_TEST(QueryPlan, ScanAllEqualsScanAllByLabelProperty) { ASSERT_FALSE(dba.Commit().HasError()); } - [[maybe_unused]] auto _ = this->db->CreateIndex(label, prop); + { + auto unique_acc = this->db->UniqueAccess(); + [[maybe_unused]] auto _ = unique_acc->CreateIndex(label, prop); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } // Make sure there are `vertex_count` vertices { diff --git a/tests/unit/query_plan_v2_create_set_remove_delete.cpp b/tests/unit/query_plan_v2_create_set_remove_delete.cpp index aa87b67c8..8e7ee092b 100644 --- a/tests/unit/query_plan_v2_create_set_remove_delete.cpp +++ b/tests/unit/query_plan_v2_create_set_remove_delete.cpp @@ -122,7 +122,11 @@ TYPED_TEST(QueryPlan, ScanAll) { TYPED_TEST(QueryPlan, ScanAllByLabel) { auto label = this->db->NameToLabel("label"); - ASSERT_FALSE(this->db->CreateIndex(label).HasError()); + { + auto unique_acc = this->db->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } { auto dba = this->db->Access(); // Add some unlabeled vertices diff --git a/tests/unit/query_required_privileges.cpp b/tests/unit/query_required_privileges.cpp index a648e396b..e43525f5b 100644 --- a/tests/unit/query_required_privileges.cpp +++ b/tests/unit/query_required_privileges.cpp @@ -111,21 +111,21 @@ TEST_F(TestPrivilegeExtractor, AuthQuery) { #endif TEST_F(TestPrivilegeExtractor, ShowIndexInfo) { - auto *query = storage.Create(); - query->info_type_ = InfoQuery::InfoType::INDEX; - EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::INDEX)); + auto *db_query = storage.Create(); + db_query->info_type_ = DatabaseInfoQuery::InfoType::INDEX; + EXPECT_THAT(GetRequiredPrivileges(db_query), UnorderedElementsAre(AuthQuery::Privilege::INDEX)); } TEST_F(TestPrivilegeExtractor, ShowStatsInfo) { - auto *query = storage.Create(); - query->info_type_ = InfoQuery::InfoType::STORAGE; - EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::STATS)); + auto *db_query = storage.Create(); + db_query->info_type_ = SystemInfoQuery::InfoType::STORAGE; + EXPECT_THAT(GetRequiredPrivileges(db_query), UnorderedElementsAre(AuthQuery::Privilege::STATS)); } TEST_F(TestPrivilegeExtractor, ShowConstraintInfo) { - auto *query = storage.Create(); - query->info_type_ = InfoQuery::InfoType::CONSTRAINT; - EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::CONSTRAINT)); + auto *db_query = storage.Create(); + db_query->info_type_ = DatabaseInfoQuery::InfoType::CONSTRAINT; + EXPECT_THAT(GetRequiredPrivileges(db_query), UnorderedElementsAre(AuthQuery::Privilege::CONSTRAINT)); } TEST_F(TestPrivilegeExtractor, CreateConstraint) { diff --git a/tests/unit/replication_persistence_helper.cpp b/tests/unit/replication_persistence_helper.cpp index 7915d315c..b2fcae53e 100644 --- a/tests/unit/replication_persistence_helper.cpp +++ b/tests/unit/replication_persistence_helper.cpp @@ -9,8 +9,9 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. -#include "storage/v2/replication/replication_persistence_helper.hpp" #include "formatters.hpp" +#include "replication/state.hpp" +#include "replication/status.hpp" #include "utils/logging.hpp" #include @@ -20,6 +21,7 @@ #include using namespace memgraph::storage::replication; +using namespace memgraph::replication; class ReplicationPersistanceHelperTest : public testing::Test { protected: diff --git a/tests/unit/storage_v2_constraints.cpp b/tests/unit/storage_v2_constraints.cpp index 8094c4ab4..990ecfbdd 100644 --- a/tests/unit/storage_v2_constraints.cpp +++ b/tests/unit/storage_v2_constraints.cpp @@ -12,8 +12,10 @@ #include #include #include +#include #include +#include "dbms/database.hpp" #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/disk/storage.hpp" #include "storage/v2/disk/unique_constraints.hpp" @@ -38,7 +40,12 @@ class ConstraintsTest : public testing::Test { ConstraintsTest() { /// TODO: andi How to make this better? Because currentlly for every test changed you need to create a configuration config_ = disk_test_utils::GenerateOnDiskConfig(testSuite); - storage = std::make_unique(config_); + config_.force_on_disk = std::is_same_v; + db_gk_.emplace(config_); + auto db_acc_opt = db_gk_->access(); + MG_ASSERT(db_acc_opt, "Failed to access db"); + db_acc_ = *db_acc_opt; + storage = db_acc_->get()->storage(); prop1 = storage->NameToProperty("prop1"); prop2 = storage->NameToProperty("prop2"); label1 = storage->NameToLabel("label1"); @@ -46,15 +53,19 @@ class ConstraintsTest : public testing::Test { } void TearDown() override { - storage.reset(nullptr); + storage = nullptr; + db_acc_.reset(); + db_gk_.reset(); if (std::is_same::value) { disk_test_utils::RemoveRocksDbDirs(testSuite); } } - std::unique_ptr storage; + Storage *storage; memgraph::storage::Config config_; + std::optional db_acc_; + std::optional> db_gk_; PropertyId prop1; PropertyId prop2; LabelId label1; @@ -66,39 +77,86 @@ TYPED_TEST_CASE(ConstraintsTest, StorageTypes); // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, ExistenceConstraintsCreateAndDrop) { - EXPECT_EQ(this->storage->ListAllConstraints().existence.size(), 0); { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); - EXPECT_FALSE(res.HasError()); + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllConstraints().existence.size(), 0); + ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().existence, - UnorderedElementsAre(std::make_pair(this->label1, this->prop1))); { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); + EXPECT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().existence, UnorderedElementsAre(std::make_pair(this->label1, this->prop1))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); EXPECT_TRUE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } - EXPECT_THAT(this->storage->ListAllConstraints().existence, - UnorderedElementsAre(std::make_pair(this->label1, this->prop1))); { - auto res = this->storage->CreateExistenceConstraint(this->label2, this->prop1, {}); - EXPECT_FALSE(res.HasError()); + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().existence, UnorderedElementsAre(std::make_pair(this->label1, this->prop1))); + ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_THAT( - this->storage->ListAllConstraints().existence, - UnorderedElementsAre(std::make_pair(this->label1, this->prop1), std::make_pair(this->label2, this->prop1))); - EXPECT_FALSE(this->storage->DropExistenceConstraint(this->label1, this->prop1, {}).HasError()); - EXPECT_TRUE(this->storage->DropExistenceConstraint(this->label1, this->prop1, {}).HasError()); - EXPECT_THAT(this->storage->ListAllConstraints().existence, - UnorderedElementsAre(std::make_pair(this->label2, this->prop1))); - EXPECT_FALSE(this->storage->DropExistenceConstraint(this->label2, this->prop1, {}).HasError()); - EXPECT_TRUE(this->storage->DropExistenceConstraint(this->label2, this->prop2, {}).HasError()); - EXPECT_EQ(this->storage->ListAllConstraints().existence.size(), 0); { - auto res = this->storage->CreateExistenceConstraint(this->label2, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label2, this->prop1); EXPECT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().existence, UnorderedElementsAre(std::make_pair(this->label1, this->prop1), + std::make_pair(this->label2, this->prop1))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_FALSE(unique_acc->DropExistenceConstraint(this->label1, this->prop1).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_TRUE(unique_acc->DropExistenceConstraint(this->label1, this->prop1).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().existence, UnorderedElementsAre(std::make_pair(this->label2, this->prop1))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_FALSE(unique_acc->DropExistenceConstraint(this->label2, this->prop1).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_TRUE(unique_acc->DropExistenceConstraint(this->label2, this->prop2).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllConstraints().existence.size(), 0); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label2, this->prop1); + EXPECT_FALSE(res.HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().existence, UnorderedElementsAre(std::make_pair(this->label2, this->prop1))); + ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().existence, - UnorderedElementsAre(std::make_pair(this->label2, this->prop1))); } // NOLINTNEXTLINE(hicpp-special-member-functions) @@ -110,11 +168,13 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsCreateFailure1) { ASSERT_NO_ERROR(acc->Commit()); } { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); ASSERT_TRUE(res.HasError()); EXPECT_EQ( std::get(res.GetError()), (ConstraintViolation{ConstraintViolation::Type::EXISTENCE, this->label1, std::set{this->prop1}})); + ASSERT_FALSE(unique_acc->Commit().HasError()); // TODO: Check if we are committing here? } { auto acc = this->storage->Access(); @@ -124,8 +184,10 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsCreateFailure1) { ASSERT_NO_ERROR(acc->Commit()); } { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); EXPECT_FALSE(res.HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); } } @@ -138,11 +200,13 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsCreateFailure2) { ASSERT_NO_ERROR(acc->Commit()); } { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); ASSERT_TRUE(res.HasError()); EXPECT_EQ( std::get(res.GetError()), (ConstraintViolation{ConstraintViolation::Type::EXISTENCE, this->label1, std::set{this->prop1}})); + ASSERT_FALSE(unique_acc->Commit().HasError()); // TODO: Check if we are committing here? } { auto acc = this->storage->Access(); @@ -152,16 +216,20 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsCreateFailure2) { ASSERT_NO_ERROR(acc->Commit()); } { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); EXPECT_FALSE(res.HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); } } // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, ExistenceConstraintsViolationOnCommit) { { - auto res = this->storage->CreateExistenceConstraint(this->label1, this->prop1, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateExistenceConstraint(this->label1, this->prop1); EXPECT_FALSE(res.HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -208,9 +276,11 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsViolationOnCommit) { ASSERT_NO_ERROR(acc->Commit()); } - - ASSERT_FALSE(this->storage->DropExistenceConstraint(this->label1, this->prop1, {}).HasError()); - + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + ASSERT_FALSE(unique_acc->DropExistenceConstraint(this->label1, this->prop1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); auto vertex = acc->CreateVertex(); @@ -221,47 +291,98 @@ TYPED_TEST(ConstraintsTest, ExistenceConstraintsViolationOnCommit) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsCreateAndDropAndList) { - EXPECT_EQ(this->storage->ListAllConstraints().unique.size(), 0); { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllConstraints().unique.size(), 0); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); EXPECT_TRUE(res.HasValue()); EXPECT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}))); { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, + UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); EXPECT_TRUE(res.HasValue()); EXPECT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::ALREADY_EXISTS); + ASSERT_NO_ERROR(unique_acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}))); { - auto res = this->storage->CreateUniqueConstraint(this->label2, {this->prop1}, {}); + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, + UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label2, {this->prop1}); EXPECT_TRUE(res.HasValue() && res.GetValue() == UniqueConstraints::CreationStatus::SUCCESS); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}), - std::make_pair(this->label2, std::set{this->prop1}))); - EXPECT_EQ(this->storage->DropUniqueConstraint(this->label1, {this->prop1}, {}).GetValue(), - UniqueConstraints::DeletionStatus::SUCCESS); - EXPECT_EQ(this->storage->DropUniqueConstraint(this->label1, {this->prop1}, {}).GetValue(), - UniqueConstraints::DeletionStatus::NOT_FOUND); - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label2, std::set{this->prop1}))); - EXPECT_EQ(this->storage->DropUniqueConstraint(this->label2, {this->prop1}, {}).GetValue(), - UniqueConstraints::DeletionStatus::SUCCESS); - EXPECT_EQ(this->storage->DropUniqueConstraint(this->label2, {this->prop2}, {}).GetValue(), - UniqueConstraints::DeletionStatus::NOT_FOUND); - EXPECT_EQ(this->storage->ListAllConstraints().unique.size(), 0); { - auto res = this->storage->CreateUniqueConstraint(this->label2, {this->prop1}, {}); + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, + UnorderedElementsAre(std::make_pair(this->label1, std::set{this->prop1}), + std::make_pair(this->label2, std::set{this->prop1}))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_EQ(unique_acc->DropUniqueConstraint(this->label1, {this->prop1}), + UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_EQ(unique_acc->DropUniqueConstraint(this->label1, {this->prop1}), + UniqueConstraints::DeletionStatus::NOT_FOUND); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, + UnorderedElementsAre(std::make_pair(this->label2, std::set{this->prop1}))); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_EQ(unique_acc->DropUniqueConstraint(this->label2, {this->prop1}), + UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + EXPECT_EQ(unique_acc->DropUniqueConstraint(this->label2, {this->prop2}), + UniqueConstraints::DeletionStatus::NOT_FOUND); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllConstraints().unique.size(), 0); + ASSERT_NO_ERROR(acc->Commit()); + } + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label2, {this->prop1}); EXPECT_TRUE(res.HasValue()); EXPECT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); } - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label2, std::set{this->prop1}))); + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, + UnorderedElementsAre(std::make_pair(this->label2, std::set{this->prop1}))); + ASSERT_NO_ERROR(acc->Commit()); + } } // NOLINTNEXTLINE(hicpp-special-member-functions) @@ -277,11 +398,13 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsCreateFailure1) { } { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasError()); EXPECT_EQ( std::get(res.GetError()), (ConstraintViolation{ConstraintViolation::Type::UNIQUE, this->label1, std::set{this->prop1}})); + ASSERT_FALSE(unique_acc->Commit().HasError()); // TODO: Check if we are committing here? } { @@ -293,9 +416,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsCreateFailure1) { } { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } } @@ -312,11 +437,13 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsCreateFailure2) { } { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasError()); EXPECT_EQ( std::get(res.GetError()), (ConstraintViolation{ConstraintViolation::Type::UNIQUE, this->label1, std::set{this->prop1}})); + ASSERT_FALSE(unique_acc->Commit().HasError()); // TODO: Check if we are committing here? } { @@ -330,9 +457,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsCreateFailure2) { } { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } } @@ -353,9 +482,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation1) { } { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -383,9 +514,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation1) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation2) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -413,9 +546,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation2) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation3) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -449,9 +584,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation3) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation4) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -484,9 +621,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsNoViolation4) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit1) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -509,9 +648,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit1) { /// TODO: andi consistency problems TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit2) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -553,9 +694,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit2) { /// TODO: andi consistency problems TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit3) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -604,9 +747,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsViolationOnCommit3) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(ConstraintsTest, UniqueConstraintsLabelAlteration) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } Gid gid1; @@ -715,14 +860,18 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsPropertySetSize) { { // This should fail since unique constraint cannot be created for an empty // property set. - auto res = this->storage->CreateUniqueConstraint(this->label1, {}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::EMPTY_PROPERTIES); + ASSERT_NO_ERROR(unique_acc->Commit()); } - // Removing a constraint with empty property set should also fail. - ASSERT_EQ(this->storage->DropUniqueConstraint(this->label1, {}, {}).GetValue(), - UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES); + { // Removing a constraint with empty property set should also fail. + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + ASSERT_EQ(unique_acc->DropUniqueConstraint(this->label1, {}), UniqueConstraints::DeletionStatus::EMPTY_PROPERTIES); + ASSERT_NO_ERROR(unique_acc->Commit()); + } // Create a set of 33 properties. std::set properties; @@ -733,48 +882,68 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsPropertySetSize) { { // This should fail since list of properties exceeds the maximum number of // properties, which is 32. - auto res = this->storage->CreateUniqueConstraint(this->label1, properties, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, properties); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED); + ASSERT_NO_ERROR(unique_acc->Commit()); } - // An attempt to delete constraint with too large property set should fail. - ASSERT_EQ(this->storage->DropUniqueConstraint(this->label1, properties, {}).GetValue(), - UniqueConstraints::DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED); + { // An attempt to delete constraint with too large property set should fail. + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + ASSERT_EQ(unique_acc->DropUniqueConstraint(this->label1, properties), + UniqueConstraints::DeletionStatus::PROPERTIES_SIZE_LIMIT_EXCEEDED); + ASSERT_NO_ERROR(unique_acc->Commit()); + } // Remove one property from the set. properties.erase(properties.begin()); { // Creating a constraint for 32 properties should succeed. - auto res = this->storage->CreateUniqueConstraint(this->label1, properties, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, properties); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } - EXPECT_THAT(this->storage->ListAllConstraints().unique, - UnorderedElementsAre(std::make_pair(this->label1, properties))); + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllConstraints().unique, UnorderedElementsAre(std::make_pair(this->label1, properties))); + ASSERT_NO_ERROR(acc->Commit()); + } - // Removing a constraint with 32 properties should succeed. - ASSERT_EQ(this->storage->DropUniqueConstraint(this->label1, properties, {}).GetValue(), - UniqueConstraints::DeletionStatus::SUCCESS); - ASSERT_TRUE(this->storage->ListAllConstraints().unique.empty()); + { // Removing a constraint with 32 properties should succeed. + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + ASSERT_EQ(unique_acc->DropUniqueConstraint(this->label1, properties), UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto acc = this->storage->Access(); + ASSERT_TRUE(acc->ListAllConstraints().unique.empty()); + ASSERT_NO_ERROR(acc->Commit()); + } } // NOLINTNEXTLINE(hicpp-special-member-functions) /// TODO: andi consistency problems TYPED_TEST(ConstraintsTest, UniqueConstraintsMultipleProperties) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { // An attempt to create an existing unique constraint. - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop2, this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop2, this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::ALREADY_EXISTS); + ASSERT_NO_ERROR(unique_acc->Commit()); } Gid gid1; @@ -826,9 +995,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsMultipleProperties) { /// TODO: andi Test passes when ran alone but fails when all tests are run TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertAbortInsert) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -852,9 +1023,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertAbortInsert) { TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertRemoveInsert) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } Gid gid; @@ -887,9 +1060,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertRemoveInsert) { TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertRemoveAbortInsert) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } Gid gid; @@ -927,9 +1102,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertRemoveAbortInsert) { TYPED_TEST(ConstraintsTest, UniqueConstraintsDeleteVertexSetProperty) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } Gid gid1; @@ -969,9 +1146,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsDeleteVertexSetProperty) { TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertDropInsert) { { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -983,8 +1162,12 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsInsertDropInsert) { ASSERT_NO_ERROR(acc->Commit()); } - ASSERT_EQ(this->storage->DropUniqueConstraint(this->label1, {this->prop2, this->prop1}, {}).GetValue(), - UniqueConstraints::DeletionStatus::SUCCESS); + { + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + ASSERT_EQ(unique_acc->DropUniqueConstraint(this->label1, {this->prop2, this->prop1}), + UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); @@ -1001,9 +1184,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsComparePropertyValues) { // are correctly compared. { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1, this->prop2}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } { @@ -1044,9 +1229,11 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsClearOldData) { auto *tx_db = disk_constraints->GetRocksDBStorage()->db_; { - auto res = this->storage->CreateUniqueConstraint(this->label1, {this->prop1}, {}); + auto unique_acc = this->db_acc_->get()->UniqueAccess(); + auto res = unique_acc->CreateUniqueConstraint(this->label1, {this->prop1}); ASSERT_TRUE(res.HasValue()); ASSERT_EQ(res.GetValue(), UniqueConstraints::CreationStatus::SUCCESS); + ASSERT_NO_ERROR(unique_acc->Commit()); } auto acc = this->storage->Access(); diff --git a/tests/unit/storage_v2_decoder_encoder.cpp b/tests/unit/storage_v2_decoder_encoder.cpp index 9e2555ebd..9b627cb77 100644 --- a/tests/unit/storage_v2_decoder_encoder.cpp +++ b/tests/unit/storage_v2_decoder_encoder.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -349,8 +349,12 @@ TEST_F(DecoderEncoderTest, PropertyValueInvalidMarker) { case memgraph::storage::durability::Marker::DELTA_TRANSACTION_END: case memgraph::storage::durability::Marker::DELTA_LABEL_INDEX_CREATE: case memgraph::storage::durability::Marker::DELTA_LABEL_INDEX_DROP: + case memgraph::storage::durability::Marker::DELTA_LABEL_INDEX_STATS_SET: + case memgraph::storage::durability::Marker::DELTA_LABEL_INDEX_STATS_CLEAR: case memgraph::storage::durability::Marker::DELTA_LABEL_PROPERTY_INDEX_CREATE: case memgraph::storage::durability::Marker::DELTA_LABEL_PROPERTY_INDEX_DROP: + case memgraph::storage::durability::Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_SET: + case memgraph::storage::durability::Marker::DELTA_LABEL_PROPERTY_INDEX_STATS_CLEAR: case memgraph::storage::durability::Marker::DELTA_EXISTENCE_CONSTRAINT_CREATE: case memgraph::storage::durability::Marker::DELTA_EXISTENCE_CONSTRAINT_DROP: case memgraph::storage::durability::Marker::DELTA_UNIQUE_CONSTRAINT_CREATE: diff --git a/tests/unit/storage_v2_durability_inmemory.cpp b/tests/unit/storage_v2_durability_inmemory.cpp index e6734f766..45664ae67 100644 --- a/tests/unit/storage_v2_durability_inmemory.cpp +++ b/tests/unit/storage_v2_durability_inmemory.cpp @@ -28,6 +28,7 @@ #include "storage/v2/durability/version.hpp" #include "storage/v2/durability/wal.hpp" #include "storage/v2/edge_accessor.hpp" +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/vertex_accessor.hpp" #include "utils/file.hpp" @@ -77,17 +78,45 @@ class DurabilityTest : public ::testing::TestWithParam { auto et1 = store->NameToEdgeType("base_et1"); auto et2 = store->NameToEdgeType("base_et2"); - // Create label index. - ASSERT_FALSE(store->CreateIndex(label_unindexed).HasError()); + { + // Create label index. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label_unindexed).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + // Create label index statistics. + auto acc = store->Access(); + acc->SetIndexStats(label_unindexed, memgraph::storage::LabelIndexStats{1, 2}); + ASSERT_TRUE(acc->GetIndexStats(label_unindexed)); + ASSERT_FALSE(acc->Commit().HasError()); + } + { + // Create label+property index. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label_indexed, property_id).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + // Create label+property index statistics. + auto acc = store->Access(); + acc->SetIndexStats(label_indexed, property_id, memgraph::storage::LabelPropertyIndexStats{1, 2, 3.4, 5.6, 0.0}); + ASSERT_TRUE(acc->GetIndexStats(label_indexed, property_id)); + ASSERT_FALSE(acc->Commit().HasError()); + } - // Create label+property index. - ASSERT_FALSE(store->CreateIndex(label_indexed, property_id).HasError()); - - // Create existence constraint. - ASSERT_FALSE(store->CreateExistenceConstraint(label_unindexed, property_id, {}).HasError()); - - // Create unique constraint. - ASSERT_FALSE(store->CreateUniqueConstraint(label_unindexed, {property_id, property_extra}, {}).HasError()); + { + // Create existence constraint. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateExistenceConstraint(label_unindexed, property_id).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + // Create unique constraint. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateUniqueConstraint(label_unindexed, {property_id, property_extra}).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } // Create vertices. for (uint64_t i = 0; i < kNumBaseVertices; ++i) { @@ -142,17 +171,47 @@ class DurabilityTest : public ::testing::TestWithParam { auto et3 = store->NameToEdgeType("extended_et3"); auto et4 = store->NameToEdgeType("extended_et4"); - // Create label index. - ASSERT_FALSE(store->CreateIndex(label_unused).HasError()); + { + // Create label index. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label_unused).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + // Create label index statistics. + auto acc = store->Access(); + acc->SetIndexStats(label_unused, memgraph::storage::LabelIndexStats{123, 9.87}); + ASSERT_TRUE(acc->GetIndexStats(label_unused)); + ASSERT_FALSE(acc->Commit().HasError()); + } + { + // Create label+property index. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(label_indexed, property_count).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + // Create label+property index statistics. + auto acc = store->Access(); + acc->SetIndexStats(label_indexed, property_count, + memgraph::storage::LabelPropertyIndexStats{456798, 312345, 12312312.2, 123123.2, 67876.9}); + ASSERT_TRUE(acc->GetIndexStats(label_indexed, property_count)); + ASSERT_FALSE(acc->Commit().HasError()); + } - // Create label+property index. - ASSERT_FALSE(store->CreateIndex(label_indexed, property_count).HasError()); + { + // Create existence constraint. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateExistenceConstraint(label_unused, property_count).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } - // Create existence constraint. - ASSERT_FALSE(store->CreateExistenceConstraint(label_unused, property_count, {}).HasError()); - - // Create unique constraint. - ASSERT_FALSE(store->CreateUniqueConstraint(label_unused, {property_count}, {}).HasError()); + { + // Create unique constraint. + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateUniqueConstraint(label_unused, {property_count}).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } // Storage accessor. std::unique_ptr acc; @@ -212,9 +271,12 @@ class DurabilityTest : public ::testing::TestWithParam { auto et3 = store->NameToEdgeType("extended_et3"); auto et4 = store->NameToEdgeType("extended_et4"); + // Create storage accessor. + auto acc = store->Access(); + // Verify indices info. { - auto info = store->ListAllIndices(); + auto info = acc->ListAllIndices(); switch (type) { case DatasetType::ONLY_BASE: ASSERT_THAT(info.label, UnorderedElementsAre(base_label_unindexed)); @@ -237,9 +299,71 @@ class DurabilityTest : public ::testing::TestWithParam { } } + // Verify index statistics + { + switch (type) { + case DatasetType::ONLY_BASE: { + const auto l_stats = acc->GetIndexStats(base_label_unindexed); + ASSERT_TRUE(l_stats); + ASSERT_EQ(l_stats->count, 1); + ASSERT_EQ(l_stats->avg_degree, 2); + const auto lp_stats = acc->GetIndexStats(base_label_indexed, property_id); + ASSERT_TRUE(lp_stats); + ASSERT_EQ(lp_stats->count, 1); + ASSERT_EQ(lp_stats->distinct_values_count, 2); + ASSERT_EQ(lp_stats->statistic, 3.4); + ASSERT_EQ(lp_stats->avg_group_size, 5.6); + ASSERT_EQ(lp_stats->avg_degree, 0.0); + break; + } + case DatasetType::ONLY_EXTENDED: { + const auto l_stats = acc->GetIndexStats(extended_label_unused); + ASSERT_TRUE(l_stats); + ASSERT_EQ(l_stats->count, 123); + ASSERT_EQ(l_stats->avg_degree, 9.87); + const auto lp_stats = acc->GetIndexStats(extended_label_indexed, property_count); + ASSERT_TRUE(lp_stats); + ASSERT_EQ(lp_stats->count, 456798); + ASSERT_EQ(lp_stats->distinct_values_count, 312345); + ASSERT_EQ(lp_stats->statistic, 12312312.2); + ASSERT_EQ(lp_stats->avg_group_size, 123123.2); + ASSERT_EQ(lp_stats->avg_degree, 67876.9); + break; + } + case DatasetType::ONLY_BASE_WITH_EXTENDED_INDICES_AND_CONSTRAINTS: + case DatasetType::ONLY_EXTENDED_WITH_BASE_INDICES_AND_CONSTRAINTS: + case DatasetType::BASE_WITH_EXTENDED: { + const auto &i = acc->ListAllIndices(); + const auto l_stats = acc->GetIndexStats(base_label_unindexed); + ASSERT_TRUE(l_stats); + ASSERT_EQ(l_stats->count, 1); + ASSERT_EQ(l_stats->avg_degree, 2); + const auto lp_stats = acc->GetIndexStats(base_label_indexed, property_id); + ASSERT_TRUE(lp_stats); + ASSERT_EQ(lp_stats->count, 1); + ASSERT_EQ(lp_stats->distinct_values_count, 2); + ASSERT_EQ(lp_stats->statistic, 3.4); + ASSERT_EQ(lp_stats->avg_group_size, 5.6); + ASSERT_EQ(lp_stats->avg_degree, 0.0); + const auto l_stats_ex = acc->GetIndexStats(extended_label_unused); + ASSERT_TRUE(l_stats_ex); + ASSERT_EQ(l_stats_ex->count, 123); + ASSERT_EQ(l_stats_ex->avg_degree, 9.87); + const auto lp_stats_ex = acc->GetIndexStats(extended_label_indexed, property_count); + ASSERT_TRUE(lp_stats_ex); + ASSERT_EQ(lp_stats_ex->count, 456798); + ASSERT_EQ(lp_stats_ex->distinct_values_count, 312345); + ASSERT_EQ(lp_stats_ex->statistic, 12312312.2); + ASSERT_EQ(lp_stats_ex->avg_group_size, 123123.2); + ASSERT_EQ(lp_stats_ex->avg_degree, 67876.9); + break; + } + } + } + // Verify constraints info. { - auto info = store->ListAllConstraints(); + auto info = acc->ListAllConstraints(); switch (type) { case DatasetType::ONLY_BASE: ASSERT_THAT(info.existence, UnorderedElementsAre(std::make_pair(base_label_unindexed, property_id))); @@ -280,9 +404,6 @@ class DurabilityTest : public ::testing::TestWithParam { break; } - // Create storage accessor. - auto acc = store->Access(); - // Verify base dataset. if (have_base_dataset) { // Verify vertices. @@ -1358,13 +1479,14 @@ TEST_P(DurabilityTest, WalCreateInSingleTransaction) { {.items = {.properties_on_edges = GetParam()}, .durability = {.storage_directory = storage_directory, .recover_on_startup = true}})); { - auto indices = store->ListAllIndices(); + auto acc = store->Access(); + + auto indices = acc->ListAllIndices(); ASSERT_EQ(indices.label.size(), 0); ASSERT_EQ(indices.label_property.size(), 0); - auto constraints = store->ListAllConstraints(); + auto constraints = acc->ListAllConstraints(); ASSERT_EQ(constraints.existence.size(), 0); ASSERT_EQ(constraints.unique.size(), 0); - auto acc = store->Access(); { auto v1 = acc->FindVertex(gid_v1, memgraph::storage::View::OLD); ASSERT_TRUE(v1); @@ -1461,20 +1583,38 @@ TEST_P(DurabilityTest, WalCreateAndRemoveEverything) { .wal_file_flush_every_n_tx = kFlushWalEvery}})); CreateBaseDataset(store.get(), GetParam()); CreateExtendedDataset(store.get()); - auto indices = store->ListAllIndices(); + auto indices = [&] { + auto acc = store->Access(); + auto res = acc->ListAllIndices(); + acc->Commit(); + return res; + }(); // iile for (const auto &index : indices.label) { - ASSERT_FALSE(store->DropIndex(index).HasError()); + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->DropIndex(index).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } for (const auto &index : indices.label_property) { - ASSERT_FALSE(store->DropIndex(index.first, index.second).HasError()); + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->DropIndex(index.first, index.second).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } - auto constraints = store->ListAllConstraints(); + auto constraints = [&] { + auto acc = store->Access(); + auto res = acc->ListAllConstraints(); + acc->Commit(); + return res; + }(); // iile for (const auto &constraint : constraints.existence) { - ASSERT_FALSE(store->DropExistenceConstraint(constraint.first, constraint.second, {}).HasError()); + auto unique_acc = store->UniqueAccess(); + ASSERT_FALSE(unique_acc->DropExistenceConstraint(constraint.first, constraint.second).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } for (const auto &constraint : constraints.unique) { - ASSERT_EQ(store->DropUniqueConstraint(constraint.first, constraint.second, {}).GetValue(), + auto unique_acc = store->UniqueAccess(); + ASSERT_EQ(unique_acc->DropUniqueConstraint(constraint.first, constraint.second), memgraph::storage::UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } auto acc = store->Access(); for (auto vertex : acc->Vertices(memgraph::storage::View::OLD)) { @@ -1493,13 +1633,13 @@ TEST_P(DurabilityTest, WalCreateAndRemoveEverything) { {.items = {.properties_on_edges = GetParam()}, .durability = {.storage_directory = storage_directory, .recover_on_startup = true}})); { - auto indices = store->ListAllIndices(); + auto acc = store->Access(); + auto indices = acc->ListAllIndices(); ASSERT_EQ(indices.label.size(), 0); ASSERT_EQ(indices.label_property.size(), 0); - auto constraints = store->ListAllConstraints(); + auto constraints = acc->ListAllConstraints(); ASSERT_EQ(constraints.existence.size(), 0); ASSERT_EQ(constraints.unique.size(), 0); - auto acc = store->Access(); uint64_t count = 0; auto iterable = acc->Vertices(memgraph::storage::View::OLD); for (auto it = iterable.begin(); it != iterable.end(); ++it) { diff --git a/tests/unit/storage_v2_gc.cpp b/tests/unit/storage_v2_gc.cpp index f17ad657b..122558c17 100644 --- a/tests/unit/storage_v2_gc.cpp +++ b/tests/unit/storage_v2_gc.cpp @@ -170,8 +170,11 @@ TEST(StorageV2Gc, Indices) { std::unique_ptr storage( std::make_unique(memgraph::storage::Config{ .gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC, .interval = std::chrono::milliseconds(100)}})); - - ASSERT_FALSE(storage->CreateIndex(storage->NameToLabel("label")).HasError()); + { + auto unique_acc = storage->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(storage->NameToLabel("label")).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } { auto acc0 = storage->Access(); diff --git a/tests/unit/storage_v2_indices.cpp b/tests/unit/storage_v2_indices.cpp index f63ac5b2c..142e15522 100644 --- a/tests/unit/storage_v2_indices.cpp +++ b/tests/unit/storage_v2_indices.cpp @@ -91,8 +91,8 @@ TYPED_TEST(IndexTest, LabelIndexCreate) { { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelIndexExists(this->label1)); + EXPECT_EQ(acc->ListAllIndices().label.size(), 0); } - EXPECT_EQ(this->storage->ListAllIndices().label.size(), 0); { auto acc = this->storage->Access(); @@ -103,7 +103,11 @@ TYPED_TEST(IndexTest, LabelIndexCreate) { ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); @@ -177,8 +181,8 @@ TYPED_TEST(IndexTest, LabelIndexDrop) { { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelIndexExists(this->label1)); + EXPECT_EQ(acc->ListAllIndices().label.size(), 0); } - EXPECT_EQ(this->storage->ListAllIndices().label.size(), 0); { auto acc = this->storage->Access(); @@ -189,7 +193,11 @@ TYPED_TEST(IndexTest, LabelIndexDrop) { ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); @@ -197,19 +205,27 @@ TYPED_TEST(IndexTest, LabelIndexDrop) { EXPECT_THAT(this->GetIds(acc->Vertices(this->label1, View::NEW), View::NEW), UnorderedElementsAre(1, 3, 5, 7, 9)); } - EXPECT_FALSE(this->storage->DropIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->DropIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelIndexExists(this->label1)); + EXPECT_EQ(acc->ListAllIndices().label.size(), 0); } - EXPECT_EQ(this->storage->ListAllIndices().label.size(), 0); - EXPECT_TRUE(this->storage->DropIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_TRUE(unique_acc->DropIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelIndexExists(this->label1)); + EXPECT_EQ(acc->ListAllIndices().label.size(), 0); } - EXPECT_EQ(this->storage->ListAllIndices().label.size(), 0); { auto acc = this->storage->Access(); @@ -220,12 +236,16 @@ TYPED_TEST(IndexTest, LabelIndexDrop) { ASSERT_NO_ERROR(acc->Commit()); } - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_TRUE(acc->LabelIndexExists(this->label1)); + EXPECT_THAT(acc->ListAllIndices().label, UnorderedElementsAre(this->label1)); } - EXPECT_THAT(this->storage->ListAllIndices().label, UnorderedElementsAre(this->label1)); { auto acc = this->storage->Access(); @@ -253,11 +273,19 @@ TYPED_TEST(IndexTest, LabelIndexBasic) { // 3. Remove Label1 from odd numbered vertices, and add it to even numbered // vertices. // 4. Delete even numbered vertices. - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); - EXPECT_FALSE(this->storage->CreateIndex(this->label2).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc = this->storage->Access(); - EXPECT_THAT(this->storage->ListAllIndices().label, UnorderedElementsAre(this->label1, this->label2)); + EXPECT_THAT(acc->ListAllIndices().label, UnorderedElementsAre(this->label1, this->label2)); EXPECT_THAT(this->GetIds(acc->Vertices(this->label1, View::OLD), View::OLD), IsEmpty()); EXPECT_THAT(this->GetIds(acc->Vertices(this->label2, View::OLD), View::OLD), IsEmpty()); EXPECT_THAT(this->GetIds(acc->Vertices(this->label1, View::NEW), View::NEW), IsEmpty()); @@ -318,8 +346,16 @@ TYPED_TEST(IndexTest, LabelIndexDuplicateVersions) { // By removing labels and adding them again we create duplicate entries for // the same vertex in the index (they only differ by the timestamp). This test // checks that duplicates are properly filtered out. - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); - EXPECT_FALSE(this->storage->CreateIndex(this->label2).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); @@ -358,8 +394,16 @@ TYPED_TEST(IndexTest, LabelIndexDuplicateVersions) { // passes TYPED_TEST(IndexTest, LabelIndexTransactionalIsolation) { // Check that transactions only see entries they are supposed to see. - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); - EXPECT_FALSE(this->storage->CreateIndex(this->label2).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc_before = this->storage->Access(); auto acc = this->storage->Access(); @@ -391,8 +435,16 @@ TYPED_TEST(IndexTest, LabelIndexTransactionalIsolation) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelIndexCountEstimate) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); - EXPECT_FALSE(this->storage->CreateIndex(this->label2).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc = this->storage->Access(); for (int i = 0; i < 20; ++i) { @@ -407,7 +459,11 @@ TYPED_TEST(IndexTest, LabelIndexCountEstimate) { TYPED_TEST(IndexTest, LabelIndexDeletedVertex) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); ASSERT_NO_ERROR(vertex1.AddLabel(this->label1)); @@ -427,7 +483,11 @@ TYPED_TEST(IndexTest, LabelIndexDeletedVertex) { TYPED_TEST(IndexTest, LabelIndexRemoveIndexedLabel) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); ASSERT_NO_ERROR(vertex1.AddLabel(this->label1)); @@ -447,7 +507,11 @@ TYPED_TEST(IndexTest, LabelIndexRemoveIndexedLabel) { TYPED_TEST(IndexTest, LabelIndexRemoveAndAddIndexedLabel) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); ASSERT_NO_ERROR(vertex1.AddLabel(this->label1)); @@ -472,7 +536,11 @@ TYPED_TEST(IndexTest, LabelIndexClearOldDataFromDisk) { auto *disk_label_index = static_cast(this->storage->indices_.label_index_.get()); - EXPECT_FALSE(this->storage->CreateIndex(this->label1).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex = this->CreateVertex(acc1.get()); ASSERT_NO_ERROR(vertex.AddLabel(this->label1)); @@ -500,46 +568,95 @@ TYPED_TEST(IndexTest, LabelIndexClearOldDataFromDisk) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelPropertyIndexCreateAndDrop) { - EXPECT_EQ(this->storage->ListAllIndices().label_property.size(), 0); - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_id).HasError()); + { + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllIndices().label_property.size(), 0); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_TRUE(acc->LabelPropertyIndexExists(this->label1, this->prop_id)); } - EXPECT_THAT(this->storage->ListAllIndices().label_property, - UnorderedElementsAre(std::make_pair(this->label1, this->prop_id))); + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllIndices().label_property, + UnorderedElementsAre(std::make_pair(this->label1, this->prop_id))); + } { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelPropertyIndexExists(this->label2, this->prop_id)); } - EXPECT_TRUE(this->storage->CreateIndex(this->label1, this->prop_id).HasError()); - EXPECT_THAT(this->storage->ListAllIndices().label_property, - UnorderedElementsAre(std::make_pair(this->label1, this->prop_id))); - EXPECT_FALSE(this->storage->CreateIndex(this->label2, this->prop_id).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_TRUE(unique_acc->CreateIndex(this->label1, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllIndices().label_property, + UnorderedElementsAre(std::make_pair(this->label1, this->prop_id))); + } + + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { auto acc = this->storage->Access(); EXPECT_TRUE(acc->LabelPropertyIndexExists(this->label2, this->prop_id)); } - EXPECT_THAT( - this->storage->ListAllIndices().label_property, - UnorderedElementsAre(std::make_pair(this->label1, this->prop_id), std::make_pair(this->label2, this->prop_id))); - EXPECT_FALSE(this->storage->DropIndex(this->label1, this->prop_id).HasError()); + { + auto acc = this->storage->Access(); + EXPECT_THAT( + acc->ListAllIndices().label_property, + UnorderedElementsAre(std::make_pair(this->label1, this->prop_id), std::make_pair(this->label2, this->prop_id))); + } + + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->DropIndex(this->label1, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelPropertyIndexExists(this->label1, this->prop_id)); } - EXPECT_THAT(this->storage->ListAllIndices().label_property, - UnorderedElementsAre(std::make_pair(this->label2, this->prop_id))); - EXPECT_TRUE(this->storage->DropIndex(this->label1, this->prop_id).HasError()); - EXPECT_FALSE(this->storage->DropIndex(this->label2, this->prop_id).HasError()); + { + auto acc = this->storage->Access(); + EXPECT_THAT(acc->ListAllIndices().label_property, + UnorderedElementsAre(std::make_pair(this->label2, this->prop_id))); + } + + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_TRUE(unique_acc->DropIndex(this->label1, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->DropIndex(this->label2, this->prop_id).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); EXPECT_FALSE(acc->LabelPropertyIndexExists(this->label2, this->prop_id)); } - EXPECT_EQ(this->storage->ListAllIndices().label_property.size(), 0); + + { + auto acc = this->storage->Access(); + EXPECT_EQ(acc->ListAllIndices().label_property.size(), 0); + } } // The following three tests are almost an exact copy-paste of the corresponding @@ -549,8 +666,16 @@ TYPED_TEST(IndexTest, LabelPropertyIndexCreateAndDrop) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelPropertyIndexBasic) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); - EXPECT_FALSE(this->storage->CreateIndex(this->label2, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label2, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc = this->storage->Access(); EXPECT_THAT(this->GetIds(acc->Vertices(this->label1, this->prop_val, View::OLD), View::OLD), IsEmpty()); @@ -636,7 +761,12 @@ TYPED_TEST(IndexTest, LabelPropertyIndexBasic) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelPropertyIndexDuplicateVersions) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } + { auto acc = this->storage->Access(); for (int i = 0; i < 5; ++i) { @@ -678,7 +808,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexDuplicateVersions) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelPropertyIndexTransactionalIsolation) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc_before = this->storage->Access(); auto acc = this->storage->Access(); @@ -717,7 +851,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexFiltering) { // We also have a mix of doubles and integers to verify that they are sorted // properly. - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } { auto acc = this->storage->Access(); @@ -787,7 +925,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexFiltering) { // NOLINTNEXTLINE(hicpp-special-member-functions) TYPED_TEST(IndexTest, LabelPropertyIndexCountEstimate) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc = this->storage->Access(); for (int i = 1; i <= 10; ++i) { @@ -811,7 +953,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexCountEstimate) { } TYPED_TEST(IndexTest, LabelPropertyIndexMixedIteration) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } const std::array temporals{TemporalData{TemporalType::Date, 23}, TemporalData{TemporalType::Date, 28}, TemporalData{TemporalType::LocalDateTime, 20}}; @@ -1019,7 +1165,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexMixedIteration) { TYPED_TEST(IndexTest, LabelPropertyIndexDeletedVertex) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); @@ -1048,7 +1198,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexDeletedVertex) { /// TODO: empty lines, so it is easier to read what is actually going on here TYPED_TEST(IndexTest, LabelPropertyIndexRemoveIndexedLabel) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); @@ -1076,7 +1230,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexRemoveIndexedLabel) { TYPED_TEST(IndexTest, LabelPropertyIndexRemoveAndAddIndexedLabel) { if constexpr ((std::is_same_v)) { - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex1 = this->CreateVertex(acc1.get()); @@ -1105,7 +1263,11 @@ TYPED_TEST(IndexTest, LabelPropertyIndexClearOldDataFromDisk) { auto *disk_label_property_index = static_cast(this->storage->indices_.label_property_index_.get()); - EXPECT_FALSE(this->storage->CreateIndex(this->label1, this->prop_val).HasError()); + { + auto unique_acc = this->storage->UniqueAccess(); + EXPECT_FALSE(unique_acc->CreateIndex(this->label1, this->prop_val).HasError()); + ASSERT_NO_ERROR(unique_acc->Commit()); + } auto acc1 = this->storage->Access(); auto vertex = this->CreateVertex(acc1.get()); ASSERT_NO_ERROR(vertex.AddLabel(this->label1)); diff --git a/tests/unit/storage_v2_replication.cpp b/tests/unit/storage_v2_replication.cpp index bbf55796a..85b7f801b 100644 --- a/tests/unit/storage_v2_replication.cpp +++ b/tests/unit/storage_v2_replication.cpp @@ -21,26 +21,30 @@ #include #include #include -#include "storage/v2/replication/config.hpp" +#include "replication/config.hpp" +#include "storage/v2/indices/label_index_stats.hpp" +#include "storage/v2/replication/replication_handler.hpp" #include "storage/v2/storage.hpp" #include "storage/v2/view.hpp" using testing::UnorderedElementsAre; +using memgraph::replication::ReplicationClientConfig; +using memgraph::replication::ReplicationMode; +using memgraph::replication::ReplicationRole; +using memgraph::replication::ReplicationServerConfig; using memgraph::storage::Config; using memgraph::storage::EdgeAccessor; using memgraph::storage::Gid; using memgraph::storage::InMemoryStorage; using memgraph::storage::PropertyValue; using memgraph::storage::RegisterReplicaError; +using memgraph::storage::RegistrationMode; +using memgraph::storage::ReplicationHandler; using memgraph::storage::Storage; +using memgraph::storage::UnregisterReplicaResult; using memgraph::storage::View; -using memgraph::storage::replication::RegistrationMode; using memgraph::storage::replication::ReplicaState; -using memgraph::storage::replication::ReplicationClientConfig; -using memgraph::storage::replication::ReplicationMode; -using memgraph::storage::replication::ReplicationRole; -using memgraph::storage::replication::ReplicationServerConfig; class ReplicationTest : public ::testing::Test { protected: @@ -71,19 +75,21 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) { std::unique_ptr main_store = std::make_unique(configuration); std::unique_ptr replica_store = std::make_unique(configuration); - replica_store->SetReplicaRole(ReplicationServerConfig{ + auto replica_store_handler = ReplicationHandler{replica_store->repl_state_, *replica_store}; + replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = "REPLICA", - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = "REPLICA", + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .HasError()); // vertex create @@ -229,27 +235,65 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) { const auto *label = "label"; const auto *property = "property"; const auto *property_extra = "property_extra"; + const memgraph::storage::LabelIndexStats l_stats{12, 34}; + const memgraph::storage::LabelPropertyIndexStats lp_stats{98, 76, 5.4, 3.2, 1.0}; + { - ASSERT_FALSE(main_store->CreateIndex(main_store->NameToLabel(label)).HasError()); + auto unique_acc = main_store->UniqueAccess(); + ASSERT_FALSE(unique_acc->CreateIndex(main_store->NameToLabel(label)).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + unique_acc->SetIndexStats(main_store->NameToLabel(label), l_stats); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); ASSERT_FALSE( - main_store->CreateIndex(main_store->NameToLabel(label), main_store->NameToProperty(property)).HasError()); + unique_acc->CreateIndex(main_store->NameToLabel(label), main_store->NameToProperty(property)).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + unique_acc->SetIndexStats(main_store->NameToLabel(label), main_store->NameToProperty(property), lp_stats); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); ASSERT_FALSE( - main_store->CreateExistenceConstraint(main_store->NameToLabel(label), main_store->NameToProperty(property), {}) + unique_acc->CreateExistenceConstraint(main_store->NameToLabel(label), main_store->NameToProperty(property)) .HasError()); - ASSERT_FALSE(main_store - ->CreateUniqueConstraint( - main_store->NameToLabel(label), - {main_store->NameToProperty(property), main_store->NameToProperty(property_extra)}, {}) - .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + ASSERT_FALSE( + unique_acc + ->CreateUniqueConstraint(main_store->NameToLabel(label), + {main_store->NameToProperty(property), main_store->NameToProperty(property_extra)}) + .HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { - const auto indices = replica_store->ListAllIndices(); + const auto indices = replica_store->Access()->ListAllIndices(); ASSERT_THAT(indices.label, UnorderedElementsAre(replica_store->NameToLabel(label))); ASSERT_THAT(indices.label_property, UnorderedElementsAre(std::make_pair(replica_store->NameToLabel(label), replica_store->NameToProperty(property)))); - - const auto constraints = replica_store->ListAllConstraints(); + const auto &l_stats_rep = replica_store->Access()->GetIndexStats(replica_store->NameToLabel(label)); + ASSERT_TRUE(l_stats_rep); + ASSERT_EQ(l_stats_rep->count, l_stats.count); + ASSERT_EQ(l_stats_rep->avg_degree, l_stats.avg_degree); + const auto &lp_stats_rep = replica_store->Access()->GetIndexStats(replica_store->NameToLabel(label), + replica_store->NameToProperty(property)); + ASSERT_TRUE(lp_stats_rep); + ASSERT_EQ(lp_stats_rep->count, lp_stats.count); + ASSERT_EQ(lp_stats_rep->distinct_values_count, lp_stats.distinct_values_count); + ASSERT_EQ(lp_stats_rep->statistic, lp_stats.statistic); + ASSERT_EQ(lp_stats_rep->avg_group_size, lp_stats.avg_group_size); + ASSERT_EQ(lp_stats_rep->avg_degree, lp_stats.avg_degree); + const auto constraints = replica_store->Access()->ListAllConstraints(); ASSERT_THAT(constraints.existence, UnorderedElementsAre(std::make_pair(replica_store->NameToLabel(label), replica_store->NameToProperty(property)))); ASSERT_THAT(constraints.unique, @@ -263,26 +307,54 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) { // existence constraint drop // unique constriant drop { - ASSERT_FALSE(main_store->DropIndex(main_store->NameToLabel(label)).HasError()); + auto unique_acc = main_store->UniqueAccess(); + unique_acc->DeleteLabelIndexStats(main_store->NameToLabel(label)); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + ASSERT_FALSE(unique_acc->DropIndex(main_store->NameToLabel(label)).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + unique_acc->DeleteLabelPropertyIndexStats(main_store->NameToLabel(label)); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); ASSERT_FALSE( - main_store->DropIndex(main_store->NameToLabel(label), main_store->NameToProperty(property)).HasError()); + unique_acc->DropIndex(main_store->NameToLabel(label), main_store->NameToProperty(property)).HasError()); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); ASSERT_FALSE( - main_store->DropExistenceConstraint(main_store->NameToLabel(label), main_store->NameToProperty(property), {}) + unique_acc->DropExistenceConstraint(main_store->NameToLabel(label), main_store->NameToProperty(property)) .HasError()); - ASSERT_EQ(main_store - ->DropUniqueConstraint( - main_store->NameToLabel(label), - {main_store->NameToProperty(property), main_store->NameToProperty(property_extra)}, {}) - .GetValue(), - memgraph::storage::UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); + } + { + auto unique_acc = main_store->UniqueAccess(); + ASSERT_EQ( + unique_acc->DropUniqueConstraint(main_store->NameToLabel(label), {main_store->NameToProperty(property), + main_store->NameToProperty(property_extra)}), + memgraph::storage::UniqueConstraints::DeletionStatus::SUCCESS); + ASSERT_FALSE(unique_acc->Commit().HasError()); } { - const auto indices = replica_store->ListAllIndices(); + const auto indices = replica_store->Access()->ListAllIndices(); ASSERT_EQ(indices.label.size(), 0); ASSERT_EQ(indices.label_property.size(), 0); - const auto constraints = replica_store->ListAllConstraints(); + const auto &l_stats_rep = replica_store->Access()->GetIndexStats(replica_store->NameToLabel(label)); + ASSERT_FALSE(l_stats_rep); + const auto &lp_stats_rep = replica_store->Access()->GetIndexStats(replica_store->NameToLabel(label), + replica_store->NameToProperty(property)); + ASSERT_FALSE(lp_stats_rep); + + const auto constraints = replica_store->Access()->ListAllConstraints(); ASSERT_EQ(constraints.existence.size(), 0); ASSERT_EQ(constraints.unique.size(), 0); } @@ -301,7 +373,8 @@ TEST_F(ReplicationTest, MultipleSynchronousReplicationTest) { .snapshot_wal_mode = Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL, }})}; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); @@ -311,28 +384,30 @@ TEST_F(ReplicationTest, MultipleSynchronousReplicationTest) { .storage_directory = storage_directory, .snapshot_wal_mode = Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL, }})}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[1], }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .HasError()); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[1], - }) + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[1], + }) .HasError()); const auto *vertex_label = "label"; @@ -362,7 +437,8 @@ TEST_F(ReplicationTest, MultipleSynchronousReplicationTest) { check_replica(replica_store1.get()); check_replica(replica_store2.get()); - main_store->UnregisterReplica(replicas[1]); + auto handler = ReplicationHandler{main_store->repl_state_, *main_store}; + handler.UnregisterReplica(replicas[1]); { auto acc = main_store->Access(); auto v = acc->CreateVertex(); @@ -459,19 +535,20 @@ TEST_F(ReplicationTest, RecoveryProcess) { {.durability = {.storage_directory = replica_storage_directory, .snapshot_wal_mode = Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL}})}; - replica_store->SetReplicaRole(ReplicationServerConfig{ + auto replica_store_handler = ReplicationHandler{replica_store->repl_state_, *replica_store}; + replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); - - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .HasError()); ASSERT_EQ(main_store->GetReplicaState(replicas[0]), ReplicaState::RECOVERY); @@ -533,19 +610,21 @@ TEST_F(ReplicationTest, BasicAsynchronousReplicationTest) { std::unique_ptr replica_store_async{new InMemoryStorage(configuration)}; - replica_store_async->SetReplicaRole(ReplicationServerConfig{ + auto replica_store_handler = ReplicationHandler{replica_store_async->repl_state_, *replica_store_async}; + replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[1], }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = "REPLICA_ASYNC", - .mode = ReplicationMode::ASYNC, - .ip_address = local_host, - .port = ports[1], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = "REPLICA_ASYNC", + .mode = ReplicationMode::ASYNC, + .ip_address = local_host, + .port = ports[1], + }) .HasError()); static constexpr size_t vertices_create_num = 10; @@ -580,36 +659,39 @@ TEST_F(ReplicationTest, EpochTest) { std::unique_ptr main_store{new InMemoryStorage(configuration)}; std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = 10001, }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .HasError()); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = 10001, - }) + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = 10001, + }) .HasError()); std::optional vertex_gid; @@ -632,18 +714,19 @@ TEST_F(ReplicationTest, EpochTest) { ASSERT_FALSE(acc->Commit().HasError()); } - main_store->UnregisterReplica(replicas[0]); - main_store->UnregisterReplica(replicas[1]); + main_store_handler.UnregisterReplica(replicas[0]); + main_store_handler.UnregisterReplica(replicas[1]); - replica_store1->SetMainReplicationRole(); - ASSERT_FALSE(replica_store1 - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = 10001, - }) + ASSERT_TRUE(replica1_store_handler.SetReplicationRoleMain()); + + ASSERT_FALSE(replica1_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = 10001, + }) .HasError()); @@ -666,18 +749,18 @@ TEST_F(ReplicationTest, EpochTest) { ASSERT_FALSE(acc->Commit().HasError()); } - replica_store1->SetReplicaRole(ReplicationServerConfig{ + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); - ASSERT_TRUE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + ASSERT_TRUE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .HasError()); @@ -702,43 +785,46 @@ TEST_F(ReplicationTest, ReplicationInformation) { std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; uint16_t replica1_port = 10001; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = replica1_port, }); uint16_t replica2_port = 10002; std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = replica2_port, }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = replica1_port, - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = replica1_port, + }) .HasError()); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::ASYNC, - .ip_address = local_host, - .port = replica2_port, - }) + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::ASYNC, + .ip_address = local_host, + .port = replica2_port, + }) .HasError()); - ASSERT_EQ(main_store->GetReplicationRole(), ReplicationRole::MAIN); - ASSERT_EQ(replica_store1->GetReplicationRole(), ReplicationRole::REPLICA); - ASSERT_EQ(replica_store2->GetReplicationRole(), ReplicationRole::REPLICA); + ASSERT_TRUE(main_store->repl_state_.IsMain()); + ASSERT_TRUE(replica_store1->repl_state_.IsReplica()); + ASSERT_TRUE(replica_store2->repl_state_.IsReplica()); const auto replicas_info = main_store->ReplicasInfo(); ASSERT_EQ(replicas_info.size(), 2); @@ -761,36 +847,38 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingName) { std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; uint16_t replica1_port = 10001; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = replica1_port, }); uint16_t replica2_port = 10002; std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = replica2_port, }); - - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = replica1_port, - }) - .HasError()); - - ASSERT_TRUE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ .name = replicas[0], - .mode = ReplicationMode::ASYNC, + .mode = ReplicationMode::SYNC, .ip_address = local_host, - .port = replica2_port, + .port = replica1_port, }) + .HasError()); + + ASSERT_TRUE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[0], + .mode = ReplicationMode::ASYNC, + .ip_address = local_host, + .port = replica2_port, + }) .GetError() == RegisterReplicaError::NAME_EXISTS); } @@ -799,35 +887,38 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingEndPoint) { std::unique_ptr main_store{new InMemoryStorage(configuration)}; std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = common_port, }); std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = common_port, }); - ASSERT_FALSE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = replicas[0], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = common_port, - }) - .HasError()); - - ASSERT_TRUE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_FALSE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::ASYNC, + .name = replicas[0], + .mode = ReplicationMode::SYNC, .ip_address = local_host, .port = common_port, }) + .HasError()); + + ASSERT_TRUE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::ASYNC, + .ip_address = local_host, + .port = common_port, + }) .GetError() == RegisterReplicaError::END_POINT_EXISTS); } @@ -837,30 +928,34 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAfterDroppingReplica) { std::unique_ptr main_store{new InMemoryStorage(main_config)}; std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[1], }); - auto res = main_store->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + auto res = + main_store_handler.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ .name = replicas[0], .mode = ReplicationMode::SYNC, .ip_address = local_host, .port = ports[0], }); ASSERT_FALSE(res.HasError()); - res = main_store->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[1], - }); + res = main_store_handler.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[1], + }); ASSERT_FALSE(res.HasError()); auto replica_infos = main_store->ReplicasInfo(); @@ -894,31 +989,34 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartup) { std::unique_ptr main_store{new InMemoryStorage(main_config)}; std::unique_ptr replica_store1{new InMemoryStorage(configuration)}; - replica_store1->SetReplicaRole(ReplicationServerConfig{ + auto replica1_store_handler = ReplicationHandler{replica_store1->repl_state_, *replica_store1}; + replica1_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[0], }); std::unique_ptr replica_store2{new InMemoryStorage(configuration)}; - replica_store2->SetReplicaRole(ReplicationServerConfig{ + auto replica2_store_handler = ReplicationHandler{replica_store2->repl_state_, *replica_store2}; + replica2_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{ .ip_address = local_host, .port = ports[1], }); - - auto res = main_store->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + auto res = + main_store_handler.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ .name = replicas[0], .mode = ReplicationMode::SYNC, .ip_address = local_host, .port = ports[0], }); ASSERT_FALSE(res.HasError()); - res = main_store->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ - .name = replicas[1], - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[1], - }); + res = main_store_handler.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, ReplicationClientConfig{ + .name = replicas[1], + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[1], + }); ASSERT_FALSE(res.HasError()); auto replica_infos = main_store->ReplicasInfo(); @@ -931,8 +1029,9 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartup) { ASSERT_EQ(replica_infos[1].endpoint.address, local_host); ASSERT_EQ(replica_infos[1].endpoint.port, ports[1]); - const auto unregister_res = main_store->UnregisterReplica(replicas[0]); - ASSERT_TRUE(unregister_res); + auto handler = ReplicationHandler{main_store->repl_state_, *main_store}; + const auto unregister_res = handler.UnregisterReplica(replicas[0]); + ASSERT_EQ(unregister_res, UnregisterReplicaResult::SUCCESS); replica_infos = main_store->ReplicasInfo(); ASSERT_EQ(replica_infos.size(), 1); @@ -953,13 +1052,14 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartup) { TEST_F(ReplicationTest, AddingInvalidReplica) { std::unique_ptr main_store{new InMemoryStorage(configuration)}; - ASSERT_TRUE(main_store - ->RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, - ReplicationClientConfig{ - .name = "REPLICA", - .mode = ReplicationMode::SYNC, - .ip_address = local_host, - .port = ports[0], - }) + auto main_store_handler = ReplicationHandler{main_store->repl_state_, *main_store}; + ASSERT_TRUE(main_store_handler + .RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, + ReplicationClientConfig{ + .name = "REPLICA", + .mode = ReplicationMode::SYNC, + .ip_address = local_host, + .port = ports[0], + }) .GetError() == RegisterReplicaError::CONNECTION_FAILED); } diff --git a/tests/unit/storage_v2_wal_file.cpp b/tests/unit/storage_v2_wal_file.cpp index dd15c511b..19b633f9f 100644 --- a/tests/unit/storage_v2_wal_file.cpp +++ b/tests/unit/storage_v2_wal_file.cpp @@ -20,6 +20,7 @@ #include "storage/v2/durability/exceptions.hpp" #include "storage/v2/durability/version.hpp" #include "storage/v2/durability/wal.hpp" +#include "storage/v2/indices/label_index_stats.hpp" #include "storage/v2/mvcc.hpp" #include "storage/v2/name_id_mapper.hpp" #include "storage_test_utils.hpp" @@ -29,24 +30,32 @@ #include "utils/uuid.hpp" // Helper function used to convert between enum types. -memgraph::storage::durability::WalDeltaData::Type StorageGlobalOperationToWalDeltaDataType( - memgraph::storage::durability::StorageGlobalOperation operation) { +memgraph::storage::durability::WalDeltaData::Type StorageMetadataOperationToWalDeltaDataType( + memgraph::storage::durability::StorageMetadataOperation operation) { switch (operation) { - case memgraph::storage::durability::StorageGlobalOperation::LABEL_INDEX_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_CREATE: return memgraph::storage::durability::WalDeltaData::Type::LABEL_INDEX_CREATE; - case memgraph::storage::durability::StorageGlobalOperation::LABEL_INDEX_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_DROP: return memgraph::storage::durability::WalDeltaData::Type::LABEL_INDEX_DROP; - case memgraph::storage::durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_STATS_SET: + return memgraph::storage::durability::WalDeltaData::Type::LABEL_INDEX_STATS_SET; + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_STATS_CLEAR: + return memgraph::storage::durability::WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR; + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE: return memgraph::storage::durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE; - case memgraph::storage::durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP: return memgraph::storage::durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP; - case memgraph::storage::durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET: + return memgraph::storage::durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET; + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR: + return memgraph::storage::durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR; + case memgraph::storage::durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE: return memgraph::storage::durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE; - case memgraph::storage::durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: + case memgraph::storage::durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP: return memgraph::storage::durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP; - case memgraph::storage::durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_CREATE: return memgraph::storage::durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE; - case memgraph::storage::durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: + case memgraph::storage::durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_DROP: return memgraph::storage::durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP; } } @@ -185,6 +194,17 @@ class DeltaGenerator final { } } + void FinalizeOperationTx() { + auto timestamp = gen_->timestamp_; + gen_->wal_file_.AppendTransactionEnd(timestamp); + if (gen_->valid_) { + gen_->UpdateStats(timestamp, 1); + memgraph::storage::durability::WalDeltaData data{ + .type = memgraph::storage::durability::WalDeltaData::Type::TRANSACTION_END}; + gen_->data_.emplace_back(timestamp, data); + } + } + private: DeltaGenerator *gen_; memgraph::storage::Transaction transaction_; @@ -210,31 +230,54 @@ class DeltaGenerator final { valid_ = false; } - void AppendOperation(memgraph::storage::durability::StorageGlobalOperation operation, const std::string &label, - const std::set properties = {}) { + void AppendOperation(memgraph::storage::durability::StorageMetadataOperation operation, const std::string &label, + const std::set properties = {}, const std::string &stats = {}) { auto label_id = memgraph::storage::LabelId::FromUint(mapper_.NameToId(label)); std::set property_ids; for (const auto &property : properties) { property_ids.insert(memgraph::storage::PropertyId::FromUint(mapper_.NameToId(property))); } - wal_file_.AppendOperation(operation, label_id, property_ids, timestamp_); + memgraph::storage::LabelIndexStats l_stats{}; + memgraph::storage::LabelPropertyIndexStats lp_stats{}; + if (!stats.empty()) { + if (operation == memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_STATS_SET) { + ASSERT_TRUE(FromJson(stats, l_stats)); + } else if (operation == memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET) { + ASSERT_TRUE(FromJson(stats, lp_stats)); + } else { + ASSERT_TRUE(false) << "Unexpected statistics operation!"; + } + } + wal_file_.AppendOperation(operation, label_id, property_ids, l_stats, lp_stats, timestamp_); if (valid_) { UpdateStats(timestamp_, 1); memgraph::storage::durability::WalDeltaData data; - data.type = StorageGlobalOperationToWalDeltaDataType(operation); + data.type = StorageMetadataOperationToWalDeltaDataType(operation); switch (operation) { - case memgraph::storage::durability::StorageGlobalOperation::LABEL_INDEX_CREATE: - case memgraph::storage::durability::StorageGlobalOperation::LABEL_INDEX_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_STATS_CLEAR: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_CLEAR: /* Special case + */ data.operation_label.label = label; break; - case memgraph::storage::durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE: - case memgraph::storage::durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP: - case memgraph::storage::durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE: - case memgraph::storage::durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_INDEX_STATS_SET: + data.operation_label_stats.label = label; + data.operation_label_stats.stats = stats; + break; + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_DROP: + case memgraph::storage::durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::EXISTENCE_CONSTRAINT_DROP: data.operation_label_property.label = label; data.operation_label_property.property = *properties.begin(); - case memgraph::storage::durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE: - case memgraph::storage::durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP: + case memgraph::storage::durability::StorageMetadataOperation::LABEL_PROPERTY_INDEX_STATS_SET: + data.operation_label_property_stats.label = label; + data.operation_label_property_stats.property = *properties.begin(); + data.operation_label_property_stats.stats = stats; + break; + case memgraph::storage::durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_CREATE: + case memgraph::storage::durability::StorageMetadataOperation::UNIQUE_CONSTRAINT_DROP: data.operation_label_properties.label = label; data.operation_label_properties.properties = properties; } @@ -299,7 +342,15 @@ class DeltaGenerator final { } // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) -#define OPERATION(op, ...) gen.AppendOperation(memgraph::storage::durability::StorageGlobalOperation::op, __VA_ARGS__) +#define OPERATION(op, ...) gen.AppendOperation(memgraph::storage::durability::StorageMetadataOperation::op, __VA_ARGS__) + +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define OPERATION_TX(op, ...) \ + { \ + auto tx = gen.CreateTransaction(); \ + OPERATION(op, __VA_ARGS__); \ + tx.FinalizeOperationTx(); \ + } void AssertWalInfoEqual(const memgraph::storage::durability::WalInfo &a, const memgraph::storage::durability::WalInfo &b) { @@ -392,7 +443,7 @@ GENERATE_SIMPLE_TEST(TransactionWithEnd, { TRANSACTION(true, { tx.CreateVertex() // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionWithoutEnd, { TRANSACTION(false, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) -GENERATE_SIMPLE_TEST(OperationSingle, { OPERATION(LABEL_INDEX_CREATE, "hello"); }); +GENERATE_SIMPLE_TEST(OperationSingle, { OPERATION_TX(LABEL_INDEX_CREATE, "hello"); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsEnd00, { @@ -417,25 +468,25 @@ GENERATE_SIMPLE_TEST(TransactionsEnd11, { // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation_00, { - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(false, { tx.CreateVertex(); }); TRANSACTION(false, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation_01, { - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(false, { tx.CreateVertex(); }); TRANSACTION(true, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation_10, { - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(true, { tx.CreateVertex(); }); TRANSACTION(false, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation_11, { - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(true, { tx.CreateVertex(); }); TRANSACTION(true, { tx.CreateVertex(); }); }); @@ -443,25 +494,25 @@ GENERATE_SIMPLE_TEST(TransactionsWithOperation_11, { // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation0_0, { TRANSACTION(false, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(false, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation0_1, { TRANSACTION(false, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(true, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation1_0, { TRANSACTION(true, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(false, { tx.CreateVertex(); }); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation1_1, { TRANSACTION(true, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); TRANSACTION(true, { tx.CreateVertex(); }); }); @@ -469,25 +520,25 @@ GENERATE_SIMPLE_TEST(TransactionsWithOperation1_1, { GENERATE_SIMPLE_TEST(TransactionsWithOperation00_, { TRANSACTION(false, { tx.CreateVertex(); }); TRANSACTION(false, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation01_, { TRANSACTION(false, { tx.CreateVertex(); }); TRANSACTION(true, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation10_, { TRANSACTION(true, { tx.CreateVertex(); }); TRANSACTION(false, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); }); // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(TransactionsWithOperation11_, { TRANSACTION(true, { tx.CreateVertex(); }); TRANSACTION(true, { tx.CreateVertex(); }); - OPERATION(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); }); // NOLINTNEXTLINE(hicpp-special-member-functions) @@ -518,16 +569,37 @@ GENERATE_SIMPLE_TEST(AllTransactionOperationsWithoutEnd, { tx.DeleteVertex(vertex1); }); }); + +// NOLINTNEXTLINE(hicpp-special-member-functions) +GENERATE_SIMPLE_TEST(MultiOpTransaction, { + namespace ms = memgraph::storage; + auto l_stats = ms::ToJson(ms::LabelIndexStats{12, 34}); + auto lp_stats = ms::ToJson(ms::LabelPropertyIndexStats{98, 76, 54., 32., 10.}); + auto tx = gen.CreateTransaction(); + OPERATION(LABEL_PROPERTY_INDEX_STATS_SET, "hello", {"world"}, lp_stats); + OPERATION(LABEL_PROPERTY_INDEX_STATS_SET, "hello", {"and"}, lp_stats); + OPERATION(LABEL_PROPERTY_INDEX_STATS_SET, "hello", {"universe"}, lp_stats); + OPERATION(LABEL_INDEX_STATS_SET, "hello", {}, l_stats); + tx.FinalizeOperationTx(); +}); + // NOLINTNEXTLINE(hicpp-special-member-functions) GENERATE_SIMPLE_TEST(AllGlobalOperations, { - OPERATION(LABEL_INDEX_CREATE, "hello"); - OPERATION(LABEL_INDEX_DROP, "hello"); - OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"}); - OPERATION(LABEL_PROPERTY_INDEX_DROP, "hello", {"world"}); - OPERATION(EXISTENCE_CONSTRAINT_CREATE, "hello", {"world"}); - OPERATION(EXISTENCE_CONSTRAINT_DROP, "hello", {"world"}); - OPERATION(UNIQUE_CONSTRAINT_CREATE, "hello", {"world", "and", "universe"}); - OPERATION(UNIQUE_CONSTRAINT_DROP, "hello", {"world", "and", "universe"}); + namespace ms = memgraph::storage; + OPERATION_TX(LABEL_INDEX_CREATE, "hello"); + OPERATION_TX(LABEL_INDEX_DROP, "hello"); + auto l_stats = ms::ToJson(ms::LabelIndexStats{12, 34}); + OPERATION_TX(LABEL_INDEX_STATS_SET, "hello", {}, l_stats); + OPERATION_TX(LABEL_INDEX_STATS_CLEAR, "hello"); + OPERATION_TX(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"}); + OPERATION_TX(LABEL_PROPERTY_INDEX_DROP, "hello", {"world"}); + auto lp_stats = ms::ToJson(ms::LabelPropertyIndexStats{98, 76, 54., 32., 10.}); + OPERATION_TX(LABEL_PROPERTY_INDEX_STATS_SET, "hello", {"world"}, lp_stats); + OPERATION_TX(LABEL_PROPERTY_INDEX_STATS_CLEAR, "hello"); + OPERATION_TX(EXISTENCE_CONSTRAINT_CREATE, "hello", {"world"}); + OPERATION_TX(EXISTENCE_CONSTRAINT_DROP, "hello", {"world"}); + OPERATION_TX(UNIQUE_CONSTRAINT_CREATE, "hello", {"world", "and", "universe"}); + OPERATION_TX(UNIQUE_CONSTRAINT_DROP, "hello", {"world", "and", "universe"}); }); // NOLINTNEXTLINE(hicpp-special-member-functions) @@ -587,7 +659,7 @@ TEST_P(WalFileTest, PartialData) { tx.AddLabel(vertex, "hello"); }); infos.emplace_back(gen.GetPosition(), gen.GetInfo()); - OPERATION(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"}); + OPERATION_TX(LABEL_PROPERTY_INDEX_CREATE, "hello", {"world"}); infos.emplace_back(gen.GetPosition(), gen.GetInfo()); TRANSACTION(true, { auto vertex1 = tx.CreateVertex(); diff --git a/tests/unit/utils_resource_lock.cpp b/tests/unit/utils_resource_lock.cpp new file mode 100644 index 000000000..bb3903824 --- /dev/null +++ b/tests/unit/utils_resource_lock.cpp @@ -0,0 +1,161 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "utils/resource_lock.hpp" + +#include +#include +#include + +using namespace memgraph::utils; + +// Define a test fixture for the ResourceLock class +class ResourceLockTest : public testing::Test { + protected: + ResourceLock lock; + + void SetUp() override { + // Setup code, if any + } + + void TearDown() override { + // Tear down code, if any + } +}; + +TEST_F(ResourceLockTest, MultiThreadedUniqueAccess) { + constexpr int num_threads = 10; + int counter = 0; + + // Lambda function representing the task performed by each thread + auto unique_task = [&]() { + for (int i = 0; i < 100; ++i) { + lock.lock(); + // Critical section: Increment counter safely using the lock + ++counter; + lock.unlock(); + } + }; + + // Create multiple threads and execute the task concurrently + std::vector threads; + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back(unique_task); + } + + // Wait for all threads to finish + for (auto &thread : threads) { + thread.join(); + } + + // Assert that the counter value is as expected (total number of iterations) + ASSERT_EQ(counter, num_threads * 100); +} + +TEST_F(ResourceLockTest, MultiThreadedSharedAccess) { + constexpr int num_threads = 10; + int counter = 123; + + // Lambda function representing the shared task performed by each thread + auto shared_task = [&]() { + for (int i = 0; i < 100; ++i) { + lock.lock_shared(); + // Read the counter value safely using shared access + EXPECT_EQ(counter, 123); + lock.unlock_shared(); + } + }; + + // Create multiple threads and execute the shared task concurrently + std::vector threads; + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back(shared_task); + } + + // Wait for all threads to finish + for (auto &thread : threads) { + thread.join(); + } +} + +TEST_F(ResourceLockTest, MultiThreadedMixedAccess) { + constexpr int num_threads = 10; + int counter = 0; + + // Lambda function representing the shared task performed by each thread + auto shared_task = [&](int expecting) { + for (int i = 0; i < 100; ++i) { + lock.lock_shared(); + // Read the counter value safely using shared access + EXPECT_EQ(counter, expecting); + lock.unlock_shared(); + } + }; + + // Lambda function representing the task performed by each thread + auto unique_task = [&]() { + for (int i = 0; i < 100; ++i) { + lock.lock(); + // Critical section: Increment counter safely using the lock + ++counter; + lock.unlock(); + } + }; + + std::vector threads; + + // Unique vs shared test 1 + { + std::unique_lock l(lock); + // Uniquely locked; spin up shared tasks and update while they are running + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back([shared_task] { return shared_task(3456); }); + } + // Update while still holding unique access + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + counter = 3456; + } + // Wait for all threads to finish + threads.clear(); + + // Unique vs shared test 2 + { + std::shared_lock l(lock); + // Shared locked; spin up unique tasks and read while they are running + for (int i = 0; i < num_threads; ++i) { + threads.emplace_back(unique_task); + } + // Update while still holding unique access + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + EXPECT_EQ(counter, 3456); + } + // Wait for all threads to finish + threads.clear(); + EXPECT_EQ(counter, 3456 + num_threads * 100); +} + +TEST_F(ResourceLockTest, TryLock) { + ASSERT_TRUE(lock.try_lock()); + ASSERT_FALSE(lock.try_lock()); + ASSERT_FALSE(lock.try_lock_shared()); + lock.unlock(); + ASSERT_TRUE(lock.try_lock_shared()); + ASSERT_TRUE(lock.try_lock_shared()); + ASSERT_FALSE(lock.try_lock()); + ASSERT_TRUE(lock.try_lock_shared()); + ASSERT_TRUE(lock.try_lock_shared()); + lock.unlock_shared(); + lock.unlock_shared(); + lock.unlock_shared(); + lock.unlock_shared(); + ASSERT_TRUE(lock.try_lock()); + lock.unlock(); +} diff --git a/tools/bench-graph-client/main.py b/tools/bench-graph-client/main.py index edb09ecde..6456ae10c 100755 --- a/tools/bench-graph-client/main.py +++ b/tools/bench-graph-client/main.py @@ -20,9 +20,7 @@ GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "") GITHUB_SHA = os.getenv("GITHUB_SHA", "") GITHUB_REF = os.getenv("GITHUB_REF", "") -BENCH_GRAPH_SERVER_ENDPOINT = os.getenv( - "BENCH_GRAPH_SERVER_ENDPOINT", - "http://bench-graph-api:9001") +BENCH_GRAPH_SERVER_ENDPOINT = os.getenv("BENCH_GRAPH_SERVER_ENDPOINT", "http://bench-graph-api:9001") log = logging.getLogger(__name__) @@ -30,7 +28,9 @@ log = logging.getLogger(__name__) def parse_args(): argp = ArgumentParser(description=__doc__) argp.add_argument("--benchmark-name", type=str, required=True) - argp.add_argument("--benchmark-results-path", type=str, required=True) + argp.add_argument("--benchmark-results", type=str, required=True) + argp.add_argument("--benchmark-results-in-memory-analytical-path", type=str, required=False) + argp.add_argument("--benchmark-results-on-disk-txn-path", type=str, required=False) argp.add_argument("--github-run-id", type=int, required=True) argp.add_argument("--github-run-number", type=int, required=True) argp.add_argument("--head-branch-name", type=str, required=True) @@ -38,26 +38,45 @@ def parse_args(): def post_measurement(args): - with open(args.benchmark_results_path, "r") as f: - data = json.load(f) - timestamp = datetime.now().timestamp() - req = requests.post( - f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements", - json={ - "name": args.benchmark_name, - "timestamp": timestamp, - "git_repo": GITHUB_REPOSITORY, - "git_ref": GITHUB_REF, - "git_sha": GITHUB_SHA, - "github_run_id": args.github_run_id, - "github_run_number": args.github_run_number, - "results": data, - "git_branch": args.head_branch_name}, - timeout=1) - assert req.status_code == 200, \ - f"Uploading {args.benchmark_name} data failed." - log.info(f"{args.benchmark_name} data sent to " - f"{BENCH_GRAPH_SERVER_ENDPOINT}") + timestamp = datetime.now().timestamp() + with open(args.benchmark_results, "r") as in_memory_txn_file: + in_memory_txn_data = json.load(in_memory_txn_file) + + in_memory_analytical_data = None + if args.benchmark_results_in_memory_analytical_path is not None: + try: + with open(args.benchmark_results_in_memory_analytical_path, "r") as in_memory_analytical_file: + in_memory_analytical_data = json.load(in_memory_analytical_file) + except IOError: + log.error(f"Failed to load {args.benchmark_results_in_memory_analytical_path}.") + + on_disk_txn_data = None + if args.benchmark_results_on_disk_txn_path is not None: + try: + with open(args.benchmark_results_on_disk_txn_path, "r") as on_disk_txn_file: + on_disk_txn_data = json.load(on_disk_txn_file) + except IOError: + log.error(f"Failed to load {args.benchmark_results_on_disk_txn_path}.") + + req = requests.post( + f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements", + json={ + "name": args.benchmark_name, + "timestamp": timestamp, + "git_repo": GITHUB_REPOSITORY, + "git_ref": GITHUB_REF, + "git_sha": GITHUB_SHA, + "github_run_id": args.github_run_id, + "github_run_number": args.github_run_number, + "results": in_memory_txn_data, + "in_memory_analytical_results": in_memory_analytical_data, + "on_disk_txn_results": on_disk_txn_data, + "git_branch": args.head_branch_name, + }, + timeout=1, + ) + assert req.status_code == 200, f"Uploading {args.benchmark_name} data failed." + log.info(f"{args.benchmark_name} data sent to " f"{BENCH_GRAPH_SERVER_ENDPOINT}") if __name__ == "__main__":