diff --git a/.github/workflows/daily_benchmark.yaml b/.github/workflows/daily_benchmark.yaml index 83e2528ce..9581d5e54 100644 --- a/.github/workflows/daily_benchmark.yaml +++ b/.github/workflows/daily_benchmark.yaml @@ -67,7 +67,11 @@ jobs: - name: Run mgbench run: | cd tests/mgbench - ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/* + ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_pokec.json pokec/medium/*/* + + ./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_supernode.json supernode + + ./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_high_write_set_property.json high_write_set_property - name: Upload mgbench results run: | @@ -76,7 +80,19 @@ jobs: source ve3/bin/activate pip install -r requirements.txt ./main.py --benchmark-name "mgbench" \ - --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \ + --benchmark-results-path "../../tests/mgbench/benchmark_pokec.json" \ + --github-run-id "${{ github.run_id }}" \ + --github-run-number "${{ github.run_number }}" \ + --head-branch-name "${{ env.BRANCH_NAME }}" + + ./main.py --benchmark-name "supernode" \ + --benchmark-results-path "../../tests/mgbench/benchmark_supernode.json" \ + --github-run-id "${{ github.run_id }}" \ + --github-run-number "${{ github.run_number }}" \ + --head-branch-name "${{ env.BRANCH_NAME }}" + + ./main.py --benchmark-name "high_write_set_property" \ + --benchmark-results-path "../../tests/mgbench/benchmark_high_write_set_property.json" \ --github-run-id "${{ github.run_id }}" \ --github-run-number "${{ github.run_number }}" \ --head-branch-name "${{ env.BRANCH_NAME }}" diff --git a/include/mgp.hpp b/include/mgp.hpp index 9085f6350..a56ac23cb 100644 --- a/include/mgp.hpp +++ b/include/mgp.hpp @@ -554,6 +554,9 @@ class List { /// @exception std::runtime_error List contains value of unknown type. bool operator!=(const List &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_list *ptr_; }; @@ -669,6 +672,9 @@ class Map { /// @exception std::runtime_error Map contains value of unknown type. bool operator!=(const Map &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_map *ptr_; }; @@ -740,6 +746,9 @@ class Node { /// @exception std::runtime_error Node properties contain value(s) of unknown type. bool operator!=(const Node &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_vertex *ptr_; }; @@ -798,6 +807,9 @@ class Relationship { /// @exception std::runtime_error Relationship properties contain value(s) of unknown type. bool operator!=(const Relationship &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_edge *ptr_; }; @@ -846,6 +858,9 @@ class Path { /// @exception std::runtime_error Path contains element(s) with unknown value. bool operator!=(const Path &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_path *ptr_; }; @@ -903,6 +918,9 @@ class Date { bool operator<(const Date &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_date *ptr_; }; @@ -962,6 +980,9 @@ class LocalTime { bool operator<(const LocalTime &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_local_time *ptr_; }; @@ -1027,6 +1048,9 @@ class LocalDateTime { bool operator<(const LocalDateTime &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_local_date_time *ptr_; }; @@ -1078,6 +1102,9 @@ class Duration { bool operator<(const Duration &other) const; + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_duration *ptr_; }; @@ -1288,6 +1315,9 @@ class Value { friend std::ostream &operator<<(std::ostream &os, const mgp::Value &value); + /// @brief returns the string representation + const std::string ToString() const; + private: mgp_value *ptr_; }; @@ -2400,6 +2430,22 @@ inline bool List::operator==(const List &other) const { return util::ListsEqual( inline bool List::operator!=(const List &other) const { return !(*this == other); } +inline const std::string List::ToString() const { + const size_t size = Size(); + if (size == 0) { + return "[]"; + } + std::string return_str{"["}; + size_t i = 0; + const mgp::List &list = (*this); + while (i < size - 1) { + return_str.append(list[i].ToString() + ", "); + i++; + } + return_str.append(list[i].ToString() + "]"); + return return_str; +} + // MapItem: inline bool MapItem::operator==(MapItem &other) const { return key == other.key && value == other.value; } @@ -2569,6 +2615,24 @@ inline bool Map::operator==(const Map &other) const { return util::MapsEqual(ptr inline bool Map::operator!=(const Map &other) const { return !(*this == other); } +inline const std::string Map::ToString() const { + const size_t map_size = Size(); + if (map_size == 0) { + return "{}"; + } + std::string return_string{"{"}; + size_t i = 0; + for (const auto &[key, value] : *this) { + if (i == map_size - 1) { + return_string.append(std::string(key) + ": " + value.ToString() + "}"); + break; + } + return_string.append(std::string(key) + ": " + value.ToString() + ", "); + ++i; + } + return return_string; +} + /* #endregion */ /* #region Graph elements (Node, Relationship & Path) */ @@ -2674,6 +2738,35 @@ inline bool Node::operator==(const Node &other) const { return util::NodesEqual( inline bool Node::operator!=(const Node &other) const { return !(*this == other); } +// this functions is used both in relationship and node ToString +inline std::string PropertiesToString(const std::map &property_map) { + std::string properties{""}; + const auto map_size = property_map.size(); + size_t i = 0; + for (const auto &[key, value] : property_map) { + if (i == map_size - 1) { + properties.append(std::string(key) + ": " + value.ToString()); + break; + } + properties.append(std::string(key) + ": " + value.ToString() + ", "); + ++i; + } + return properties; +} + +inline const std::string Node::ToString() const { + std::string labels{", "}; + for (auto label : Labels()) { + labels.append(":" + std::string(label)); + } + if (labels == ", ") { + labels = ""; // dont use labels if they dont exist + } + std::map properties_map{Properties()}; + std::string properties{PropertiesToString(properties_map)}; + return "(id: " + std::to_string(Id().AsInt()) + labels + ", properties: {" + properties + "})"; +} + // Relationship: inline Relationship::Relationship(mgp_edge *ptr) : ptr_(mgp::MemHandlerCallback(edge_copy, ptr)) {} @@ -2748,6 +2841,18 @@ inline bool Relationship::operator==(const Relationship &other) const { inline bool Relationship::operator!=(const Relationship &other) const { return !(*this == other); } +inline const std::string Relationship::ToString() const { + const auto from = From(); + const auto to = To(); + + const std::string type{Type()}; + std::map properties_map{Properties()}; + std::string properties{PropertiesToString(properties_map)}; + const std::string relationship{"[type: " + type + ", id: " + std::to_string(Id().AsInt()) + ", properties: {" + + properties + "}]"}; + + return from.ToString() + "-" + relationship + "->" + to.ToString(); +} // Path: inline Path::Path(mgp_path *ptr) : ptr_(mgp::MemHandlerCallback(path_copy, ptr)) {} @@ -2810,6 +2915,26 @@ inline bool Path::operator==(const Path &other) const { return util::PathsEqual( inline bool Path::operator!=(const Path &other) const { return !(*this == other); } +inline const std::string Path::ToString() const { + const auto length = Length(); + size_t i = 0; + std::string return_string{""}; + for (i = 0; i < length; i++) { + const auto node = GetNodeAt(i); + return_string.append(node.ToString() + "-"); + + const Relationship rel = GetRelationshipAt(i); + std::map properties_map{rel.Properties()}; + std::string properties = PropertiesToString(properties_map); + return_string.append("[type: " + std::string(rel.Type()) + ", id: " + std::to_string(rel.Id().AsInt()) + + ", properties: {" + properties + "}]->"); + } + + const auto node = GetNodeAt(i); + return_string.append(node.ToString()); + return return_string; +} + /* #endregion */ /* #region Temporal types (Date, LocalTime, LocalDateTime, Duration) */ @@ -2907,6 +3032,10 @@ inline bool Date::operator<(const Date &other) const { return is_less; } +inline const std::string Date::ToString() const { + return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()); +} + // LocalTime: inline LocalTime::LocalTime(mgp_local_time *ptr) : ptr_(mgp::MemHandlerCallback(local_time_copy, ptr)) {} @@ -3006,6 +3135,11 @@ inline bool LocalTime::operator<(const LocalTime &other) const { return is_less; } +inline const std::string LocalTime::ToString() const { + return std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + + std::to_string(Millisecond()) + std::to_string(Microsecond()); +} + // LocalDateTime: inline LocalDateTime::LocalDateTime(mgp_local_date_time *ptr) @@ -3120,6 +3254,12 @@ inline bool LocalDateTime::operator<(const LocalDateTime &other) const { return is_less; } +inline const std::string LocalDateTime::ToString() const { + return std::to_string(Year()) + "-" + std::to_string(Month()) + "-" + std::to_string(Day()) + "T" + + std::to_string(Hour()) + ":" + std::to_string(Minute()) + ":" + std::to_string(Second()) + "," + + std::to_string(Millisecond()) + std::to_string(Microsecond()); +} + // Duration: inline Duration::Duration(mgp_duration *ptr) : ptr_(mgp::MemHandlerCallback(duration_copy, ptr)) {} @@ -3209,6 +3349,8 @@ inline bool Duration::operator<(const Duration &other) const { return is_less; } +inline const std::string Duration::ToString() const { return std::to_string(Microseconds()) + "ms"; } + /* #endregion */ /* #endregion */ @@ -3673,6 +3815,42 @@ inline std::ostream &operator<<(std::ostream &os, const mgp::Type &type) { } } +inline const std::string Value::ToString() const { + const mgp::Type &type = Type(); + switch (type) { + case Type::Null: + return ""; + case Type::Bool: + return ValueBool() ? "true" : "false"; + case Type::Int: + return std::to_string(ValueInt()); + case Type::Double: + return std::to_string(ValueDouble()); + case Type::String: + return std::string(ValueString()); + case Type::Node: + return ValueNode().ToString(); + case Type::Relationship: + return ValueRelationship().ToString(); + case Type::Date: + return ValueDate().ToString(); + case Type::LocalTime: + return ValueLocalTime().ToString(); + case Type::LocalDateTime: + return ValueLocalDateTime().ToString(); + case Type::Duration: + return ValueDuration().ToString(); + case Type::List: + return ValueList().ToString(); + case Type::Map: + return ValueMap().ToString(); + case Type::Path: + return ValuePath().ToString(); + default: + throw ValueException("Undefined behaviour"); + } +} + /* #endregion */ /* #region Record */ diff --git a/src/glue/communication.cpp b/src/glue/communication.cpp index f1a785888..5db3e2726 100644 --- a/src/glue/communication.cpp +++ b/src/glue/communication.cpp @@ -151,8 +151,8 @@ storage::Result ToBoltVertex(const storage::VertexA properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second); } // Introduced in Bolt v5 (for now just send the ID) - const auto element_id = std::to_string(id.AsInt()); - return communication::bolt::Vertex{id, labels, properties, element_id}; + auto element_id = std::to_string(id.AsInt()); + return communication::bolt::Vertex{id, std::move(labels), std::move(properties), std::move(element_id)}; } storage::Result ToBoltEdge(const storage::EdgeAccessor &edge, const storage::Storage &db, @@ -171,7 +171,8 @@ storage::Result ToBoltEdge(const storage::EdgeAccesso const auto element_id = std::to_string(id.AsInt()); const auto from_element_id = std::to_string(from.AsInt()); const auto to_element_id = std::to_string(to.AsInt()); - return communication::bolt::Edge{id, from, to, type, properties, element_id, from_element_id, to_element_id}; + return communication::bolt::Edge{ + id, from, to, std::move(type), std::move(properties), element_id, from_element_id, to_element_id}; } storage::Result ToBoltPath(const query::Path &path, const storage::Storage &db, diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 845259607..18f387c87 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -217,16 +217,15 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional port) override { - auto *mem_storage = static_cast(db_); if (replication_role == ReplicationQuery::ReplicationRole::MAIN) { - if (!mem_storage->SetMainReplicationRole()) { + if (!db_->SetMainReplicationRole()) { throw QueryRuntimeException("Couldn't set role to main!"); } } else { if (!port || *port < 0 || *port > std::numeric_limits::max()) { throw QueryRuntimeException("Port number invalid!"); } - if (!mem_storage->SetReplicaRole( + if (!db_->SetReplicaRole( io::network::Endpoint(storage::replication::kDefaultReplicationServerIp, static_cast(*port)), storage::replication::ReplicationServerConfig{})) { throw QueryRuntimeException("Couldn't set role to replica!"); @@ -236,7 +235,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. ReplicationQuery::ReplicationRole ShowReplicationRole() const override { - switch (static_cast(db_)->GetReplicationRole()) { + switch (db_->GetReplicationRole()) { case storage::replication::ReplicationRole::MAIN: return ReplicationQuery::ReplicationRole::MAIN; case storage::replication::ReplicationRole::REPLICA: @@ -249,8 +248,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { void RegisterReplica(const std::string &name, const std::string &socket_address, const ReplicationQuery::SyncMode sync_mode, const std::chrono::seconds replica_check_frequency) override { - auto *mem_storage = static_cast(db_); - if (mem_storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't register another replica throw QueryRuntimeException("Replica can't register another replica!"); } @@ -275,9 +273,9 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { io::network::Endpoint::ParseSocketOrIpAddress(socket_address, storage::replication::kDefaultReplicationPort); if (maybe_ip_and_port) { auto [ip, port] = *maybe_ip_and_port; - auto ret = mem_storage->RegisterReplica( - name, {std::move(ip), port}, repl_mode, storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID, - {.replica_check_frequency = replica_check_frequency, .ssl = std::nullopt}); + auto ret = db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, + storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID, + {.replica_check_frequency = replica_check_frequency, .ssl = std::nullopt}); if (ret.HasError()) { throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name)); } @@ -288,26 +286,23 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. void DropReplica(const std::string &replica_name) override { - auto *mem_storage = static_cast(db_); - - if (mem_storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't unregister a replica throw QueryRuntimeException("Replica can't unregister a replica!"); } - if (!mem_storage->UnregisterReplica(replica_name)) { + if (!db_->UnregisterReplica(replica_name)) { throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name)); } } using Replica = ReplicationQueryHandler::Replica; std::vector ShowReplicas() const override { - auto *mem_storage = static_cast(db_); - if (mem_storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't show registered replicas (it shouldn't have any) throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!"); } - auto repl_infos = mem_storage->ReplicasInfo(); + auto repl_infos = db_->ReplicasInfo(); std::vector replicas; replicas.reserve(repl_infos.size()); @@ -1333,8 +1328,7 @@ bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage, const query::plan::ReadWriteTypeChecker::RWType query_type) { if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL || storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) { - auto *mem_storage = static_cast(storage); - return (mem_storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) && + return (storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) && (query_type == RWType::W || query_type == RWType::RW); } return false; @@ -1343,8 +1337,7 @@ bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage, storage::replication::ReplicationRole GetReplicaRole(storage::Storage *storage) { if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL || storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) { - auto *mem_storage = static_cast(storage); - return mem_storage->GetReplicationRole(); + return storage->GetReplicationRole(); } return storage::replication::ReplicationRole::MAIN; } diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index 3aa427ccf..d3bdc5abf 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -5,6 +5,7 @@ find_package(Threads REQUIRED) add_library(mg-storage-v2 STATIC commit_log.cpp constraints/existence_constraints.cpp + constraints/constraints.cpp temporal.cpp durability/durability.cpp durability/serialization.cpp diff --git a/src/storage/v2/constraints/constraints.cpp b/src/storage/v2/constraints/constraints.cpp new file mode 100644 index 000000000..42128511c --- /dev/null +++ b/src/storage/v2/constraints/constraints.cpp @@ -0,0 +1,32 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "storage/v2/constraints/constraints.hpp" +#include "storage/v2/disk/unique_constraints.hpp" +#include "storage/v2/inmemory/unique_constraints.hpp" + +namespace memgraph::storage { + +Constraints::Constraints(const Config &config, StorageMode storage_mode) { + std::invoke([this, config, storage_mode]() { + existence_constraints_ = std::make_unique(); + switch (storage_mode) { + case StorageMode::IN_MEMORY_TRANSACTIONAL: + case StorageMode::IN_MEMORY_ANALYTICAL: + unique_constraints_ = std::make_unique(); + break; + case StorageMode::ON_DISK_TRANSACTIONAL: + unique_constraints_ = std::make_unique(config); + break; + }; + }); +} +} // namespace memgraph::storage diff --git a/src/storage/v2/constraints/constraints.hpp b/src/storage/v2/constraints/constraints.hpp index 665c5518b..8469a5470 100644 --- a/src/storage/v2/constraints/constraints.hpp +++ b/src/storage/v2/constraints/constraints.hpp @@ -13,27 +13,13 @@ #include "storage/v2/config.hpp" #include "storage/v2/constraints/existence_constraints.hpp" -#include "storage/v2/disk/unique_constraints.hpp" -#include "storage/v2/inmemory/unique_constraints.hpp" +#include "storage/v2/constraints/unique_constraints.hpp" #include "storage/v2/storage_mode.hpp" namespace memgraph::storage { struct Constraints { - Constraints(const Config &config, StorageMode storage_mode) { - std::invoke([this, config, storage_mode]() { - existence_constraints_ = std::make_unique(); - switch (storage_mode) { - case StorageMode::IN_MEMORY_TRANSACTIONAL: - case StorageMode::IN_MEMORY_ANALYTICAL: - unique_constraints_ = std::make_unique(); - break; - case StorageMode::ON_DISK_TRANSACTIONAL: - unique_constraints_ = std::make_unique(config); - break; - }; - }); - } + Constraints(const Config &config, StorageMode storage_mode); Constraints(const Constraints &) = delete; Constraints(Constraints &&) = delete; diff --git a/src/storage/v2/disk/label_property_index.cpp b/src/storage/v2/disk/label_property_index.cpp index 450000034..6c9d25fe3 100644 --- a/src/storage/v2/disk/label_property_index.cpp +++ b/src/storage/v2/disk/label_property_index.cpp @@ -12,13 +12,8 @@ /// TODO: clear dependencies #include "storage/v2/disk/label_property_index.hpp" -#include "storage/v2/id_types.hpp" -#include "storage/v2/inmemory/indices_utils.hpp" -#include "storage/v2/property_value.hpp" #include "utils/disk_utils.hpp" -#include "utils/exceptions.hpp" -#include "utils/file.hpp" -#include "utils/skip_list.hpp" +#include "utils/rocksdb_serialization.hpp" namespace memgraph::storage { diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index e5d9d4517..5da3eb2ba 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -28,6 +28,8 @@ #include "kvstore/kvstore.hpp" #include "spdlog/spdlog.h" #include "storage/v2/constraints/unique_constraints.hpp" +#include "storage/v2/disk/label_index.hpp" +#include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/disk/rocksdb_storage.hpp" #include "storage/v2/disk/storage.hpp" #include "storage/v2/disk/unique_constraints.hpp" diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index 3522750d0..d80eac1fe 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -27,7 +27,9 @@ #include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/wal.hpp" -#include "storage/v2/indices/label_property_index.hpp" +#include "storage/v2/inmemory/label_index.hpp" +#include "storage/v2/inmemory/label_property_index.hpp" +#include "storage/v2/inmemory/unique_constraints.hpp" #include "utils/event_histogram.hpp" #include "utils/logging.hpp" #include "utils/memory_tracker.hpp" diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index 7e9ff0289..7df4dbebb 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -1377,7 +1377,7 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps bool is_visible = true; Delta *delta = nullptr; { - std::lock_guard guard(edge.lock); + auto guard = std::shared_lock{edge.lock}; is_visible = !edge.deleted; delta = edge.delta; } diff --git a/src/storage/v2/durability/storage_global_operation.hpp b/src/storage/v2/durability/storage_global_operation.hpp new file mode 100644 index 000000000..f130b2a62 --- /dev/null +++ b/src/storage/v2/durability/storage_global_operation.hpp @@ -0,0 +1,28 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +namespace memgraph::storage::durability { + +/// Enum used to indicate a global database operation that isn't transactional. +enum class StorageGlobalOperation { + LABEL_INDEX_CREATE, + LABEL_INDEX_DROP, + LABEL_PROPERTY_INDEX_CREATE, + LABEL_PROPERTY_INDEX_DROP, + EXISTENCE_CONSTRAINT_CREATE, + EXISTENCE_CONSTRAINT_DROP, + UNIQUE_CONSTRAINT_CREATE, + UNIQUE_CONSTRAINT_DROP, +}; + +} // namespace memgraph::storage::durability diff --git a/src/storage/v2/durability/wal.cpp b/src/storage/v2/durability/wal.cpp index 570520bbd..ffecf1869 100644 --- a/src/storage/v2/durability/wal.cpp +++ b/src/storage/v2/durability/wal.cpp @@ -490,7 +490,7 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, Config::Ite // actions. encoder->WriteMarker(Marker::SECTION_DELTA); encoder->WriteUint(timestamp); - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; switch (delta.action) { case Delta::Action::DELETE_DESERIALIZED_OBJECT: case Delta::Action::DELETE_OBJECT: @@ -546,7 +546,7 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, const Delta // actions. encoder->WriteMarker(Marker::SECTION_DELTA); encoder->WriteUint(timestamp); - std::lock_guard guard(edge.lock); + auto guard = std::shared_lock{edge.lock}; switch (delta.action) { case Delta::Action::SET_PROPERTY: { encoder->WriteMarker(Marker::DELTA_EDGE_SET_PROPERTY); diff --git a/src/storage/v2/durability/wal.hpp b/src/storage/v2/durability/wal.hpp index ffd12fdb3..757ffc8a2 100644 --- a/src/storage/v2/durability/wal.hpp +++ b/src/storage/v2/durability/wal.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -20,6 +20,7 @@ #include "storage/v2/delta.hpp" #include "storage/v2/durability/metadata.hpp" #include "storage/v2/durability/serialization.hpp" +#include "storage/v2/durability/storage_global_operation.hpp" #include "storage/v2/edge.hpp" #include "storage/v2/id_types.hpp" #include "storage/v2/name_id_mapper.hpp" @@ -107,18 +108,6 @@ struct WalDeltaData { bool operator==(const WalDeltaData &a, const WalDeltaData &b); bool operator!=(const WalDeltaData &a, const WalDeltaData &b); -/// Enum used to indicate a global database operation that isn't transactional. -enum class StorageGlobalOperation { - LABEL_INDEX_CREATE, - LABEL_INDEX_DROP, - LABEL_PROPERTY_INDEX_CREATE, - LABEL_PROPERTY_INDEX_DROP, - EXISTENCE_CONSTRAINT_CREATE, - EXISTENCE_CONSTRAINT_DROP, - UNIQUE_CONSTRAINT_CREATE, - UNIQUE_CONSTRAINT_DROP, -}; - constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) { switch (type) { // These delta actions are all found inside transactions so they don't diff --git a/src/storage/v2/edge.hpp b/src/storage/v2/edge.hpp index 89bbc83e5..bdb224dfb 100644 --- a/src/storage/v2/edge.hpp +++ b/src/storage/v2/edge.hpp @@ -17,7 +17,7 @@ #include "storage/v2/id_types.hpp" #include "storage/v2/property_store.hpp" #include "utils/logging.hpp" -#include "utils/spin_lock.hpp" +#include "utils/rw_spin_lock.hpp" namespace memgraph::storage { @@ -34,7 +34,7 @@ struct Edge { PropertyStore properties; - mutable utils::SpinLock lock; + mutable utils::RWSpinLock lock; bool deleted; // uint8_t PAD; // uint16_t PAD; diff --git a/src/storage/v2/edge_accessor.cpp b/src/storage/v2/edge_accessor.cpp index 8c7e09c2a..c039441f2 100644 --- a/src/storage/v2/edge_accessor.cpp +++ b/src/storage/v2/edge_accessor.cpp @@ -32,7 +32,7 @@ bool EdgeAccessor::IsVisible(const View view) const { if (!config_.properties_on_edges) { Delta *delta = nullptr; { - std::lock_guard guard(from_vertex_->lock); + auto guard = std::shared_lock{from_vertex_->lock}; // Initialize deleted by checking if out edges contain edge_ deleted = std::find_if(from_vertex_->out_edges.begin(), from_vertex_->out_edges.end(), [&](const auto &out_edge) { return std::get<2>(out_edge) == edge_; @@ -69,7 +69,7 @@ bool EdgeAccessor::IsVisible(const View view) const { Delta *delta = nullptr; { - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::shared_lock{edge_.ptr->lock}; deleted = edge_.ptr->deleted; delta = edge_.ptr->delta; } @@ -118,7 +118,7 @@ Result EdgeAccessor::SetProperty(PropertyId property, co utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED; - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::unique_lock{edge_.ptr->lock}; if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR; @@ -142,7 +142,7 @@ Result EdgeAccessor::InitProperties(const std::map guard(edge_.ptr->lock); + auto guard = std::unique_lock{edge_.ptr->lock}; if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR; @@ -161,7 +161,7 @@ Result>> EdgeAc utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED; - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::unique_lock{edge_.ptr->lock}; if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR; @@ -179,7 +179,7 @@ Result>> EdgeAc Result> EdgeAccessor::ClearProperties() { if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED; - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::unique_lock{edge_.ptr->lock}; if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR; @@ -202,7 +202,7 @@ Result EdgeAccessor::GetProperty(PropertyId property, View view) PropertyValue value; Delta *delta = nullptr; { - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::shared_lock{edge_.ptr->lock}; deleted = edge_.ptr->deleted; value = edge_.ptr->properties.GetProperty(property); delta = edge_.ptr->delta; @@ -245,7 +245,7 @@ Result> EdgeAccessor::Properties(View view) std::map properties; Delta *delta = nullptr; { - std::lock_guard guard(edge_.ptr->lock); + auto guard = std::shared_lock{edge_.ptr->lock}; deleted = edge_.ptr->deleted; properties = edge_.ptr->properties.Properties(); delta = edge_.ptr->delta; diff --git a/src/storage/v2/indices/indices.cpp b/src/storage/v2/indices/indices.cpp index 59915ab18..44741463d 100644 --- a/src/storage/v2/indices/indices.cpp +++ b/src/storage/v2/indices/indices.cpp @@ -10,7 +10,10 @@ // licenses/APL.txt. #include "storage/v2/indices/indices.hpp" +#include "storage/v2/disk/label_index.hpp" +#include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/inmemory/label_index.hpp" +#include "storage/v2/inmemory/label_property_index.hpp" namespace memgraph::storage { @@ -35,4 +38,16 @@ void Indices::UpdateOnSetProperty(PropertyId property, const PropertyValue &valu label_property_index_->UpdateOnSetProperty(property, value, vertex, tx); } +Indices::Indices(Constraints *constraints, const Config &config, StorageMode storage_mode) { + std::invoke([this, constraints, config, storage_mode]() { + if (storage_mode == StorageMode::IN_MEMORY_TRANSACTIONAL || storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) { + label_index_ = std::make_unique(this, constraints, config); + label_property_index_ = std::make_unique(this, constraints, config); + } else { + label_index_ = std::make_unique(this, constraints, config); + label_property_index_ = std::make_unique(this, constraints, config); + } + }); +} + } // namespace memgraph::storage diff --git a/src/storage/v2/indices/indices.hpp b/src/storage/v2/indices/indices.hpp index 06d0837c4..c03ab96f4 100644 --- a/src/storage/v2/indices/indices.hpp +++ b/src/storage/v2/indices/indices.hpp @@ -12,28 +12,14 @@ #pragma once #include -#include "storage/v2/disk/label_index.hpp" -#include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/indices/label_index.hpp" #include "storage/v2/indices/label_property_index.hpp" -#include "storage/v2/inmemory/label_index.hpp" -#include "storage/v2/inmemory/label_property_index.hpp" #include "storage/v2/storage_mode.hpp" namespace memgraph::storage { struct Indices { - Indices(Constraints *constraints, const Config &config, StorageMode storage_mode) { - std::invoke([this, constraints, config, storage_mode]() { - if (storage_mode == StorageMode::IN_MEMORY_TRANSACTIONAL || storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) { - label_index_ = std::make_unique(this, constraints, config); - label_property_index_ = std::make_unique(this, constraints, config); - } else { - label_index_ = std::make_unique(this, constraints, config); - label_property_index_ = std::make_unique(this, constraints, config); - } - }); - } + Indices(Constraints *constraints, const Config &config, StorageMode storage_mode); Indices(const Indices &) = delete; Indices(Indices &&) = delete; diff --git a/src/storage/v2/inmemory/indices_utils.hpp b/src/storage/v2/indices/indices_utils.hpp similarity index 98% rename from src/storage/v2/inmemory/indices_utils.hpp rename to src/storage/v2/indices/indices_utils.hpp index 9914e6ac5..1fa5cab87 100644 --- a/src/storage/v2/inmemory/indices_utils.hpp +++ b/src/storage/v2/indices/indices_utils.hpp @@ -51,7 +51,7 @@ inline bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t tim bool deleted{false}; const Delta *delta = nullptr; { - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; has_label = utils::Contains(vertex.labels, label); deleted = vertex.deleted; delta = vertex.delta; @@ -105,7 +105,7 @@ inline bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, Prop bool deleted{false}; const Delta *delta = nullptr; { - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; has_label = utils::Contains(vertex.labels, label); current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value); deleted = vertex.deleted; @@ -168,7 +168,7 @@ inline bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, bool current_value_equal_to_value = value.IsNull(); const Delta *delta = nullptr; { - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; deleted = vertex.deleted; has_label = utils::Contains(vertex.labels, label); current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value); diff --git a/src/storage/v2/inmemory/label_index.cpp b/src/storage/v2/inmemory/label_index.cpp index 7e794d859..d8279f555 100644 --- a/src/storage/v2/inmemory/label_index.cpp +++ b/src/storage/v2/inmemory/label_index.cpp @@ -10,7 +10,7 @@ // licenses/APL.txt. #include "storage/v2/inmemory/label_index.hpp" -#include "storage/v2/inmemory/indices_utils.hpp" +#include "storage/v2/indices/indices_utils.hpp" namespace memgraph::storage { diff --git a/src/storage/v2/inmemory/label_property_index.cpp b/src/storage/v2/inmemory/label_property_index.cpp index 9823cc580..11ae7c2d7 100644 --- a/src/storage/v2/inmemory/label_property_index.cpp +++ b/src/storage/v2/inmemory/label_property_index.cpp @@ -10,7 +10,7 @@ // licenses/APL.txt. #include "storage/v2/inmemory/label_property_index.hpp" -#include "storage/v2/inmemory/indices_utils.hpp" +#include "storage/v2/indices/indices_utils.hpp" namespace memgraph::storage { diff --git a/src/storage/v2/inmemory/replication/replication_client.cpp b/src/storage/v2/inmemory/replication/replication_client.cpp index e740b67ac..fc368d31c 100644 --- a/src/storage/v2/inmemory/replication/replication_client.cpp +++ b/src/storage/v2/inmemory/replication/replication_client.cpp @@ -12,6 +12,7 @@ #include "storage/v2/inmemory/replication/replication_client.hpp" #include "storage/v2/durability/durability.hpp" +#include "storage/v2/inmemory/storage.hpp" namespace memgraph::storage { @@ -67,212 +68,50 @@ void CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buf replication::CurrentWalRes CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); } +////// ReplicationClient Helpers ////// + +replication::WalFilesRes TransferWalFiles(rpc::Client &client, const std::vector &wal_files) { + MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); + auto stream = client.Stream(wal_files.size()); + replication::Encoder encoder(stream.GetBuilder()); + for (const auto &wal : wal_files) { + spdlog::debug("Sending wal file: {}", wal); + encoder.WriteFile(wal); + } + return stream.AwaitResponse(); +} + +replication::SnapshotRes TransferSnapshot(rpc::Client &client, const std::filesystem::path &path) { + auto stream = client.Stream(); + replication::Encoder encoder(stream.GetBuilder()); + encoder.WriteFile(path); + return stream.AwaitResponse(); +} + +uint64_t ReplicateCurrentWal(CurrentWalHandler &stream, durability::WalFile const &wal_file) { + stream.AppendFilename(wal_file.Path().filename()); + utils::InputFile file; + MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file!"); + const auto [buffer, buffer_size] = wal_file.CurrentFileBuffer(); + stream.AppendSize(file.GetSize() + buffer_size); + stream.AppendFileData(&file); + stream.AppendBufferData(buffer, buffer_size); + auto response = stream.Finalize(); + return response.current_commit_timestamp; +} + ////// ReplicationClient ////// + InMemoryReplicationClient::InMemoryReplicationClient(InMemoryStorage *storage, std::string name, io::network::Endpoint endpoint, replication::ReplicationMode mode, const replication::ReplicationClientConfig &config) - : ReplicationClient{std::move(name), std::move(endpoint), mode, config}, storage_{storage} {} + : ReplicationClient{storage, std::move(name), std::move(endpoint), mode, config} {} -void InMemoryReplicationClient::TryInitializeClientAsync() { - thread_pool_.AddTask([this] { - rpc_client_.Abort(); - this->TryInitializeClientSync(); - }); -} -void InMemoryReplicationClient::FrequentCheck() { - const auto is_success = std::invoke([this]() { - try { - auto stream{rpc_client_.Stream()}; - const auto response = stream.AwaitResponse(); - return response.success; - } catch (const rpc::RpcFailedException &) { - return false; - } - }); - // States: READY, REPLICATING, RECOVERY, INVALID - // If success && ready, replicating, recovery -> stay the same because something good is going on. - // If success && INVALID -> [it's possible that replica came back to life] -> TryInitializeClient. - // If fail -> [replica is not reachable at all] -> INVALID state. - // NOTE: TryInitializeClient might return nothing if there is a branching point. - // NOTE: The early return pattern simplified the code, but the behavior should be as explained. - if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - return; - } - if (replica_state_.load() == replication::ReplicaState::INVALID) { - TryInitializeClientAsync(); - } -} /// @throws rpc::RpcFailedException - -void InMemoryReplicationClient::Start() { - auto const &endpoint = rpc_client_.Endpoint(); - spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); - - TryInitializeClientSync(); - - // Help the user to get the most accurate replica state possible. - if (replica_check_frequency_ > std::chrono::seconds(0)) { - replica_checker_.Run("Replica Checker", replica_check_frequency_, [this] { FrequentCheck(); }); - } -} - -void InMemoryReplicationClient::InitializeClient() { - uint64_t current_commit_timestamp{kTimestampInitialId}; - - const auto &main_epoch = storage_->replication_state_.GetEpoch(); - - auto stream{rpc_client_.Stream(storage_->replication_state_.last_commit_timestamp_, - main_epoch.id)}; - - const auto replica = stream.AwaitResponse(); - std::optional branching_point; - if (replica.epoch_id != main_epoch.id && replica.current_commit_timestamp != kTimestampInitialId) { - auto const &history = storage_->replication_state_.history; - const auto epoch_info_iter = std::find_if(history.crbegin(), history.crend(), [&](const auto &main_epoch_info) { - return main_epoch_info.first == replica.epoch_id; - }); - if (epoch_info_iter == history.crend()) { - branching_point = 0; - } else if (epoch_info_iter->second != replica.current_commit_timestamp) { - branching_point = epoch_info_iter->second; - } - } - if (branching_point) { - spdlog::error( - "You cannot register Replica {} to this Main because at one point " - "Replica {} acted as the Main instance. Both the Main and Replica {} " - "now hold unique data. Please resolve data conflicts and start the " - "replication on a clean instance.", - name_, name_, name_); - return; - } - - current_commit_timestamp = replica.current_commit_timestamp; - spdlog::trace("Current timestamp on replica {}: {}", name_, current_commit_timestamp); - spdlog::trace("Current timestamp on main: {}", storage_->replication_state_.last_commit_timestamp_.load()); - if (current_commit_timestamp == storage_->replication_state_.last_commit_timestamp_.load()) { - spdlog::debug("Replica '{}' up to date", name_); - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::READY); - } else { - spdlog::debug("Replica '{}' is behind", name_); - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::RECOVERY); - } - thread_pool_.AddTask([=, this] { this->RecoverReplica(current_commit_timestamp); }); - } -} -void InMemoryReplicationClient::TryInitializeClientSync() { - try { - InitializeClient(); - } catch (const rpc::RpcFailedException &) { - std::unique_lock client_guarde{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_, - rpc_client_.Endpoint(), "https://memgr.ph/replication")); - } -} -void InMemoryReplicationClient::HandleRpcFailure() { - spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication")); - TryInitializeClientAsync(); -} -void InMemoryReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) { - std::unique_lock guard(client_lock_); - const auto status = replica_state_.load(); - switch (status) { - case replication::ReplicaState::RECOVERY: - spdlog::debug("Replica {} is behind MAIN instance", name_); - return; - case replication::ReplicaState::REPLICATING: - spdlog::debug("Replica {} missed a transaction", name_); - // We missed a transaction because we're still replicating - // the previous transaction so we need to go to RECOVERY - // state to catch up with the missing transaction - // We cannot queue the recovery process here because - // an error can happen while we're replicating the previous - // transaction after which the client should go to - // INVALID state before starting the recovery process - replica_state_.store(replication::ReplicaState::RECOVERY); - return; - case replication::ReplicaState::INVALID: - HandleRpcFailure(); - return; - case replication::ReplicaState::READY: - MG_ASSERT(!replica_stream_); - try { - replica_stream_.emplace( - ReplicaStream{this, storage_->replication_state_.last_commit_timestamp_.load(), current_wal_seq_num}); - replica_state_.store(replication::ReplicaState::REPLICATING); - } catch (const rpc::RpcFailedException &) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); - } - return; - } -} -void InMemoryReplicationClient::IfStreamingTransaction(const std::function &callback) { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return; - } - - try { - callback(*replica_stream_); - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard{client_lock_}; - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - } -} -bool InMemoryReplicationClient::FinalizeTransactionReplication() { - // We can only check the state because it guarantees to be only - // valid during a single transaction replication (if the assumption - // that this and other transaction replication functions can only be - // called from a one thread stands) - if (replica_state_ != replication::ReplicaState::REPLICATING) { - return false; - } - - if (mode_ == replication::ReplicationMode::ASYNC) { - thread_pool_.AddTask([this] { static_cast(this->FinalizeTransactionReplicationInternal()); }); - return true; - } - - return FinalizeTransactionReplicationInternal(); -} -bool InMemoryReplicationClient::FinalizeTransactionReplicationInternal() { - MG_ASSERT(replica_stream_, "Missing stream for transaction deltas"); - try { - auto response = replica_stream_->Finalize(); - replica_stream_.reset(); - std::unique_lock client_guard(client_lock_); - if (!response.success || replica_state_ == replication::ReplicaState::RECOVERY) { - replica_state_.store(replication::ReplicaState::RECOVERY); - thread_pool_.AddTask([&, this] { this->RecoverReplica(response.current_commit_timestamp); }); - } else { - replica_state_.store(replication::ReplicaState::READY); - return true; - } - } catch (const rpc::RpcFailedException &) { - replica_stream_.reset(); - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); - } - return false; -} void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { spdlog::debug("Starting replica recover"); + auto *storage = static_cast(storage_); while (true) { - auto file_locker = storage_->file_retainer_.AddLocker(); + auto file_locker = storage->file_retainer_.AddLocker(); const auto steps = GetRecoverySteps(replica_commit, &file_locker); int i = 0; @@ -284,21 +123,22 @@ void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { using StepType = std::remove_cvref_t; if constexpr (std::is_same_v) { spdlog::debug("Sending the latest snapshot file: {}", arg); - auto response = TransferSnapshot(arg); + auto response = TransferSnapshot(rpc_client_, arg); replica_commit = response.current_commit_timestamp; } else if constexpr (std::is_same_v) { spdlog::debug("Sending the latest wal files"); - auto response = TransferWalFiles(arg); + auto response = TransferWalFiles(rpc_client_, arg); replica_commit = response.current_commit_timestamp; spdlog::debug("Wal files successfully transferred."); } else if constexpr (std::is_same_v) { - std::unique_lock transaction_guard(storage_->engine_lock_); - if (storage_->wal_file_ && storage_->wal_file_->SequenceNumber() == arg.current_wal_seq_num) { - storage_->wal_file_->DisableFlushing(); + std::unique_lock transaction_guard(storage->engine_lock_); + if (storage->wal_file_ && storage->wal_file_->SequenceNumber() == arg.current_wal_seq_num) { + storage->wal_file_->DisableFlushing(); transaction_guard.unlock(); spdlog::debug("Sending current wal file"); - replica_commit = ReplicateCurrentWal(); - storage_->wal_file_->EnableFlushing(); + auto streamHandler = CurrentWalHandler{this}; + replica_commit = ReplicateCurrentWal(streamHandler, *storage->wal_file_); + storage->wal_file_->EnableFlushing(); } else { spdlog::debug("Cannot recover using current wal file"); } @@ -328,27 +168,17 @@ void InMemoryReplicationClient::RecoverReplica(uint64_t replica_commit) { // and we will go to recovery. // By adding this lock, we can avoid that, and go to RECOVERY immediately. std::unique_lock client_guard{client_lock_}; + const auto last_commit_timestamp = LastCommitTimestamp(); SPDLOG_INFO("Replica timestamp: {}", replica_commit); - SPDLOG_INFO("Last commit: {}", storage_->replication_state_.last_commit_timestamp_); - if (storage_->replication_state_.last_commit_timestamp_.load() == replica_commit) { + SPDLOG_INFO("Last commit: {}", last_commit_timestamp); + if (last_commit_timestamp == replica_commit) { replica_state_.store(replication::ReplicaState::READY); return; } } } -uint64_t InMemoryReplicationClient::ReplicateCurrentWal() { - const auto &wal_file = storage_->wal_file_; - auto stream = CurrentWalHandler{this}; - stream.AppendFilename(wal_file->Path().filename()); - utils::InputFile file; - MG_ASSERT(file.Open(storage_->wal_file_->Path()), "Failed to open current WAL file!"); - const auto [buffer, buffer_size] = wal_file->CurrentFileBuffer(); - stream.AppendSize(file.GetSize() + buffer_size); - stream.AppendFileData(&file); - stream.AppendBufferData(buffer, buffer_size); - auto response = stream.Finalize(); - return response.current_commit_timestamp; -} /// This method tries to find the optimal path for recoverying a single replica. + +/// This method tries to find the optimal path for recoverying a single replica. /// Based on the last commit transfered to replica it tries to update the /// replica using durability files - WALs and Snapshots. WAL files are much /// smaller in size as they contain only the Deltas (changes) made during the @@ -375,16 +205,17 @@ std::vector InMemoryReplicationClient:: // This lock is also necessary to force the missed transaction to finish. std::optional current_wal_seq_num; std::optional current_wal_from_timestamp; - if (std::unique_lock transtacion_guard(storage_->engine_lock_); storage_->wal_file_) { - current_wal_seq_num.emplace(storage_->wal_file_->SequenceNumber()); - current_wal_from_timestamp.emplace(storage_->wal_file_->FromTimestamp()); + auto *storage = static_cast(storage_); + if (std::unique_lock transtacion_guard(storage->engine_lock_); storage->wal_file_) { + current_wal_seq_num.emplace(storage->wal_file_->SequenceNumber()); + current_wal_from_timestamp.emplace(storage->wal_file_->FromTimestamp()); } auto locker_acc = file_locker->Access(); - auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_, current_wal_seq_num); + auto wal_files = durability::GetWalFiles(storage->wal_directory_, storage->uuid_, current_wal_seq_num); MG_ASSERT(wal_files, "Wal files could not be loaded"); - auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_); + auto snapshot_files = durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_); std::optional latest_snapshot; if (!snapshot_files.empty()) { std::sort(snapshot_files.begin(), snapshot_files.end()); @@ -513,54 +344,5 @@ std::vector InMemoryReplicationClient:: return recovery_steps; } -TimestampInfo InMemoryReplicationClient::GetTimestampInfo() { - TimestampInfo info; - info.current_timestamp_of_replica = 0; - info.current_number_of_timestamp_behind_master = 0; - - try { - auto stream{rpc_client_.Stream()}; - const auto response = stream.AwaitResponse(); - const auto is_success = response.success; - if (!is_success) { - replica_state_.store(replication::ReplicaState::INVALID); - HandleRpcFailure(); - } - auto main_time_stamp = storage_->replication_state_.last_commit_timestamp_.load(); - info.current_timestamp_of_replica = response.current_commit_timestamp; - info.current_number_of_timestamp_behind_master = response.current_commit_timestamp - main_time_stamp; - } catch (const rpc::RpcFailedException &) { - { - std::unique_lock client_guard(client_lock_); - replica_state_.store(replication::ReplicaState::INVALID); - } - HandleRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably won't - // block - } - - return info; -} -std::string const &InMemoryReplicationClient::GetEpochId() const { return storage_->replication_state_.GetEpoch().id; } -auto InMemoryReplicationClient::GetStorage() -> Storage * { return storage_; } - -replication::SnapshotRes InMemoryReplicationClient::TransferSnapshot(const std::filesystem::path &path) { - auto stream{rpc_client_.Stream()}; - replication::Encoder encoder(stream.GetBuilder()); - encoder.WriteFile(path); - return stream.AwaitResponse(); -} - -replication::WalFilesRes InMemoryReplicationClient::TransferWalFiles( - const std::vector &wal_files) { - MG_ASSERT(!wal_files.empty(), "Wal files list is empty!"); - auto stream{rpc_client_.Stream(wal_files.size())}; - replication::Encoder encoder(stream.GetBuilder()); - for (const auto &wal : wal_files) { - spdlog::debug("Sending wal file: {}", wal); - encoder.WriteFile(wal); - } - - return stream.AwaitResponse(); -} } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_client.hpp b/src/storage/v2/inmemory/replication/replication_client.hpp index e278c7c3f..de939835b 100644 --- a/src/storage/v2/inmemory/replication/replication_client.hpp +++ b/src/storage/v2/inmemory/replication/replication_client.hpp @@ -10,42 +10,21 @@ // licenses/APL.txt. #pragma once -#include "storage/v2/inmemory/storage.hpp" #include "storage/v2/replication/replication_client.hpp" + namespace memgraph::storage { +class InMemoryStorage; + class InMemoryReplicationClient : public ReplicationClient { public: InMemoryReplicationClient(InMemoryStorage *storage, std::string name, io::network::Endpoint endpoint, replication::ReplicationMode mode, const replication::ReplicationClientConfig &config = {}); - void StartTransactionReplication(uint64_t current_wal_seq_num) override; - // Replication clients can be removed at any point - // so to avoid any complexity of checking if the client was removed whenever - // we want to send part of transaction and to avoid adding some GC logic this - // function will run a callback if, after previously callling - // StartTransactionReplication, stream is created. - void IfStreamingTransaction(const std::function &callback) override; - auto GetEpochId() const -> std::string const & override; - - auto GetStorage() -> Storage * override; - - void Start() override; - - // Return whether the transaction could be finalized on the replication client or not. - [[nodiscard]] bool FinalizeTransactionReplication() override; - TimestampInfo GetTimestampInfo() override; - - private: - void TryInitializeClientAsync(); - void FrequentCheck(); - void InitializeClient(); - void TryInitializeClientSync(); - void HandleRpcFailure(); - [[nodiscard]] bool FinalizeTransactionReplicationInternal(); - void RecoverReplica(uint64_t replica_commit); - uint64_t ReplicateCurrentWal(); + protected: + void RecoverReplica(uint64_t replica_commit) override; + // TODO: move the GetRecoverySteps stuff below as an internal detail using RecoverySnapshot = std::filesystem::path; using RecoveryWals = std::vector; struct RecoveryCurrentWal { @@ -54,15 +33,6 @@ class InMemoryReplicationClient : public ReplicationClient { }; using RecoveryStep = std::variant; std::vector GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker); - - // Transfer the snapshot file. - // @param path Path of the snapshot file. - replication::SnapshotRes TransferSnapshot(const std::filesystem::path &path); - - // Transfer the WAL files - replication::WalFilesRes TransferWalFiles(const std::vector &wal_files); - - InMemoryStorage *storage_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/replication/replication_server.cpp b/src/storage/v2/inmemory/replication/replication_server.cpp index 060c577d6..c7c049997 100644 --- a/src/storage/v2/inmemory/replication/replication_server.cpp +++ b/src/storage/v2/inmemory/replication/replication_server.cpp @@ -14,6 +14,7 @@ #include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/version.hpp" #include "storage/v2/inmemory/storage.hpp" +#include "storage/v2/inmemory/unique_constraints.hpp" namespace memgraph::storage { namespace { @@ -130,7 +131,7 @@ void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Bu MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!"); spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path); - std::unique_lock storage_guard(storage_->main_lock_); + auto storage_guard = std::unique_lock{storage_->main_lock_}; spdlog::trace("Clearing database since recovering from snapshot."); // Clear the database storage_->vertices_.clear(); @@ -415,7 +416,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, bool is_visible = true; Delta *delta = nullptr; { - std::lock_guard guard(edge->lock); + auto guard = std::shared_lock{edge->lock}; is_visible = !edge->deleted; delta = edge->delta; } diff --git a/src/storage/v2/inmemory/storage.cpp b/src/storage/v2/inmemory/storage.cpp index 232668825..acb13d829 100644 --- a/src/storage/v2/inmemory/storage.cpp +++ b/src/storage/v2/inmemory/storage.cpp @@ -10,19 +10,13 @@ // licenses/APL.txt. #include "storage/v2/inmemory/storage.hpp" -#include "storage/v2/constraints/constraints.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" -#include "storage/v2/durability/wal.hpp" -#include "storage/v2/edge_accessor.hpp" -#include "storage/v2/edge_direction.hpp" -#include "storage/v2/storage_mode.hpp" -#include "storage/v2/vertex_accessor.hpp" -#include "utils/stat.hpp" /// REPLICATION /// #include "storage/v2/inmemory/replication/replication_client.hpp" #include "storage/v2/inmemory/replication/replication_server.hpp" +#include "storage/v2/inmemory/unique_constraints.hpp" namespace memgraph::storage { @@ -34,9 +28,7 @@ InMemoryStorage::InMemoryStorage(Config config) lock_file_path_(config.durability.storage_directory / durability::kLockFile), wal_directory_(config.durability.storage_directory / durability::kWalDirectory), uuid_(utils::GenerateUUID()), - global_locker_(file_retainer_.AddLocker()), - replication_state_(config_.durability.restore_replication_state_on_startup, - config_.durability.storage_directory) { + global_locker_(file_retainer_.AddLocker()) { if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || config_.durability.snapshot_on_exit || config_.durability.recover_on_startup) { // Create the directory initially to crash the database in case of @@ -256,7 +248,7 @@ Result> InMemoryStorage::InMemoryAccessor::DeleteV "accessor when deleting a vertex!"); auto *vertex_ptr = vertex->vertex_; - std::lock_guard guard(vertex_ptr->lock); + auto guard = std::unique_lock{vertex_ptr->lock}; if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; @@ -294,7 +286,7 @@ InMemoryStorage::InMemoryAccessor::DetachDeleteVertex(VertexAccessor *vertex) { std::vector> out_edges; { - std::lock_guard guard(vertex_ptr->lock); + auto guard = std::unique_lock{vertex_ptr->lock}; if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR; @@ -334,7 +326,7 @@ InMemoryStorage::InMemoryAccessor::DetachDeleteVertex(VertexAccessor *vertex) { } } - std::lock_guard guard(vertex_ptr->lock); + auto guard = std::unique_lock{vertex_ptr->lock}; // We need to check again for serialization errors because we unlocked the // vertex. Some other transaction could have modified the vertex in the @@ -374,8 +366,8 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso auto *to_vertex = to->vertex_; // Obtain the locks by `gid` order to avoid lock cycles. - std::unique_lock guard_from(from_vertex->lock, std::defer_lock); - std::unique_lock guard_to(to_vertex->lock, std::defer_lock); + auto guard_from = std::unique_lock{from_vertex->lock, std::defer_lock}; + auto guard_to = std::unique_lock{to_vertex->lock, std::defer_lock}; if (from_vertex->gid < to_vertex->gid) { guard_from.lock(); guard_to.lock(); @@ -440,8 +432,8 @@ Result InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAcces auto *to_vertex = to->vertex_; // Obtain the locks by `gid` order to avoid lock cycles. - std::unique_lock guard_from(from_vertex->lock, std::defer_lock); - std::unique_lock guard_to(to_vertex->lock, std::defer_lock); + auto guard_from = std::unique_lock{from_vertex->lock, std::defer_lock}; + auto guard_to = std::unique_lock{to_vertex->lock, std::defer_lock}; if (from_vertex->gid < to_vertex->gid) { guard_from.lock(); guard_to.lock(); @@ -508,10 +500,10 @@ Result> InMemoryStorage::InMemoryAccessor::DeleteEdg auto edge_ref = edge->edge_; auto edge_type = edge->edge_type_; - std::unique_lock guard; + std::unique_lock guard; if (config_.properties_on_edges) { auto *edge_ptr = edge_ref.ptr; - guard = std::unique_lock(edge_ptr->lock); + guard = std::unique_lock{edge_ptr->lock}; if (!PrepareForWrite(&transaction_, edge_ptr)) return Error::SERIALIZATION_ERROR; @@ -522,8 +514,8 @@ Result> InMemoryStorage::InMemoryAccessor::DeleteEdg auto *to_vertex = edge->to_vertex_; // Obtain the locks by `gid` order to avoid lock cycles. - std::unique_lock guard_from(from_vertex->lock, std::defer_lock); - std::unique_lock guard_to(to_vertex->lock, std::defer_lock); + auto guard_from = std::unique_lock{from_vertex->lock, std::defer_lock}; + auto guard_to = std::unique_lock{to_vertex->lock, std::defer_lock}; if (from_vertex->gid < to_vertex->gid) { guard_from.lock(); guard_to.lock(); @@ -739,7 +731,7 @@ void InMemoryStorage::InMemoryAccessor::Abort() { switch (prev.type) { case PreviousPtr::Type::VERTEX: { auto *vertex = prev.vertex; - std::lock_guard guard(vertex->lock); + auto guard = std::unique_lock{vertex->lock}; Delta *current = vertex->delta; while (current != nullptr && current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id.load(std::memory_order_acquire)) { @@ -827,7 +819,7 @@ void InMemoryStorage::InMemoryAccessor::Abort() { } case PreviousPtr::Type::EDGE: { auto *edge = prev.edge; - std::lock_guard guard(edge->lock); + auto guard = std::lock_guard{edge->lock}; Delta *current = edge->delta; while (current != nullptr && current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id.load(std::memory_order_acquire)) { @@ -1272,7 +1264,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) switch (prev.type) { case PreviousPtr::Type::VERTEX: { Vertex *vertex = prev.vertex; - std::lock_guard vertex_guard(vertex->lock); + auto vertex_guard = std::unique_lock{vertex->lock}; if (vertex->delta != &delta) { // Something changed, we're not the first delta in the chain // anymore. @@ -1286,7 +1278,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) } case PreviousPtr::Type::EDGE: { Edge *edge = prev.edge; - std::lock_guard edge_guard(edge->lock); + auto edge_guard = std::unique_lock{edge->lock}; if (edge->delta != &delta) { // Something changed, we're not the first delta in the chain // anymore. @@ -1305,7 +1297,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) // part of the suffix later. break; } - std::unique_lock guard; + std::unique_lock guard; { // We need to find the parent object in order to be able to use // its lock. @@ -1315,10 +1307,10 @@ void InMemoryStorage::CollectGarbage(std::unique_lock main_guard) } switch (parent.type) { case PreviousPtr::Type::VERTEX: - guard = std::unique_lock(parent.vertex->lock); + guard = std::unique_lock{parent.vertex->lock}; break; case PreviousPtr::Type::EDGE: - guard = std::unique_lock(parent.edge->lock); + guard = std::unique_lock{parent.edge->lock}; break; case PreviousPtr::Type::DELTA: case PreviousPtr::Type::NULLPTR: @@ -1668,8 +1660,8 @@ bool InMemoryStorage::AppendToWalDataDefinition(durability::StorageGlobalOperati wal_file_->AppendOperation(operation, label, properties, final_commit_timestamp); FinalizeWalFile(); - return replication_state_.AppendToWalDataDefinition(wal_file_->SequenceNumber(), operation, label, properties, - final_commit_timestamp); + return replication_state_.AppendOperation(wal_file_->SequenceNumber(), operation, label, properties, + final_commit_timestamp); } utils::BasicResult InMemoryStorage::CreateSnapshot( @@ -1780,7 +1772,7 @@ auto InMemoryStorage::CreateReplicationClient(std::string name, io::network::End replication::ReplicationMode mode, replication::ReplicationClientConfig const &config) -> std::unique_ptr { - return std::make_unique(this, std::move(name), endpoint, mode, config); + return std::make_unique(this, std::move(name), std::move(endpoint), mode, config); } std::unique_ptr InMemoryStorage::CreateReplicationServer( diff --git a/src/storage/v2/inmemory/storage.hpp b/src/storage/v2/inmemory/storage.hpp index 13b071fef..a0b56fe03 100644 --- a/src/storage/v2/inmemory/storage.hpp +++ b/src/storage/v2/inmemory/storage.hpp @@ -17,15 +17,13 @@ #include "storage/v2/storage.hpp" /// REPLICATION /// -#include "rpc/server.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" +#include "storage/v2/replication/replication.hpp" #include "storage/v2/replication/replication_persistence_helper.hpp" #include "storage/v2/replication/rpc.hpp" #include "storage/v2/replication/serialization.hpp" -#include "storage/v2/replication/replication.hpp" - namespace memgraph::storage { // The storage is based on this paper: @@ -36,7 +34,6 @@ namespace memgraph::storage { class InMemoryStorage final : public Storage { friend class InMemoryReplicationServer; friend class InMemoryReplicationClient; - friend class ReplicationClient; public: enum class CreateSnapshotError : uint8_t { @@ -363,33 +360,6 @@ class InMemoryStorage final : public Storage { utils::BasicResult DropUniqueConstraint( LabelId label, const std::set &properties, std::optional desired_commit_timestamp) override; - bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) { - return replication_state_.SetReplicaRole(std::move(endpoint), config, this); - } - - bool SetMainReplicationRole() { return replication_state_.SetMainReplicationRole(this); } - - /// @pre The instance should have a MAIN role - /// @pre Timeout can only be set for SYNC replication - auto RegisterReplica(std::string name, io::network::Endpoint endpoint, - const replication::ReplicationMode replication_mode, - const replication::RegistrationMode registration_mode, - const replication::ReplicationClientConfig &config) { - return replication_state_.RegisterReplica(std::move(name), std::move(endpoint), replication_mode, registration_mode, - config, this); - } - - /// @pre The instance should have a MAIN role - bool UnregisterReplica(const std::string &name) { return replication_state_.UnregisterReplica(name); } - - replication::ReplicationRole GetReplicationRole() const { return replication_state_.GetRole(); } - - auto ReplicasInfo() { return replication_state_.ReplicasInfo(); } - - std::optional GetReplicaState(std::string_view name) { - return replication_state_.GetReplicaState(name); - } - void FreeMemory(std::unique_lock main_guard) override; utils::FileRetainer::FileLockerAccessor::ret_type IsPathLocked(); @@ -435,10 +405,6 @@ class InMemoryStorage final : public Storage { uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); - void RestoreReplicationRole() { return replication_state_.RestoreReplicationRole(this); } - - void RestoreReplicas() { return replication_state_.RestoreReplicas(this); } - void EstablishNewEpoch() override; // Main object storage @@ -494,8 +460,6 @@ class InMemoryStorage final : public Storage { // Flags to inform CollectGarbage that it needs to do the more expensive full scans std::atomic gc_full_scan_vertices_delete_ = false; std::atomic gc_full_scan_edges_delete_ = false; - - ReplicationState replication_state_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/inmemory/unique_constraints.cpp b/src/storage/v2/inmemory/unique_constraints.cpp index 02e88f67c..71ff1e160 100644 --- a/src/storage/v2/inmemory/unique_constraints.cpp +++ b/src/storage/v2/inmemory/unique_constraints.cpp @@ -50,7 +50,7 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c bool deleted; bool has_label; { - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; delta = vertex.delta; deleted = vertex.deleted; has_label = utils::Contains(vertex.labels, label); @@ -136,7 +136,7 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std:: bool deleted; Delta *delta; { - std::lock_guard guard(vertex.lock); + auto guard = std::shared_lock{vertex.lock}; has_label = utils::Contains(vertex.labels, label); deleted = vertex.deleted; delta = vertex.delta; diff --git a/src/storage/v2/replication/replication.cpp b/src/storage/v2/replication/replication.cpp index d05644b9b..66431a3f4 100644 --- a/src/storage/v2/replication/replication.cpp +++ b/src/storage/v2/replication/replication.cpp @@ -13,22 +13,9 @@ #include "storage/v2/constraints/constraints.hpp" #include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/snapshot.hpp" -#include "storage/v2/durability/wal.hpp" -#include "storage/v2/edge_accessor.hpp" -#include "storage/v2/edge_direction.hpp" -#include "storage/v2/inmemory/storage.hpp" -#include "storage/v2/storage_mode.hpp" -#include "storage/v2/vertex_accessor.hpp" -#include "utils/stat.hpp" - -/// REPLICATION /// #include "storage/v2/replication/replication_client.hpp" #include "storage/v2/replication/replication_server.hpp" -#include "storage/v2/replication/rpc.hpp" -#include "storage/v2/storage_error.hpp" - -#include "storage/v2/inmemory/replication/replication_client.hpp" -#include "storage/v2/inmemory/replication/replication_server.hpp" +#include "storage/v2/storage.hpp" namespace memgraph::storage { @@ -97,10 +84,9 @@ bool storage::ReplicationState::SetMainReplicationRole(storage::Storage *storage return true; } -bool storage::ReplicationState::AppendToWalDataDefinition(const uint64_t seq_num, - durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, - uint64_t final_commit_timestamp) { +bool storage::ReplicationState::AppendOperation(const uint64_t seq_num, durability::StorageGlobalOperation operation, + LabelId label, const std::set &properties, + uint64_t final_commit_timestamp) { bool finalized_on_all_replicas = true; // TODO Should we return true if not MAIN? if (GetRole() == replication::ReplicationRole::MAIN) { @@ -199,7 +185,7 @@ utils::BasicResult ReplicationState::Reg } } - auto client = storage->CreateReplicationClient(std::move(name), endpoint, replication_mode, config); + auto client = storage->CreateReplicationClient(std::move(name), std::move(endpoint), replication_mode, config); client->Start(); if (client->State() == replication::ReplicaState::INVALID) { diff --git a/src/storage/v2/replication/replication.hpp b/src/storage/v2/replication/replication.hpp index 287f8a312..71c4d2b2b 100644 --- a/src/storage/v2/replication/replication.hpp +++ b/src/storage/v2/replication/replication.hpp @@ -11,20 +11,18 @@ #pragma once -#include "storage/v2/inmemory/label_index.hpp" -#include "storage/v2/inmemory/label_property_index.hpp" -#include "storage/v2/storage.hpp" +#include "kvstore/kvstore.hpp" +#include "storage/v2/durability/storage_global_operation.hpp" +#include "utils/result.hpp" /// REPLICATION /// -#include "rpc/server.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" +#include "storage/v2/replication/global.hpp" #include "storage/v2/replication/replication_persistence_helper.hpp" #include "storage/v2/replication/rpc.hpp" #include "storage/v2/replication/serialization.hpp" -#include "storage/v2/replication/global.hpp" - // TODO use replication namespace namespace memgraph::storage { @@ -56,8 +54,8 @@ struct ReplicationState { void RestoreReplicationRole(Storage *storage); // MAIN actually doing the replication - bool AppendToWalDataDefinition(uint64_t seq_num, durability::StorageGlobalOperation operation, LabelId label, - const std::set &properties, uint64_t final_commit_timestamp); + bool AppendOperation(uint64_t seq_num, durability::StorageGlobalOperation operation, LabelId label, + const std::set &properties, uint64_t final_commit_timestamp); void InitializeTransaction(uint64_t seq_num); void AppendDelta(const Delta &delta, const Vertex &parent, uint64_t timestamp); void AppendDelta(const Delta &delta, const Edge &parent, uint64_t timestamp); diff --git a/src/storage/v2/replication/replication_client.cpp b/src/storage/v2/replication/replication_client.cpp index f9c481762..5b22e6628 100644 --- a/src/storage/v2/replication/replication_client.cpp +++ b/src/storage/v2/replication/replication_client.cpp @@ -15,9 +15,9 @@ #include #include "storage/v2/durability/durability.hpp" -#include "storage/v2/inmemory/storage.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" +#include "storage/v2/storage.hpp" #include "storage/v2/transaction.hpp" #include "utils/file_locker.hpp" #include "utils/logging.hpp" @@ -30,18 +30,258 @@ static auto CreateClientContext(const replication::ReplicationClientConfig &conf : communication::ClientContext{}; } -ReplicationClient::ReplicationClient(std::string name, memgraph::io::network::Endpoint endpoint, +ReplicationClient::ReplicationClient(Storage *storage, std::string name, memgraph::io::network::Endpoint endpoint, replication::ReplicationMode mode, replication::ReplicationClientConfig const &config) : name_{std::move(name)}, rpc_context_{CreateClientContext(config)}, rpc_client_{std::move(endpoint), &rpc_context_}, replica_check_frequency_{config.replica_check_frequency}, - mode_{mode} {} + mode_{mode}, + storage_{storage} {} ReplicationClient::~ReplicationClient() { auto endpoint = rpc_client_.Endpoint(); spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port); + thread_pool_.Shutdown(); +} + +uint64_t ReplicationClient::LastCommitTimestamp() const { + return storage_->replication_state_.last_commit_timestamp_.load(); +} + +void ReplicationClient::InitializeClient() { + uint64_t current_commit_timestamp{kTimestampInitialId}; + + const auto &main_epoch = storage_->replication_state_.GetEpoch(); + + auto stream{rpc_client_.Stream(storage_->replication_state_.last_commit_timestamp_, + main_epoch.id)}; + + const auto replica = stream.AwaitResponse(); + std::optional branching_point; + if (replica.epoch_id != main_epoch.id && replica.current_commit_timestamp != kTimestampInitialId) { + auto const &history = storage_->replication_state_.history; + const auto epoch_info_iter = std::find_if(history.crbegin(), history.crend(), [&](const auto &main_epoch_info) { + return main_epoch_info.first == replica.epoch_id; + }); + if (epoch_info_iter == history.crend()) { + branching_point = 0; + } else if (epoch_info_iter->second != replica.current_commit_timestamp) { + branching_point = epoch_info_iter->second; + } + } + if (branching_point) { + spdlog::error( + "You cannot register Replica {} to this Main because at one point " + "Replica {} acted as the Main instance. Both the Main and Replica {} " + "now hold unique data. Please resolve data conflicts and start the " + "replication on a clean instance.", + name_, name_, name_); + return; + } + + current_commit_timestamp = replica.current_commit_timestamp; + spdlog::trace("Current timestamp on replica {}: {}", name_, current_commit_timestamp); + spdlog::trace("Current timestamp on main: {}", storage_->replication_state_.last_commit_timestamp_.load()); + if (current_commit_timestamp == storage_->replication_state_.last_commit_timestamp_.load()) { + spdlog::debug("Replica '{}' up to date", name_); + std::unique_lock client_guard{client_lock_}; + replica_state_.store(replication::ReplicaState::READY); + } else { + spdlog::debug("Replica '{}' is behind", name_); + { + std::unique_lock client_guard{client_lock_}; + replica_state_.store(replication::ReplicaState::RECOVERY); + } + thread_pool_.AddTask([=, this] { this->RecoverReplica(current_commit_timestamp); }); + } +} + +TimestampInfo ReplicationClient::GetTimestampInfo() { + TimestampInfo info; + info.current_timestamp_of_replica = 0; + info.current_number_of_timestamp_behind_master = 0; + + try { + auto stream{rpc_client_.Stream()}; + const auto response = stream.AwaitResponse(); + const auto is_success = response.success; + if (!is_success) { + replica_state_.store(replication::ReplicaState::INVALID); + HandleRpcFailure(); + } + auto main_time_stamp = storage_->replication_state_.last_commit_timestamp_.load(); + info.current_timestamp_of_replica = response.current_commit_timestamp; + info.current_number_of_timestamp_behind_master = response.current_commit_timestamp - main_time_stamp; + } catch (const rpc::RpcFailedException &) { + { + std::unique_lock client_guard(client_lock_); + replica_state_.store(replication::ReplicaState::INVALID); + } + HandleRpcFailure(); // mutex already unlocked, if the new enqueued task dispatches immediately it probably won't + // block + } + + return info; +} + +void ReplicationClient::HandleRpcFailure() { + spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication")); + TryInitializeClientAsync(); +} + +void ReplicationClient::TryInitializeClientAsync() { + thread_pool_.AddTask([this] { + rpc_client_.Abort(); + this->TryInitializeClientSync(); + }); +} + +void ReplicationClient::TryInitializeClientSync() { + try { + InitializeClient(); + } catch (const rpc::RpcFailedException &) { + std::unique_lock client_guarde{client_lock_}; + replica_state_.store(replication::ReplicaState::INVALID); + spdlog::error(utils::MessageWithLink("Failed to connect to replica {} at the endpoint {}.", name_, + rpc_client_.Endpoint(), "https://memgr.ph/replication")); + } +} + +void ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) { + std::unique_lock guard(client_lock_); + const auto status = replica_state_.load(); + switch (status) { + case replication::ReplicaState::RECOVERY: + spdlog::debug("Replica {} is behind MAIN instance", name_); + return; + case replication::ReplicaState::REPLICATING: + spdlog::debug("Replica {} missed a transaction", name_); + // We missed a transaction because we're still replicating + // the previous transaction so we need to go to RECOVERY + // state to catch up with the missing transaction + // We cannot queue the recovery process here because + // an error can happen while we're replicating the previous + // transaction after which the client should go to + // INVALID state before starting the recovery process + replica_state_.store(replication::ReplicaState::RECOVERY); + return; + case replication::ReplicaState::INVALID: + HandleRpcFailure(); + return; + case replication::ReplicaState::READY: + MG_ASSERT(!replica_stream_); + try { + replica_stream_.emplace( + ReplicaStream{this, storage_->replication_state_.last_commit_timestamp_.load(), current_wal_seq_num}); + replica_state_.store(replication::ReplicaState::REPLICATING); + } catch (const rpc::RpcFailedException &) { + replica_state_.store(replication::ReplicaState::INVALID); + HandleRpcFailure(); + } + return; + } +} + +auto ReplicationClient::GetEpochId() const -> std::string const & { return storage_->replication_state_.GetEpoch().id; } + +bool ReplicationClient::FinalizeTransactionReplication() { + // We can only check the state because it guarantees to be only + // valid during a single transaction replication (if the assumption + // that this and other transaction replication functions can only be + // called from a one thread stands) + if (replica_state_ != replication::ReplicaState::REPLICATING) { + return false; + } + + auto task = [this]() { + MG_ASSERT(replica_stream_, "Missing stream for transaction deltas"); + try { + auto response = replica_stream_->Finalize(); + replica_stream_.reset(); + std::unique_lock client_guard(client_lock_); + if (!response.success || replica_state_ == replication::ReplicaState::RECOVERY) { + replica_state_.store(replication::ReplicaState::RECOVERY); + thread_pool_.AddTask([&, this] { this->RecoverReplica(response.current_commit_timestamp); }); + } else { + replica_state_.store(replication::ReplicaState::READY); + return true; + } + } catch (const rpc::RpcFailedException &) { + replica_stream_.reset(); + { + std::unique_lock client_guard(client_lock_); + replica_state_.store(replication::ReplicaState::INVALID); + } + HandleRpcFailure(); + } + return false; + }; + + if (mode_ == replication::ReplicationMode::ASYNC) { + thread_pool_.AddTask([=] { (void)task(); }); + return true; + } + + return task(); +} + +void ReplicationClient::FrequentCheck() { + const auto is_success = std::invoke([this]() { + try { + auto stream{rpc_client_.Stream()}; + const auto response = stream.AwaitResponse(); + return response.success; + } catch (const rpc::RpcFailedException &) { + return false; + } + }); + // States: READY, REPLICATING, RECOVERY, INVALID + // If success && ready, replicating, recovery -> stay the same because something good is going on. + // If success && INVALID -> [it's possible that replica came back to life] -> TryInitializeClient. + // If fail -> [replica is not reachable at all] -> INVALID state. + // NOTE: TryInitializeClient might return nothing if there is a branching point. + // NOTE: The early return pattern simplified the code, but the behavior should be as explained. + if (!is_success) { + replica_state_.store(replication::ReplicaState::INVALID); + return; + } + if (replica_state_.load() == replication::ReplicaState::INVALID) { + TryInitializeClientAsync(); + } +} + +void ReplicationClient::Start() { + auto const &endpoint = rpc_client_.Endpoint(); + spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port); + + TryInitializeClientSync(); + + // Help the user to get the most accurate replica state possible. + if (replica_check_frequency_ > std::chrono::seconds(0)) { + replica_checker_.Run("Replica Checker", replica_check_frequency_, [this] { this->FrequentCheck(); }); + } +} + +void ReplicationClient::IfStreamingTransaction(const std::function &callback) { + // We can only check the state because it guarantees to be only + // valid during a single transaction replication (if the assumption + // that this and other transaction replication functions can only be + // called from a one thread stands) + if (replica_state_ != replication::ReplicaState::REPLICATING) { + return; + } + + try { + callback(*replica_stream_); + } catch (const rpc::RpcFailedException &) { + { + std::unique_lock client_guard{client_lock_}; + replica_state_.store(replication::ReplicaState::INVALID); + } + HandleRpcFailure(); + } } ////// ReplicaStream ////// @@ -50,6 +290,7 @@ ReplicaStream::ReplicaStream(ReplicationClient *self, const uint64_t previous_co : self_(self), stream_(self_->rpc_client_.Stream(previous_commit_timestamp, current_seq_num)) { replication::Encoder encoder{stream_.GetBuilder()}; + encoder.WriteString(self_->GetEpochId()); } diff --git a/src/storage/v2/replication/replication_client.hpp b/src/storage/v2/replication/replication_client.hpp index ce0f3c742..76c2150a9 100644 --- a/src/storage/v2/replication/replication_client.hpp +++ b/src/storage/v2/replication/replication_client.hpp @@ -12,23 +12,30 @@ #pragma once #include "rpc/client.hpp" -#include "storage/v2/durability/wal.hpp" +#include "storage/v2/durability/storage_global_operation.hpp" +#include "storage/v2/id_types.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" #include "storage/v2/replication/global.hpp" #include "storage/v2/replication/rpc.hpp" -#include "storage/v2/vertex.hpp" -#include "utils/file.hpp" +#include "utils/file_locker.hpp" #include "utils/scheduler.hpp" #include "utils/thread_pool.hpp" +#include +#include +#include +#include + namespace memgraph::storage { +struct Delta; +struct Vertex; +struct Edge; class Storage; - class ReplicationClient; -// Handler used for transfering the current transaction. +// Handler used for transferring the current transaction. class ReplicaStream { public: explicit ReplicaStream(ReplicationClient *self, uint64_t previous_commit_timestamp, uint64_t current_seq_num); @@ -59,8 +66,8 @@ class ReplicationClient { friend class ReplicaStream; public: - ReplicationClient(std::string name, memgraph::io::network::Endpoint endpoint, replication::ReplicationMode mode, - const replication::ReplicationClientConfig &config); + ReplicationClient(Storage *storage, std::string name, memgraph::io::network::Endpoint endpoint, + replication::ReplicationMode mode, const replication::ReplicationClientConfig &config); ReplicationClient(ReplicationClient const &) = delete; ReplicationClient &operator=(ReplicationClient const &) = delete; @@ -69,24 +76,35 @@ class ReplicationClient { virtual ~ReplicationClient(); - virtual void Start() = 0; - - const auto &Name() const { return name_; } - - auto State() const { return replica_state_.load(); } - - auto Mode() const { return mode_; } - + auto Mode() const -> replication::ReplicationMode { return mode_; } + auto Name() const -> std::string const & { return name_; } auto Endpoint() const -> io::network::Endpoint const & { return rpc_client_.Endpoint(); } + auto State() const -> replication::ReplicaState { return replica_state_.load(); } + auto GetTimestampInfo() -> TimestampInfo; - virtual void StartTransactionReplication(uint64_t current_wal_seq_num) = 0; - virtual void IfStreamingTransaction(const std::function &callback) = 0; - virtual auto GetEpochId() const -> std::string const & = 0; // TODO: make non-virtual once epoch is moved to storage - virtual auto GetStorage() -> Storage * = 0; - [[nodiscard]] virtual bool FinalizeTransactionReplication() = 0; - virtual TimestampInfo GetTimestampInfo() = 0; + void Start(); + void StartTransactionReplication(const uint64_t current_wal_seq_num); + // Replication clients can be removed at any point + // so to avoid any complexity of checking if the client was removed whenever + // we want to send part of transaction and to avoid adding some GC logic this + // function will run a callback if, after previously callling + // StartTransactionReplication, stream is created. + void IfStreamingTransaction(const std::function &callback); + // Return whether the transaction could be finalized on the replication client or not. + [[nodiscard]] bool FinalizeTransactionReplication(); protected: + virtual void RecoverReplica(uint64_t replica_commit) = 0; + + auto GetStorage() -> Storage * { return storage_; } + auto GetEpochId() const -> std::string const &; + auto LastCommitTimestamp() const -> uint64_t; + void InitializeClient(); + void HandleRpcFailure(); + void TryInitializeClientAsync(); + void TryInitializeClientSync(); + void FrequentCheck(); + std::string name_; communication::ClientContext rpc_context_; rpc::Client rpc_client_; @@ -113,6 +131,7 @@ class ReplicationClient { std::atomic replica_state_{replication::ReplicaState::INVALID}; utils::Scheduler replica_checker_; + Storage *storage_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index b991ebc95..96e0a52c1 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -51,7 +51,9 @@ Storage::Storage(Config config, StorageMode storage_mode) storage_mode_(storage_mode), indices_(&constraints_, config, storage_mode), constraints_(config, storage_mode), - id_(config.name) {} + id_(config.name), + replication_state_(config_.durability.restore_replication_state_on_startup, + config_.durability.storage_directory) {} Storage::Accessor::Accessor(Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode) : storage_(storage), diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index f2b10e5c8..54c6fc31b 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -26,6 +26,7 @@ #include "storage/v2/mvcc.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" +#include "storage/v2/replication/replication.hpp" #include "storage/v2/replication/replication_client.hpp" #include "storage/v2/replication/replication_server.hpp" #include "storage/v2/storage_error.hpp" @@ -68,6 +69,9 @@ struct StorageInfo { }; class Storage { + friend class ReplicationServer; + friend class ReplicationClient; + public: Storage(Config config, StorageMode storage_mode); @@ -293,6 +297,34 @@ class Storage { replication::ReplicationServerConfig const &config) -> std::unique_ptr = 0; + /// REPLICATION + bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) { + return replication_state_.SetReplicaRole(std::move(endpoint), config, this); + } + bool SetMainReplicationRole() { return replication_state_.SetMainReplicationRole(this); } + + /// @pre The instance should have a MAIN role + /// @pre Timeout can only be set for SYNC replication + auto RegisterReplica(std::string name, io::network::Endpoint endpoint, + const replication::ReplicationMode replication_mode, + const replication::RegistrationMode registration_mode, + const replication::ReplicationClientConfig &config) { + return replication_state_.RegisterReplica(std::move(name), std::move(endpoint), replication_mode, registration_mode, + config, this); + } + /// @pre The instance should have a MAIN role + bool UnregisterReplica(const std::string &name) { return replication_state_.UnregisterReplica(name); } + replication::ReplicationRole GetReplicationRole() const { return replication_state_.GetRole(); } + auto ReplicasInfo() { return replication_state_.ReplicasInfo(); } + std::optional GetReplicaState(std::string_view name) { + return replication_state_.GetReplicaState(name); + } + + protected: + void RestoreReplicas() { return replication_state_.RestoreReplicas(this); } + void RestoreReplicationRole() { return replication_state_.RestoreReplicationRole(this); } + + public: // Main storage lock. // Accessors take a shared lock when starting, so it is possible to block // creation of new accessors by taking a unique lock. This is used when doing @@ -323,6 +355,9 @@ class Storage { std::atomic vertex_id_{0}; std::atomic edge_id_{0}; const std::string id_; //!< High-level assigned ID + + protected: + ReplicationState replication_state_; }; } // namespace memgraph::storage diff --git a/src/storage/v2/vertex.hpp b/src/storage/v2/vertex.hpp index 5ec612f68..d051b95fd 100644 --- a/src/storage/v2/vertex.hpp +++ b/src/storage/v2/vertex.hpp @@ -19,7 +19,7 @@ #include "storage/v2/edge_ref.hpp" #include "storage/v2/id_types.hpp" #include "storage/v2/property_store.hpp" -#include "utils/spin_lock.hpp" +#include "utils/rw_spin_lock.hpp" namespace memgraph::storage { @@ -38,7 +38,7 @@ struct Vertex { std::vector> in_edges; std::vector> out_edges; - mutable utils::SpinLock lock; + mutable utils::RWSpinLock lock; bool deleted; // uint8_t PAD; // uint16_t PAD; diff --git a/src/storage/v2/vertex_accessor.cpp b/src/storage/v2/vertex_accessor.cpp index 3d604b6eb..fcdfdf41c 100644 --- a/src/storage/v2/vertex_accessor.cpp +++ b/src/storage/v2/vertex_accessor.cpp @@ -35,7 +35,7 @@ std::pair IsVisible(Vertex const *vertex, Transaction const *transac bool deleted = false; Delta *delta = nullptr; { - std::lock_guard guard(vertex->lock); + auto guard = std::shared_lock{vertex->lock}; deleted = vertex->deleted; delta = vertex->delta; } @@ -90,7 +90,7 @@ bool VertexAccessor::IsVisible(View view) const { Result VertexAccessor::AddLabel(LabelId label) { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; if (vertex_->deleted) return Error::DELETED_OBJECT; @@ -109,7 +109,7 @@ Result VertexAccessor::AddLabel(LabelId label) { /// TODO: move to after update and change naming to vertex after update Result VertexAccessor::RemoveLabel(LabelId label) { - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; if (vertex_->deleted) return Error::DELETED_OBJECT; @@ -135,7 +135,7 @@ Result VertexAccessor::HasLabel(LabelId label, View view) const { bool has_label = false; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; has_label = std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end(); delta = vertex_->delta; @@ -182,7 +182,7 @@ Result> VertexAccessor::Labels(View view) const { std::vector labels; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; labels = vertex_->labels; delta = vertex_->delta; @@ -225,7 +225,7 @@ Result> VertexAccessor::Labels(View view) const { Result VertexAccessor::SetProperty(PropertyId property, const PropertyValue &value) { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; @@ -250,7 +250,7 @@ Result VertexAccessor::SetProperty(PropertyId property, const Pro Result VertexAccessor::InitProperties(const std::map &properties) { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; @@ -269,7 +269,7 @@ Result VertexAccessor::InitProperties(const std::map>> VertexAccessor::UpdateProperties( std::map &properties) const { utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception; - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; @@ -287,7 +287,7 @@ Result>> Vertex } Result> VertexAccessor::ClearProperties() { - std::lock_guard guard(vertex_->lock); + auto guard = std::unique_lock{vertex_->lock}; if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR; @@ -311,7 +311,7 @@ Result VertexAccessor::GetProperty(PropertyId property, View view PropertyValue value; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; value = vertex_->properties.GetProperty(property); delta = vertex_->delta; @@ -359,7 +359,7 @@ Result> VertexAccessor::Properties(View view std::map properties; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; properties = vertex_->properties.Properties(); delta = vertex_->delta; @@ -424,7 +424,7 @@ Result> VertexAccessor::InEdges(View view, const std:: auto in_edges = edge_store{}; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; // TODO: a better filter copy if (edge_types.empty() && !destination) { @@ -500,7 +500,7 @@ Result> VertexAccessor::OutEdges(View view, const std: auto out_edges = edge_store{}; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; if (edge_types.empty() && !destination) { out_edges = vertex_->out_edges; @@ -558,7 +558,7 @@ Result VertexAccessor::InDegree(View view) const { size_t degree = 0; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; degree = vertex_->in_edges.size(); delta = vertex_->delta; @@ -606,7 +606,7 @@ Result VertexAccessor::OutDegree(View view) const { size_t degree = 0; Delta *delta = nullptr; { - std::lock_guard guard(vertex_->lock); + auto guard = std::shared_lock{vertex_->lock}; deleted = vertex_->deleted; degree = vertex_->out_edges.size(); delta = vertex_->delta; diff --git a/src/utils/rw_spin_lock.hpp b/src/utils/rw_spin_lock.hpp new file mode 100644 index 000000000..c76fc598b --- /dev/null +++ b/src/utils/rw_spin_lock.hpp @@ -0,0 +1,124 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. +#pragma once + +#include +#include +#include +#include + +namespace memgraph::utils { +namespace { +/// A helper for RWSpinLock, allows a contended spin lock to yield to another thread. +struct yeilder { + void operator()() noexcept { +#if defined(__i386__) || defined(__x86_64__) + // TODO: make portable + __builtin_ia32_pause(); +#endif + ++count; + if (count > 8) [[unlikely]] { + count = 0; + nanosleep(&shortpause, nullptr); + // Increase the backoff + shortpause.tv_nsec = std::min(shortpause.tv_nsec << 1, 512); + } + } + + private: + uint_fast32_t count{0}; + timespec shortpause = {.tv_sec = 0, .tv_nsec = 1}; +}; +} // namespace + +/** + * A reader/writer spin lock. + * Stores in a uint32_t, + * 0x0000'0001 - is the write bit + * rest of the bits hold the count for the number of current writers. + * The lock is friendly to writers. + * - writer lock() will wait for all readers to leave unlock_shared() + * - new reader lock_shared() will wait until writer has unlock() + **/ +struct RWSpinLock { + RWSpinLock() = default; + + void lock() { + // spin: to grant the UNIQUE_LOCKED bit + while (true) { + // optimistic: assume we will be granted the lock + auto const phase1 = std::atomic_ref{lock_status_}.fetch_or(UNIQUE_LOCKED, std::memory_order_acq_rel); + // check: we were granted UNIQUE_LOCK and no current readers + if (phase1 == 0) [[likely]] + return; + // check: we were granted UNIQUE_LOCK, but need to wait for readers + if ((phase1 & UNIQUE_LOCKED) != UNIQUE_LOCKED) [[likely]] + break; + + // spin: to wait for UNIQUE_LOCKED to be available + auto maybe_yield = yeilder{}; + while (true) { + auto const phase2 = std::atomic_ref{lock_status_}.load(std::memory_order_relaxed); + // check: we are able to obtain UNIQUE_LOCK + if ((phase2 & UNIQUE_LOCKED) != UNIQUE_LOCKED) [[likely]] + break; + maybe_yield(); + } + } + + // spin: to wait for readers to leave + auto maybe_yield = yeilder{}; + while (true) { + auto const phase3 = std::atomic_ref{lock_status_}.load(std::memory_order_relaxed); + // check: all readers have gone (leaving only the UNIQUE_LOCKED bit set) + if (phase3 == UNIQUE_LOCKED) return; + maybe_yield(); + } + } + + void unlock() { std::atomic_ref{lock_status_}.fetch_and(~UNIQUE_LOCKED, std::memory_order_release); } + + void lock_shared() { + while (true) { + // optimistic: assume we will be granted the lock + auto const phase1 = std::atomic_ref{lock_status_}.fetch_add(READER, std::memory_order_acquire); + // check: we incremented reader count without the UNIQUE_LOCK already being held + if ((phase1 & UNIQUE_LOCKED) != UNIQUE_LOCKED) [[likely]] + return; + // correct for our optimism, we shouldn't have modified the reader count + std::atomic_ref{lock_status_}.fetch_sub(READER, std::memory_order_release); + + // spin: to wait for UNIQUE_LOCKED to be available + auto maybe_yield = yeilder{}; + while (true) { + auto const phase2 = std::atomic_ref{lock_status_}.load(std::memory_order_relaxed); + // check: UNIQUE_LOCK was released + if ((phase2 & UNIQUE_LOCKED) != UNIQUE_LOCKED) [[likely]] + break; + maybe_yield(); + } + } + } + + void unlock_shared() { std::atomic_ref{lock_status_}.fetch_sub(READER, std::memory_order_release); } + + private: + using status_t = uint32_t; + enum FLAGS : status_t { + UNIQUE_LOCKED = 1, + READER = 2, + }; + + // TODO: ATM not atomic, just used via atomic_ref, because the type needs to be movable into skip_list + // fix the design flaw and then make RWSpinLock a non-copy/non-move type + status_t lock_status_ = 0; +}; +} // namespace memgraph::utils diff --git a/tests/e2e/batched_procedures/workloads.yaml b/tests/e2e/batched_procedures/workloads.yaml index 3ec550770..f4a042c73 100644 --- a/tests/e2e/batched_procedures/workloads.yaml +++ b/tests/e2e/batched_procedures/workloads.yaml @@ -1,14 +1,32 @@ -template_cluster: &template_cluster +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "batched-procedures-e2e.log" setup_queries: [] validation_queries: [] +disk_cluster: &disk_cluster + cluster: + main: + args: *args + log_file: "batched-procedures-disk-e2e.log" + setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + validation_queries: [] + workloads: - name: "Batched procedures read" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/batched_procedures/procedures/" args: ["batched_procedures/simple_read.py"] - <<: *template_cluster + <<: *in_memory_cluster + - name: "Disk batched procedures read" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/batched_procedures/procedures/" + args: ["batched_procedures/simple_read.py"] + <<: *disk_cluster diff --git a/tests/e2e/configuration/workloads.yaml b/tests/e2e/configuration/workloads.yaml index edead4eef..7894b5ba3 100644 --- a/tests/e2e/configuration/workloads.yaml +++ b/tests/e2e/configuration/workloads.yaml @@ -1,25 +1,38 @@ -template_cluster: &template_cluster +args: &args + - "--log-level=TRACE" + - "--storage-properties-on-edges=True" + - "--storage-snapshot-interval-sec" + - "300" + - "--storage-wal-enabled=True" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: - [ - "--log-level=TRACE", - "--storage-properties-on-edges=True", - "--storage-snapshot-interval-sec", - "300", - "--storage-wal-enabled=True", - ] + args: *args log_file: "configuration-check-e2e.log" setup_queries: [] validation_queries: [] +disk_cluster: &disk_cluster + cluster: + main: + args: *args + log_file: "configuration-check-disk-e2e.log" + setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + validation_queries: [] + workloads: - name: "Configuration check" binary: "tests/e2e/pytest_runner.sh" args: ["configuration/configuration_check.py"] - <<: *template_cluster + <<: *in_memory_cluster - name: "SHOW STORAGE INFO check" binary: "tests/e2e/pytest_runner.sh" args: ["configuration/storage_info.py"] - <<: *template_cluster + <<: *in_memory_cluster + + - name: "Disk configuration check" + binary: "tests/e2e/pytest_runner.sh" + args: ["configuration/configuration_check.py"] + <<: *disk_cluster diff --git a/tests/e2e/fine_grained_access/workloads.yaml b/tests/e2e/fine_grained_access/workloads.yaml index aed6eb8bb..ad1dd43b2 100644 --- a/tests/e2e/fine_grained_access/workloads.yaml +++ b/tests/e2e/fine_grained_access/workloads.yaml @@ -1,135 +1,187 @@ -bolt_port: &bolt_port "7687" -create_delete_filtering_cluster: &create_delete_filtering_cluster - cluster: - main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] - log_file: "fine_grained_access.log" - setup_queries: - [ - "CREATE USER admin IDENTIFIED BY 'test';", - "CREATE USER user IDENTIFIED BY 'test';", - "GRANT ALL PRIVILEGES TO admin;", - "GRANT DATABASE * TO admin;", - "GRANT ALL PRIVILEGES TO user;", - "GRANT DATABASE * TO user;", - ] +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" -edge_type_filtering_cluster: &edge_type_filtering_cluster +create_delete_filtering_setup_queries: &create_delete_filtering_setup_queries + - "CREATE USER admin IDENTIFIED BY 'test';" + - "CREATE USER user IDENTIFIED BY 'test';" + - "GRANT ALL PRIVILEGES TO admin;" + - "GRANT DATABASE * TO admin;" + - "GRANT ALL PRIVILEGES TO user;" + - "GRANT DATABASE * TO user;" + +edge_type_filtering_setup_queries: &edge_type_filtering_setup_queries + - "CREATE USER admin IDENTIFIED BY 'test';" + - "CREATE USER user IDENTIFIED BY 'test';" + - "GRANT ALL PRIVILEGES TO admin;" + - "GRANT DATABASE * TO admin;" + - "GRANT ALL PRIVILEGES TO user;" + - "GRANT DATABASE * TO user;" + - "GRANT CREATE_DELETE ON LABELS * TO admin;" + - "GRANT CREATE_DELETE ON EDGE_TYPES * TO admin;" + - "MERGE (l1:label1 {name: 'test1'});" + - "MERGE (l2:label2 {name: 'test2'});" + - "MATCH (l1:label1),(l2:label2) WHERE l1.name = 'test1' AND l2.name = 'test2' CREATE (l1)-[r:edgeType1]->(l2);" + - "MERGE (l3:label3 {name: 'test3'});" + - "MATCH (l1:label1),(l3:label3) WHERE l1.name = 'test1' AND l3.name = 'test3' CREATE (l1)-[r:edgeType2]->(l3);" + - "MERGE (mix:label3:label1 {name: 'test4'});" + - "MATCH (l1:label1),(mix:label3) WHERE l1.name = 'test1' AND mix.name = 'test4' CREATE (l1)-[r:edgeType2]->(mix);" + - "CREATE DATABASE clean;" + - "USE DATABASE clean" + - "MATCH (n) DETACH DELETE n;" + - "MERGE (l1:label1 {name: 'test1'});" + - "MERGE (l2:label2 {name: 'test2'});" + - "MATCH (l1:label1),(l2:label2) WHERE l1.name = 'test1' AND l2.name = 'test2' CREATE (l1)-[r:edgeType1]->(l2);" + - "MERGE (l3:label3 {name: 'test3'});" + - "MATCH (l1:label1),(l3:label3) WHERE l1.name = 'test1' AND l3.name = 'test3' CREATE (l1)-[r:edgeType2]->(l3);" + - "MERGE (mix:label3:label1 {name: 'test4'});" + - "MATCH (l1:label1),(mix:label3) WHERE l1.name = 'test1' AND mix.name = 'test4' CREATE (l1)-[r:edgeType2]->(mix);" + - "USE DATABASE memgraph" + +path_filtering_setup_queries: &path_filtering_setup_queries + - "CREATE USER admin IDENTIFIED BY 'test';" + - "CREATE USER user IDENTIFIED BY 'test';" + - "GRANT ALL PRIVILEGES TO admin;" + - "GRANT DATABASE * TO admin;" + - "GRANT ALL PRIVILEGES TO user;" + - "GRANT DATABASE * TO user;" + - "MERGE (a:label0 {id: 0}) MERGE (b:label1 {id: 1}) CREATE (a)-[:edge_type_1 {weight: 6}]->(b);" + - "MERGE (a:label0 {id: 0}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_1 {weight: 14}]->(b);" + - "MERGE (a:label1 {id: 1}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_2 {weight: 1}]->(b);" + - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_2 {weight: 10}]->(b);" + - "MERGE (a:label1 {id: 1}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 5}]->(b);" + - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 7}]->(b);" + - "MERGE (a:label3 {id: 3}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);" + - "MERGE (a:label3 {id: 4}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);" + - "MERGE (a:label3 {id: 3}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 14}]->(b);" + - "MERGE (a:label3 {id: 4}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 8}]->(b);" + - "CREATE DATABASE clean;" + - "USE DATABASE clean" + - "MATCH (n) DETACH DELETE n;" + - "MERGE (a:label0 {id: 0}) MERGE (b:label1 {id: 1}) CREATE (a)-[:edge_type_1 {weight: 6}]->(b);" + - "MERGE (a:label0 {id: 0}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_1 {weight: 14}]->(b);" + - "MERGE (a:label1 {id: 1}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_2 {weight: 1}]->(b);" + - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_2 {weight: 10}]->(b);" + - "MERGE (a:label1 {id: 1}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 5}]->(b);" + - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 7}]->(b);" + - "MERGE (a:label3 {id: 3}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);" + - "MERGE (a:label3 {id: 4}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);" + - "MERGE (a:label3 {id: 3}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 14}]->(b);" + - "MERGE (a:label3 {id: 4}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 8}]->(b);" + - "USE DATABASE memgraph" + +show_databases_w_user_setup_queries: &show_databases_w_user_setup_queries + - "CREATE USER admin IDENTIFIED BY 'test';" + - "CREATE USER user IDENTIFIED BY 'test';" + - "CREATE USER user2 IDENTIFIED BY 'test';" + - "CREATE USER user3 IDENTIFIED BY 'test';" + - "CREATE DATABASE db1;" + - "CREATE DATABASE db2;" + - "GRANT ALL PRIVILEGES TO admin;" + - "GRANT DATABASE * TO admin;" + - "GRANT ALL PRIVILEGES TO user;" + - "GRANT DATABASE db1 TO user;" + - "GRANT ALL PRIVILEGES TO user2;" + - "GRANT DATABASE db2 TO user2;" + - "REVOKE DATABASE memgraph FROM user2;" + - "SET MAIN DATABASE db2 FOR user2" + - "GRANT ALL PRIVILEGES TO user3;" + - "GRANT DATABASE * TO user3;" + - "REVOKE DATABASE memgraph FROM user3;" + - "SET MAIN DATABASE db1 FOR user3" + +create_delete_filtering_in_memory_cluster: &create_delete_filtering_in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "fine_grained_access.log" - setup_queries: - [ - "CREATE USER admin IDENTIFIED BY 'test';", - "CREATE USER user IDENTIFIED BY 'test';", - "GRANT ALL PRIVILEGES TO admin;", - "GRANT DATABASE * TO admin;", - "GRANT ALL PRIVILEGES TO user;", - "GRANT DATABASE * TO user;", - "GRANT CREATE_DELETE ON LABELS * TO admin;", - "GRANT CREATE_DELETE ON EDGE_TYPES * TO admin;", - "MERGE (l1:label1 {name: 'test1'});", - "MERGE (l2:label2 {name: 'test2'});", - "MATCH (l1:label1),(l2:label2) WHERE l1.name = 'test1' AND l2.name = 'test2' CREATE (l1)-[r:edgeType1]->(l2);", - "MERGE (l3:label3 {name: 'test3'});", - "MATCH (l1:label1),(l3:label3) WHERE l1.name = 'test1' AND l3.name = 'test3' CREATE (l1)-[r:edgeType2]->(l3);", - "MERGE (mix:label3:label1 {name: 'test4'});", - "MATCH (l1:label1),(mix:label3) WHERE l1.name = 'test1' AND mix.name = 'test4' CREATE (l1)-[r:edgeType2]->(mix);", - "CREATE DATABASE clean;", - "USE DATABASE clean", - "MATCH (n) DETACH DELETE n;", - "MERGE (l1:label1 {name: 'test1'});", - "MERGE (l2:label2 {name: 'test2'});", - "MATCH (l1:label1),(l2:label2) WHERE l1.name = 'test1' AND l2.name = 'test2' CREATE (l1)-[r:edgeType1]->(l2);", - "MERGE (l3:label3 {name: 'test3'});", - "MATCH (l1:label1),(l3:label3) WHERE l1.name = 'test1' AND l3.name = 'test3' CREATE (l1)-[r:edgeType2]->(l3);", - "MERGE (mix:label3:label1 {name: 'test4'});", - "MATCH (l1:label1),(mix:label3) WHERE l1.name = 'test1' AND mix.name = 'test4' CREATE (l1)-[r:edgeType2]->(mix);", - "USE DATABASE memgraph", - ] + setup_queries: *create_delete_filtering_setup_queries validation_queries: [] -path_filtering_cluster: &path_filtering_cluster +create_delete_filtering_disk_cluster: &create_delete_filtering_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] - log_file: "fine_grained_access.log" + args: *args + log_file: "disk_fine_grained_access.log" setup_queries: - [ - "CREATE USER admin IDENTIFIED BY 'test';", - "CREATE USER user IDENTIFIED BY 'test';", - "GRANT ALL PRIVILEGES TO admin;", - "GRANT DATABASE * TO admin;", - "GRANT ALL PRIVILEGES TO user;", - "GRANT DATABASE * TO user;", - "MERGE (a:label0 {id: 0}) MERGE (b:label1 {id: 1}) CREATE (a)-[:edge_type_1 {weight: 6}]->(b);", - "MERGE (a:label0 {id: 0}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_1 {weight: 14}]->(b);", - "MERGE (a:label1 {id: 1}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_2 {weight: 1}]->(b);", - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_2 {weight: 10}]->(b);", - "MERGE (a:label1 {id: 1}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 5}]->(b);", - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 7}]->(b);", - "MERGE (a:label3 {id: 3}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);", - "MERGE (a:label3 {id: 4}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);", - "MERGE (a:label3 {id: 3}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 14}]->(b);", - "MERGE (a:label3 {id: 4}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 8}]->(b);", - "CREATE DATABASE clean;", - "USE DATABASE clean", - "MATCH (n) DETACH DELETE n;", - "MERGE (a:label0 {id: 0}) MERGE (b:label1 {id: 1}) CREATE (a)-[:edge_type_1 {weight: 6}]->(b);", - "MERGE (a:label0 {id: 0}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_1 {weight: 14}]->(b);", - "MERGE (a:label1 {id: 1}) MERGE (b:label2 {id: 2}) CREATE (a)-[:edge_type_2 {weight: 1}]->(b);", - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_2 {weight: 10}]->(b);", - "MERGE (a:label1 {id: 1}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 5}]->(b);", - "MERGE (a:label2 {id: 2}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_3 {weight: 7}]->(b);", - "MERGE (a:label3 {id: 3}) MERGE (b:label3 {id: 4}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);", - "MERGE (a:label3 {id: 4}) MERGE (b:label3 {id: 3}) CREATE (a)-[:edge_type_4 {weight: 1}]->(b);", - "MERGE (a:label3 {id: 3}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 14}]->(b);", - "MERGE (a:label3 {id: 4}) MERGE (b:label4 {id: 5}) CREATE (a)-[:edge_type_4 {weight: 8}]->(b);", - "USE DATABASE memgraph", - ] + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *create_delete_filtering_setup_queries + validation_queries: [] + +edge_type_filtering_in_memory_cluster: &edge_type_filtering_in_memory_cluster + cluster: + main: + args: *args + log_file: "fine_grained_access.log" + setup_queries: *edge_type_filtering_setup_queries + validation_queries: [] + +edge_type_filtering_disk_cluster: &edge_type_filtering_disk_cluster + cluster: + main: + args: *args + log_file: "disk_fine_grained_access.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *edge_type_filtering_setup_queries + validation_queries: [] + +path_filtering_in_memory_cluster: &path_filtering_in_memory_cluster + cluster: + main: + args: *args + log_file: "fine_grained_access.log" + setup_queries: *path_filtering_setup_queries + +path_filtering_disk_cluster: &path_filtering_disk_cluster + cluster: + main: + args: *args + log_file: "disk_fine_grained_access.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *path_filtering_setup_queries show_databases_w_user: &show_databases_w_user cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "fine_grained_access.log" - setup_queries: - [ - "CREATE USER admin IDENTIFIED BY 'test';", - "CREATE USER user IDENTIFIED BY 'test';", - "CREATE USER user2 IDENTIFIED BY 'test';", - "CREATE USER user3 IDENTIFIED BY 'test';", - "CREATE DATABASE db1;", - "CREATE DATABASE db2;", - "GRANT ALL PRIVILEGES TO admin;", - "GRANT DATABASE * TO admin;", - "GRANT ALL PRIVILEGES TO user;", - "GRANT DATABASE db1 TO user;", - "GRANT ALL PRIVILEGES TO user2;", - "GRANT DATABASE db2 TO user2;", - "REVOKE DATABASE memgraph FROM user2;", - "SET MAIN DATABASE db2 FOR user2", - "GRANT ALL PRIVILEGES TO user3;", - "GRANT DATABASE * TO user3;", - "REVOKE DATABASE memgraph FROM user3;", - "SET MAIN DATABASE db1 FOR user3", - ] + setup_queries: *show_databases_w_user_setup_queries workloads: - name: "Create delete filtering" binary: "tests/e2e/pytest_runner.sh" args: ["fine_grained_access/create_delete_filtering_tests.py"] - <<: *create_delete_filtering_cluster + <<: *create_delete_filtering_in_memory_cluster + + - name: "Create delete filtering on disk" + binary: "tests/e2e/pytest_runner.sh" + args: ["fine_grained_access/create_delete_filtering_tests.py"] + <<: *create_delete_filtering_disk_cluster + - name: "EdgeType filtering" binary: "tests/e2e/pytest_runner.sh" args: ["fine_grained_access/edge_type_filtering_tests.py"] - <<: *edge_type_filtering_cluster + <<: *edge_type_filtering_in_memory_cluster + + - name: "EdgeType filtering on disk" + binary: "tests/e2e/pytest_runner.sh" + args: ["fine_grained_access/edge_type_filtering_tests.py"] + <<: *edge_type_filtering_disk_cluster + - name: "Path filtering" binary: "tests/e2e/pytest_runner.sh" args: ["fine_grained_access/path_filtering_tests.py"] - <<: *path_filtering_cluster + <<: *path_filtering_in_memory_cluster + - name: "Show databases with users" binary: "tests/e2e/pytest_runner.sh" args: ["fine_grained_access/show_db.py"] <<: *show_databases_w_user + + - name: "Path filtering on disk" + binary: "tests/e2e/pytest_runner.sh" + args: ["fine_grained_access/path_filtering_tests.py"] + <<: *path_filtering_disk_cluster diff --git a/tests/e2e/init_file_flags/workloads.yaml b/tests/e2e/init_file_flags/workloads.yaml index 85ae169fa..35ac28c1a 100644 --- a/tests/e2e/init_file_flags/workloads.yaml +++ b/tests/e2e/init_file_flags/workloads.yaml @@ -1,22 +1,26 @@ -init_file_cluster: &init_file_cluster +init_file_args: &init_file_args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--init-file=init_file_flags/init_file.cypherl" + +init_data_file_args: &init_data_file_args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--init-data-file=init_file_flags/init_file.cypherl" + +init_file_in_memory_cluster: &init_file_in_memory_cluster cluster: main: - args: [ - "--bolt-port", "7687", - "--log-level=TRACE", - "--init-file=init_file_flags/init_file.cypherl" - ] + args: *init_file_args log_file: "init-file-flags-e2e.log" validation_queries: [] -init_data_file_cluster: &init_data_file_cluster +init_data_file_in_memory_cluster: &init_data_file_in_memory_cluster cluster: main: - args: [ - "--bolt-port", "7687", - "--log-level=TRACE", - "--init-data-file=init_file_flags/init_file.cypherl" - ] + args: *init_data_file_args log_file: "init-data-file-flags-e2e.log" validation_queries: [] @@ -24,9 +28,9 @@ workloads: - name: "Init file flags" binary: "tests/e2e/pytest_runner.sh" args: ["init_file_flags/init_file_setup.py"] - <<: *init_file_cluster + <<: *init_file_in_memory_cluster - name: "Init data file flags" binary: "tests/e2e/pytest_runner.sh" args: ["init_file_flags/init_data_file_setup.py"] - <<: *init_data_file_cluster + <<: *init_data_file_in_memory_cluster diff --git a/tests/e2e/isolation_levels/workloads.yaml b/tests/e2e/isolation_levels/workloads.yaml index cca4693ab..b1d04a9ff 100644 --- a/tests/e2e/isolation_levels/workloads.yaml +++ b/tests/e2e/isolation_levels/workloads.yaml @@ -1,24 +1,31 @@ bolt_port: &bolt_port "7687" -template_cluster: &template_cluster + +args: &args + - "--bolt-port" + - *bolt_port + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "isolation-levels-e2e.log" setup_queries: [] validation_queries: [] + disk_cluster: &disk_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "isolation-levels-disk-e2e.log" - setup_queries: ["storage mode on_disk_transactional"] + setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] validation_queries: [] workloads: - name: "Isolation levels" binary: "tests/e2e/isolation_levels/memgraph__e2e__isolation_levels" args: ["--bolt-port", *bolt_port] - <<: *template_cluster + <<: *in_memory_cluster - name: "Isolation levels for disk storage" binary: "tests/e2e/isolation_levels/memgraph__e2e__isolation_levels" diff --git a/tests/e2e/lba_procedures/workloads.yaml b/tests/e2e/lba_procedures/workloads.yaml index d7de0de2a..c6dce9ac3 100644 --- a/tests/e2e/lba_procedures/workloads.yaml +++ b/tests/e2e/lba_procedures/workloads.yaml @@ -1,108 +1,152 @@ -read_query_modules_cluster: &read_query_modules_cluster +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + +query_modules_setup_queries: &query_modules_setup_queries + - "CREATE USER admin IDENTIFIED BY 'test';" + - "GRANT ALL PRIVILEGES TO admin" + - "GRANT DATABASE * TO admin" + - "CREATE USER user IDENTIFIED BY 'test';" + - "GRANT ALL PRIVILEGES TO user" + - "GRANT DATABASE * TO user" + +show_privileges_setup_queries: &show_privileges_setup_queries + - "Create User Josip;" + - "Grant Read On Labels :Label1 to Josip;" + - "Grant Nothing On Labels :Label2 to Josip;" + - "Grant Update On Labels :Label3 to Josip;" + - "Grant Read On Labels :Label4 to Josip;" + - "Grant Create_Delete On Labels :Label5 to Josip;" + - "Grant Update On Labels :Label6 to Josip;" + - "Grant Create_Delete On Labels :Label7 to Josip;" + - "Grant Nothing On Labels :Label7 to Josip;" + - "Create User Boris;" + - "Grant Auth to Boris;" + - "Grant Read On Labels :Label1 to Boris;" + - "Grant Nothing On Labels :Label2 to Boris;" + - "Grant Update On Labels :Label3 to Boris;" + - "Grant Read On Labels :Label4 to Boris;" + - "Grant Create_Delete On Labels :Label5 to Boris;" + - "Grant Update On Labels :Label6 to Boris;" + - "Grant Create_Delete On Labels :Label7 to Boris;" + - "Grant Nothing On Labels :Label7 to Boris;" + - "Create User Niko;" + - "Grant Auth to Niko;" + - "Grant Create_Delete On Labels * to Niko;" + - "Grant Read On Labels * to Niko;" + - "Create User Bruno;" + - "Grant Auth to Bruno;" + - "Grant Update On Labels * to Bruno;" + + +read_query_modules_in_memory_cluster: &read_query_modules_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "lba-e2e.log" - setup_queries: - - "CREATE USER admin IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO admin" - - "GRANT DATABASE * TO admin" - - "CREATE USER user IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO user" - - "GRANT DATABASE * TO user" + setup_queries: *query_modules_setup_queries validation_queries: [] -update_query_modules_cluster: &update_query_modules_cluster +read_query_modules_disk_cluster: &read_query_modules_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] - log_file: "lba-e2e.log" + args: *args + log_file: "disk-lba-e2e.log" setup_queries: - - "CREATE USER admin IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO admin" - - "GRANT DATABASE * TO admin" - - "CREATE USER user IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO user" - - "GRANT DATABASE * TO user" + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *query_modules_setup_queries validation_queries: [] -show_privileges_cluster: &show_privileges_cluster +update_query_modules_in_memory_cluster: &update_query_modules_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "lba-e2e.log" - setup_queries: - - "Create User Josip;" - - "Grant Read On Labels :Label1 to Josip;" - - "Grant Nothing On Labels :Label2 to Josip;" - - "Grant Update On Labels :Label3 to Josip;" - - "Grant Read On Labels :Label4 to Josip;" - - "Grant Create_Delete On Labels :Label5 to Josip;" - - "Grant Update On Labels :Label6 to Josip;" - - "Grant Create_Delete On Labels :Label7 to Josip;" - - "Grant Nothing On Labels :Label7 to Josip;" - - - "Create User Boris;" - - "Grant Auth to Boris;" - - "Grant Read On Labels :Label1 to Boris;" - - "Grant Nothing On Labels :Label2 to Boris;" - - "Grant Update On Labels :Label3 to Boris;" - - "Grant Read On Labels :Label4 to Boris;" - - "Grant Create_Delete On Labels :Label5 to Boris;" - - "Grant Update On Labels :Label6 to Boris;" - - "Grant Create_Delete On Labels :Label7 to Boris;" - - "Grant Nothing On Labels :Label7 to Boris;" - - - "Create User Niko;" - - "Grant Auth to Niko;" - - "Grant Create_Delete On Labels * to Niko" - - "Grant Read On Labels * to Niko" - - - "Create User Bruno;" - - "Grant Auth to Bruno;" - - "Grant Update On Labels * to Bruno" + setup_queries: *query_modules_setup_queries validation_queries: [] -read_permission_queries: &read_permission_queries +update_query_modules_disk_cluster: &update_query_modules_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] - log_file: "lba-e2e.log" + args: *args + log_file: "disk-lba-e2e.log" setup_queries: - - "CREATE USER admin IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO admin" - - "GRANT DATABASE * TO admin" - - "CREATE USER user IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO user" - - "GRANT DATABASE * TO user" + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *query_modules_setup_queries validation_queries: [] -create_delete_query_modules_cluster: &create_delete_query_modules_cluster +show_privileges_in_memory_cluster: &show_privileges_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "lba-e2e.log" - setup_queries: - - "CREATE USER admin IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO admin;" - - "GRANT DATABASE * TO admin" - - "CREATE USER user IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO user;" - - "GRANT DATABASE * TO user" + setup_queries: *show_privileges_setup_queries validation_queries: [] -update_permission_queries_cluster: &update_permission_queries_cluster +show_privileges_disk_cluster: &show_privileges_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] - log_file: "lba-e2e.log" + args: *args + log_file: "disk-lba-e2e.log" setup_queries: - - "CREATE USER admin IDENTIFIED BY 'test';" - - "GRANT ALL PRIVILEGES TO admin;" - - "GRANT DATABASE * TO admin" - - "CREATE USER user IDENTIFIED BY 'test'" - - "GRANT ALL PRIVILEGES TO user;" - - "GRANT DATABASE * TO user" + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *show_privileges_setup_queries + validation_queries: [] + +read_permission_in_memory_queries: &read_permission_in_memory_queries + cluster: + main: + args: *args + log_file: "lba-e2e.log" + setup_queries: *query_modules_setup_queries + validation_queries: [] + +read_permission_disk_queries: &read_permission_disk_queries + cluster: + main: + args: *args + log_file: "disk-lba-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *query_modules_setup_queries + validation_queries: [] + + +create_delete_query_modules_in_memory_cluster: &create_delete_query_modules_in_memory_cluster + cluster: + main: + args: *args + log_file: "lba-e2e.log" + setup_queries: *query_modules_setup_queries + validation_queries: [] + +create_delete_query_modules_disk_cluster: &create_delete_query_modules_disk_cluster + cluster: + main: + args: *args + log_file: "disk-lba-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *query_modules_setup_queries + validation_queries: [] + +update_permission_queries_in_memory_cluster: &update_permission_queries_in_memory_cluster + cluster: + main: + args: *args + log_file: "lba-e2e.log" + setup_queries: *query_modules_setup_queries + validation_queries: [] + +update_permission_queries_disk_cluster: &update_permission_queries_disk_cluster + cluster: + main: + args: *args + log_file: "disk-lba-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *query_modules_setup_queries validation_queries: [] workloads: @@ -110,34 +154,70 @@ workloads: binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/read_query_modules.py"] - <<: *read_query_modules_cluster + <<: *read_query_modules_in_memory_cluster + + - name: "read-query-modules on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/read_query_modules.py"] + <<: *read_query_modules_disk_cluster - name: "update-query-modules" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/update_query_modules.py"] - <<: *update_query_modules_cluster + <<: *update_query_modules_in_memory_cluster + + - name: "update-query-modules on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/update_query_modules.py"] + <<: *update_query_modules_disk_cluster - name: "create-delete-query-modules" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/create_delete_query_modules.py"] - <<: *create_delete_query_modules_cluster + <<: *create_delete_query_modules_in_memory_cluster + + - name: "create-delete-query-modules on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/create_delete_query_modules.py"] + <<: *create_delete_query_modules_disk_cluster - name: "show-privileges" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/show_privileges.py"] - <<: *show_privileges_cluster + <<: *show_privileges_in_memory_cluster + + - name: "show-privileges on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/show_privileges.py"] + <<: *show_privileges_disk_cluster - name: "read-permission-queries" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/read_permission_queries.py"] - <<: *read_permission_queries + <<: *read_permission_in_memory_queries + + - name: "read-permission-queries on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/read_permission_queries.py"] + <<: *read_permission_disk_queries - name: "update-permission-queries" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/lba_procedures/procedures/" args: ["lba_procedures/update_permission_queries.py"] - <<: *update_permission_queries_cluster + <<: *update_permission_queries_in_memory_cluster + + - name: "update-permission-queries on disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/lba_procedures/procedures/" + args: ["lba_procedures/update_permission_queries.py"] + <<: *update_permission_queries_disk_cluster diff --git a/tests/e2e/load_csv/workloads.yaml b/tests/e2e/load_csv/workloads.yaml index d07609699..15e615538 100644 --- a/tests/e2e/load_csv/workloads.yaml +++ b/tests/e2e/load_csv/workloads.yaml @@ -1,16 +1,42 @@ -nullif_cluster: &nullif_cluster +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + +nullif_in_memory_cluster: &nullif_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "load_csv_log_file.txt" + setup_queries: [] validation_queries: [] -load_csv_cluster: &load_csv_cluster +nullif_disk_cluster: &nullif_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "load_csv_log_file.txt" setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + validation_queries: [] + +load_csv_in_memory_cluster: &load_csv_in_memory_cluster + cluster: + main: + args: *args + log_file: "load_csv_log_file.txt" + setup_queries: + - "CREATE (n {prop: 1});" + - "CREATE (n {prop: 2});" + validation_queries: [] + +load_csv_disk_cluster: &load_csv_disk_cluster + cluster: + main: + args: *args + log_file: "load_csv_log_file.txt" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" - "CREATE (n {prop: 1});" - "CREATE (n {prop: 2});" validation_queries: [] @@ -19,8 +45,19 @@ workloads: - name: "LOAD CSV nullif" binary: "tests/e2e/pytest_runner.sh" args: ["load_csv/load_csv_nullif.py"] - <<: *nullif_cluster + <<: *nullif_in_memory_cluster + + - name: "LOAD CSV nullif on disk" + binary: "tests/e2e/pytest_runner.sh" + args: ["load_csv/load_csv_nullif.py"] + <<: *nullif_disk_cluster + - name: "MATCH + LOAD CSV" binary: "tests/e2e/pytest_runner.sh" args: ["load_csv/load_csv.py"] - <<: *load_csv_cluster + <<: *load_csv_in_memory_cluster + + - name: "MATCH + LOAD CSV on disk" + binary: "tests/e2e/pytest_runner.sh" + args: ["load_csv/load_csv.py"] + <<: *load_csv_disk_cluster diff --git a/tests/e2e/magic_functions/workloads.yaml b/tests/e2e/magic_functions/workloads.yaml index a2346b373..36c7b9cc2 100644 --- a/tests/e2e/magic_functions/workloads.yaml +++ b/tests/e2e/magic_functions/workloads.yaml @@ -1,14 +1,20 @@ -template_cluster: &template_cluster +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "magic-functions-e2e.log" setup_queries: [] validation_queries: [] + disk_cluster: &disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "magic-functions-e2e.log" setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] validation_queries: [] @@ -18,7 +24,7 @@ workloads: binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/magic_functions/functions/" args: ["magic_functions/function_example.py"] - <<: *template_cluster + <<: *in_memory_cluster - name: "Magic functions runner for disk storage" binary: "tests/e2e/pytest_runner.sh" diff --git a/tests/e2e/memgraph.py b/tests/e2e/memgraph.py index 2c7546292..dff0f1d49 100755 --- a/tests/e2e/memgraph.py +++ b/tests/e2e/memgraph.py @@ -70,8 +70,12 @@ class MemgraphInstanceRunner: conn = mgclient.connect(host=self.host, port=self.bolt_port, sslmode=self.ssl) conn.autocommit = True cursor = conn.cursor() - for query in setup_queries: - cursor.execute(query) + for query_coll in setup_queries: + if isinstance(query_coll, str): + cursor.execute(query_coll) + elif isinstance(query_coll, list): + for query in query_coll: + cursor.execute(query) cursor.close() conn.close() diff --git a/tests/e2e/memory/workloads.yaml b/tests/e2e/memory/workloads.yaml index 64e45979d..667613114 100644 --- a/tests/e2e/memory/workloads.yaml +++ b/tests/e2e/memory/workloads.yaml @@ -1,17 +1,26 @@ bolt_port: &bolt_port "7687" -template_cluster: &template_cluster +args: &args + - "--bolt-port" + - *bolt_port + - "--memory-limit=1000" + - "--storage-gc-cycle-sec=180" + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"] + args: *args log_file: "memory-e2e.log" setup_queries: [] validation_queries: [] + disk_cluster: &disk_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"] + args: *args log_file: "memory-e2e.log" - setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] @@ -19,33 +28,45 @@ workloads: - name: "Memory control" binary: "tests/e2e/memory/memgraph__e2e__memory__control" args: ["--bolt-port", *bolt_port, "--timeout", "180"] - <<: *template_cluster + <<: *in_memory_cluster - name: "Memory control multi database" binary: "tests/e2e/memory/memgraph__e2e__memory__control" args: ["--bolt-port", *bolt_port, "--timeout", "180", "--multi-db", "true"] - <<: *template_cluster + <<: *in_memory_cluster - name: "Memory limit for modules upon loading" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc" args: ["--bolt-port", *bolt_port, "--timeout", "180"] proc: "tests/e2e/memory/procedures/" - <<: *template_cluster + <<: *in_memory_cluster - name: "Memory limit for modules upon loading multi database" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc" args: ["--bolt-port", *bolt_port, "--timeout", "180", "--multi-db", "true"] proc: "tests/e2e/memory/procedures/" - <<: *template_cluster + <<: *in_memory_cluster - name: "Memory limit for modules inside a procedure" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc_proc" args: ["--bolt-port", *bolt_port, "--timeout", "180"] proc: "tests/e2e/memory/procedures/" - <<: *template_cluster + <<: *in_memory_cluster - name: "Memory limit for modules inside a procedure multi database" binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc_proc" args: ["--bolt-port", *bolt_port, "--timeout", "180", "--multi-db", "true"] proc: "tests/e2e/memory/procedures/" - <<: *template_cluster + <<: *in_memory_cluster + + - name: "Memory limit for modules upon loading for on-disk storage" + binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc" + args: ["--bolt-port", *bolt_port, "--timeout", "180"] + proc: "tests/e2e/memory/procedures/" + <<: *disk_cluster + + - name: "Memory limit for modules inside a procedure for on-disk storage" + binary: "tests/e2e/memory/memgraph__e2e__memory__limit_global_alloc_proc" + args: ["--bolt-port", *bolt_port, "--timeout", "180"] + proc: "tests/e2e/memory/procedures/" + <<: *disk_cluster diff --git a/tests/e2e/mock_api/workloads.yaml b/tests/e2e/mock_api/workloads.yaml index f4b638a11..a3e0d8bb0 100644 --- a/tests/e2e/mock_api/workloads.yaml +++ b/tests/e2e/mock_api/workloads.yaml @@ -1,166 +1,108 @@ -compare_mock: &compare_mock +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--also-log-to-stderr" + +mock_setup_queries: &mock_setup_queries + - "CREATE INDEX ON :__mg_vertex__(__mg_id__);" + - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 0, `name`: 'Peter', `surname`: 'Yang'});" + - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 1, `name`: 'Engineering'});" + - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 2, `name`: 'Memgraph'});" + - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 3, `name`: 'MAGE'});" + - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 4, `name`: 'GQLAlchemy'});" + - "CREATE (:__mg_vertex__:`Company`:`Startup` {__mg_id__: 5, `name`: 'Memgraph'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 6, `name`: 'welcome_to_engineering.txt'});" + - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 7, `name`: 'Google Drive'});" + - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 8, `name`: 'Notion'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 9, `name`: 'welcome_to_memgraph.txt'});" + - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 10, `name`: 'Carl'});" + - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 11, `name`: 'engineering_folder'});" + - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 12, `name`: 'Anna'});" + - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 13, `name`: 'operations_folder'});" + - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 14, `name`: 'Operations'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 15, `name`: 'operations101.txt'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 16, `name`: 'expenses2022.csv'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 17, `name`: 'salaries2022.csv'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 18, `name`: 'engineering101.txt'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 19, `name`: 'working_with_github.txt'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 20, `name`: 'working_with_notion.txt'});" + - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 21, `name`: 'Marketing'});" + - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 22, `name`: 'Julie'});" + - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 23, `name`: 'Facebook'});" + - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 24, `name`: 'LinkedIn'});" + - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 25, `name`: 'HackerNews'});" + - "CREATE (:__mg_vertex__:`File` {__mg_id__: 26, `name`: 'welcome_to_marketing.txt'});" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 0}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 1}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 2}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 3}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 4}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 5}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 4 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 6}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 6 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 7}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 11 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 8}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 9}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 21 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 10}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 14 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 11}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 12}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 13}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:`CREATED_BY` {`permanent_id`: 14}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 15}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 16}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 17}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 7 CREATE(u)-[:`IS_STORED_IN` {`permanent_id`: 18}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 18 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 19}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 19 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 20}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 20 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 21}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 22}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 15 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 23}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 16 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 24}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 17 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 25}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 26}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 27}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 23 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 28}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 24 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 29}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 25 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 30}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 26 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 31}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 21 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 32}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 33}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 34}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 35}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 36}]->(v);" + - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM_2` {`importance`: 'HIGH', `permanent_id`: 37}]->(v);" + - "DROP INDEX ON :__mg_vertex__(__mg_id__);" + - "MATCH (u) SET u.permanent_id = u.__mg_id__;" + - "MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;" + + +compare_mock_in_memory_cluster: &compare_mock_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr"] + args: *args log_file: "test-compare-mock-e2e.log" - setup_queries: - - "CREATE INDEX ON :__mg_vertex__(__mg_id__);" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 0, `name`: 'Peter', `surname`: 'Yang'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 1, `name`: 'Engineering'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 2, `name`: 'Memgraph'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 3, `name`: 'MAGE'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 4, `name`: 'GQLAlchemy'});" - - "CREATE (:__mg_vertex__:`Company`:`Startup` {__mg_id__: 5, `name`: 'Memgraph'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 6, `name`: 'welcome_to_engineering.txt'});" - - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 7, `name`: 'Google Drive'});" - - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 8, `name`: 'Notion'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 9, `name`: 'welcome_to_memgraph.txt'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 10, `name`: 'Carl'});" - - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 11, `name`: 'engineering_folder'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 12, `name`: 'Anna'});" - - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 13, `name`: 'operations_folder'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 14, `name`: 'Operations'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 15, `name`: 'operations101.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 16, `name`: 'expenses2022.csv'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 17, `name`: 'salaries2022.csv'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 18, `name`: 'engineering101.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 19, `name`: 'working_with_github.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 20, `name`: 'working_with_notion.txt'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 21, `name`: 'Marketing'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 22, `name`: 'Julie'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 23, `name`: 'Facebook'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 24, `name`: 'LinkedIn'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 25, `name`: 'HackerNews'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 26, `name`: 'welcome_to_marketing.txt'});" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 0}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 1}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 2}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 3}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 4}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 5}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 4 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 6}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 6 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 7}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 11 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 8}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 9}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 21 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 10}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 14 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 11}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 12}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 13}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:`CREATED_BY` {`permanent_id`: 14}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 15}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 16}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 17}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 7 CREATE(u)-[:`IS_STORED_IN` {`permanent_id`: 18}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 18 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 19}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 19 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 20}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 20 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 21}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 22}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 15 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 23}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 16 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 24}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 17 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 25}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 26}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 27}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 23 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 28}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 24 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 29}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 25 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 30}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 26 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 31}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 21 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 32}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 33}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 34}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 35}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 36}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM_2` {`importance`: 'HIGH', `permanent_id`: 37}]->(v);" - - "DROP INDEX ON :__mg_vertex__(__mg_id__);" - - "MATCH (u) SET u.permanent_id = u.__mg_id__;" - - "MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;" + setup_queries: *mock_setup_queries validation_queries: [] -disk_cluster: &disk_cluster + +compare_mock_disk_cluster: &compare_mock_disk_cluster cluster: main: args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr"] log_file: "test-compare-mock-e2e.log" setup_queries: - - "STORAGE MODE ON_DISK_TRANSACTIONAL;" - - "CREATE INDEX ON :__mg_vertex__(__mg_id__);" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 0, `name`: 'Peter', `surname`: 'Yang'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 1, `name`: 'Engineering'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 2, `name`: 'Memgraph'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 3, `name`: 'MAGE'});" - - "CREATE (:__mg_vertex__:`Repository` {__mg_id__: 4, `name`: 'GQLAlchemy'});" - - "CREATE (:__mg_vertex__:`Company`:`Startup` {__mg_id__: 5, `name`: 'Memgraph'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 6, `name`: 'welcome_to_engineering.txt'});" - - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 7, `name`: 'Google Drive'});" - - "CREATE (:__mg_vertex__:`Storage` {__mg_id__: 8, `name`: 'Notion'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 9, `name`: 'welcome_to_memgraph.txt'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 10, `name`: 'Carl'});" - - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 11, `name`: 'engineering_folder'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 12, `name`: 'Anna'});" - - "CREATE (:__mg_vertex__:`Folder` {__mg_id__: 13, `name`: 'operations_folder'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 14, `name`: 'Operations'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 15, `name`: 'operations101.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 16, `name`: 'expenses2022.csv'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 17, `name`: 'salaries2022.csv'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 18, `name`: 'engineering101.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 19, `name`: 'working_with_github.txt'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 20, `name`: 'working_with_notion.txt'});" - - "CREATE (:__mg_vertex__:`Team` {__mg_id__: 21, `name`: 'Marketing'});" - - "CREATE (:__mg_vertex__:`Person` {__mg_id__: 22, `name`: 'Julie'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 23, `name`: 'Facebook'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 24, `name`: 'LinkedIn'});" - - "CREATE (:__mg_vertex__:`Account` {__mg_id__: 25, `name`: 'HackerNews'});" - - "CREATE (:__mg_vertex__:`File` {__mg_id__: 26, `name`: 'welcome_to_marketing.txt'});" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 0}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 1}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 2}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 3}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 4}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 5}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 4 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 6}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 6 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 7}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 11 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 8}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 9}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 21 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 10}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 14 CREATE (u)-[:`HAS_TEAM` {`permanent_id`: 11}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 12}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 13}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:`CREATED_BY` {`permanent_id`: 14}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 1 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 15}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 16}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 17}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 7 CREATE(u)-[:`IS_STORED_IN` {`permanent_id`: 18}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 18 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 19}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 19 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 20}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 20 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 21}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 14 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 22}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 15 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 23}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 16 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 24}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 17 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 25}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 26}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 27}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 23 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 28}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 24 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 29}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 25 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 30}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 21 AND v.__mg_id__ = 26 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 31}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 21 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 32}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 5 CREATE (u)-[:`IS_PART_OF` {`permanent_id`: 33}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 22 AND v.__mg_id__ = 9 CREATE (u)-[:`HAS_ACCESS_TO` {`permanent_id`: 34}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 7 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 35}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 26 AND v.__mg_id__ = 8 CREATE (u)-[:`IS_STORED_IN` {`permanent_id`: 36}]->(v);" - - "MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 1 CREATE (u)-[:`HAS_TEAM_2` {`importance`: 'HIGH', `permanent_id`: 37}]->(v);" - - "DROP INDEX ON :__mg_vertex__(__mg_id__);" - - "MATCH (u) SET u.permanent_id = u.__mg_id__;" - - "MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;" + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + - *mock_setup_queries validation_queries: [] workloads: - - name: "test-compare-mock on disk" # should be the same as the python file + - name: "test-compare-mock" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/mock_api/procedures/" args: ["mock_api/test_compare_mock.py"] - <<: *disk_cluster + <<: *compare_mock_in_memory_cluster - - name: "test-compare-mock" # should be the same as the python file + - name: "test-compare-mock on disk" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/mock_api/procedures/" args: ["mock_api/test_compare_mock.py"] - <<: *compare_mock + <<: *compare_mock_disk_cluster diff --git a/tests/e2e/module_file_manager/workloads.yaml b/tests/e2e/module_file_manager/workloads.yaml index b40242e21..f34de0fea 100644 --- a/tests/e2e/module_file_manager/workloads.yaml +++ b/tests/e2e/module_file_manager/workloads.yaml @@ -1,19 +1,38 @@ bolt_port: &bolt_port "7687" -template_cluster: &template_cluster +args: &args + - "--bolt-port" + - *bolt_port + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "module-file-manager-e2e.log" setup_queries: [] validation_queries: [] +disk_cluster: &disk_cluster + cluster: + main: + args: *args + log_file: "module-file-manager-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + validation_queries: [] + workloads: - name: "Module File Manager" binary: "tests/e2e/module_file_manager/memgraph__e2e__module_file_manager" args: ["--bolt-port", *bolt_port] - <<: *template_cluster + <<: *in_memory_cluster - name: "Module File Manager multi database" binary: "tests/e2e/module_file_manager/memgraph__e2e__module_file_manager" args: ["--bolt-port", *bolt_port, "--multi-db", "true"] - <<: *template_cluster + <<: *in_memory_cluster + + - name: "Module File Manager on disk" + binary: "tests/e2e/module_file_manager/memgraph__e2e__module_file_manager" + args: ["--bolt-port", *bolt_port] + <<: *disk_cluster diff --git a/tests/e2e/python_query_modules_reloading/workloads.yaml b/tests/e2e/python_query_modules_reloading/workloads.yaml index 56cd65ff9..50fc35b96 100644 --- a/tests/e2e/python_query_modules_reloading/workloads.yaml +++ b/tests/e2e/python_query_modules_reloading/workloads.yaml @@ -1,27 +1,35 @@ -test_reload_query_module: &test_reload_query_module +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + - "--also-log-to-stderr" + +test_reload_query_module_in_memory_cluster: &test_reload_query_module_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr"] + args: *args log_file: "py-query-modules-reloading-e2e.log" setup_queries: [] - validation_queries: [] -disk_test_reload_query_module: &disk_test_reload_query_module + validation_querie: [] + +disk_test_reload_query_module_disk_cluster: &disk_test_reload_query_module_disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr"] + args: *args log_file: "py-query-modules-reloading-e2e.log" - setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] - workloads: - - name: "test-reload-query-module" # should be the same as the python file + - name: "test-reload-query-module" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/python_query_modules_reloading/procedures/" args: ["python_query_modules_reloading/test_reload_query_module.py"] - <<: *test_reload_query_module - - name: "test-reload-query-module on disk" # should be the same as the python file + <<: *test_reload_query_module_in_memory_cluster + + - name: "test-reload-query-module on disk" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/python_query_modules_reloading/procedures/" args: ["python_query_modules_reloading/test_reload_query_module.py"] - <<: *disk_test_reload_query_module + <<: *disk_test_reload_query_module_disk_cluster diff --git a/tests/e2e/streams/workloads.yaml b/tests/e2e/streams/workloads.yaml index 25a04c114..195fe834c 100644 --- a/tests/e2e/streams/workloads.yaml +++ b/tests/e2e/streams/workloads.yaml @@ -1,24 +1,62 @@ -template_cluster: &template_cluster +stream_args: &stream_args + - "--bolt-port" + - "7687" + - "--log-level=DEBUG" + - "--kafka-bootstrap-servers=localhost:9092" + - "--query-execution-timeout-sec=0" + - "--pulsar-service-url=pulsar://127.0.0.1:6650" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=DEBUG", "--kafka-bootstrap-servers=localhost:9092", "--query-execution-timeout-sec=0", "--pulsar-service-url=pulsar://127.0.0.1:6650"] + args: *stream_args log_file: "streams-e2e.log" setup_queries: [] validation_queries: [] +disk_cluster: &disk_cluster + cluster: + main: + args: *stream_args + log_file: "streams-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + validation_queries: [] + + workloads: - name: "Kafka streams start, stop and show" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/streams/transformations/" args: ["streams/kafka_streams_tests.py"] - <<: *template_cluster + <<: *in_memory_cluster + - name: "Streams with users" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/streams/transformations/" args: ["streams/streams_owner_tests.py"] - <<: *template_cluster + <<: *in_memory_cluster + - name: "Pulsar streams start, stop and show" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/streams/transformations/" args: ["streams/pulsar_streams_tests.py"] - <<: *template_cluster + <<: *in_memory_cluster + + - name: "Kafka streams start, stop and show for on-disk storage" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/streams/transformations/" + args: ["streams/kafka_streams_tests.py"] + <<: *disk_cluster + + - name: "Streams with users for on-disk storage" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/streams/transformations/" + args: ["streams/streams_owner_tests.py"] + <<: *disk_cluster + + - name: "Pulsar streams start, stop and show for on-disk storage" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/streams/transformations/" + args: ["streams/pulsar_streams_tests.py"] + <<: *disk_cluster diff --git a/tests/e2e/temporal_types/workloads.yaml b/tests/e2e/temporal_types/workloads.yaml index f4bccefae..9aeec5d4b 100644 --- a/tests/e2e/temporal_types/workloads.yaml +++ b/tests/e2e/temporal_types/workloads.yaml @@ -1,17 +1,24 @@ bolt_port: &bolt_port "7687" +args: &args + - "--bolt_port" + - *bolt_port + - "--log-level=TRACE" + template_cluster: &template_cluster cluster: main: - args: ["--bolt_port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "temporal-types-e2e.log" setup_queries: [] validation_queries: [] + disk_template_cluster: &disk_template_cluster cluster: main: - args: ["--bolt_port", *bolt_port, "--log-level=TRACE"] + args: *args log_file: "temporal-types-e2e.log" - setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] @@ -20,6 +27,7 @@ workloads: binary: "tests/e2e/temporal_types/memgraph__e2e__temporal_roundtrip" args: ["--bolt_port", *bolt_port] <<: *template_cluster + - name: "Temporal on disk" binary: "tests/e2e/temporal_types/memgraph__e2e__temporal_roundtrip" args: ["--bolt_port", *bolt_port] diff --git a/tests/e2e/transaction_queue/workloads.yaml b/tests/e2e/transaction_queue/workloads.yaml index b5f15facf..12f58c381 100644 --- a/tests/e2e/transaction_queue/workloads.yaml +++ b/tests/e2e/transaction_queue/workloads.yaml @@ -1,14 +1,34 @@ -test_transaction_queue: &test_transaction_queue +args: &args + - "--bolt-port" + - "7687" + - "--log-level=TRACE" + +test_transaction_queue_in_memory_cluster: &test_transaction_queue_in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE", "--also-log-to-stderr"] + args: *args log_file: "transaction_queue.log" setup_queries: [] validation_queries: [] +test_transaction_queue_disk_cluster: &test_transaction_queue_disk_cluster + cluster: + main: + args: *args + log_file: "transaction_queue.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" + validation_queries: [] + workloads: - - name: "test-transaction-queue" # should be the same as the python file + - name: "test-transaction-queue" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/transaction_queue/procedures/" args: ["transaction_queue/test_transaction_queue.py"] - <<: *test_transaction_queue + <<: *test_transaction_queue_in_memory_cluster + + - name: "test-transaction-queue for on-disk" + binary: "tests/e2e/pytest_runner.sh" + proc: "tests/e2e/transaction_queue/procedures/" + args: ["transaction_queue/test_transaction_queue.py"] + <<: *test_transaction_queue_disk_cluster diff --git a/tests/e2e/triggers/workloads.yaml b/tests/e2e/triggers/workloads.yaml index efd658f2f..0aa138b25 100644 --- a/tests/e2e/triggers/workloads.yaml +++ b/tests/e2e/triggers/workloads.yaml @@ -1,80 +1,106 @@ bolt_port: &bolt_port "7687" -template_cluster: &template_cluster +args_properties_false: &args_properties_false + - "--bolt-port" + - *bolt_port + - "--log-level=TRACE" + - "--storage-properties-on-edges=False" + +args_properties_true: &args_properties_true + - "--bolt-port" + - *bolt_port + - "--log-level=TRACE" + - "--storage-properties-on-edges=True" + +storage_properties_edges_true_in_memory_cluster: &storage_properties_edges_true_in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--storage-properties-on-edges=True"] + args: *args_properties_true log_file: "triggers-e2e.log" setup_queries: [] validation_queries: [] -storage_properties_edges_false: &storage_properties_edges_false + +storage_properties_edges_false_in_memory_cluster: &storage_properties_edges_false_in_memory_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--also-log-to-stderr", "--storage-properties-on-edges=False"] + args: *args_properties_false log_file: "triggers-e2e.log" setup_queries: [] validation_queries: [] -disk_template_cluster: &disk_template_cluster + +storage_properties_edges_true_disk_cluster: &storage_properties_edges_true_disk_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--storage-properties-on-edges=True"] - log_file: "triggers-e2e-disk.log" - setup_queries: ["storage mode on_disk_transactional"] + args: *args_properties_true + log_file: "triggers-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] -disk_storage_properties_edges_false: &disk_storage_properties_edges_false + +storage_properties_edges_false_disk_cluster: &storage_properties_edges_false_disk_cluster cluster: main: - args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--also-log-to-stderr", "--storage-properties-on-edges=False"] - log_file: "triggers-e2e-disk.log" - setup_queries: ["storage mode on_disk_transactional"] + args: *args_properties_false + log_file: "triggers-e2e.log" + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] + workloads: - name: "ON CREATE Triggers" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_create" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *template_cluster + <<: *storage_properties_edges_true_in_memory_cluster + - name: "ON UPDATE Triggers" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_update" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *template_cluster + <<: *storage_properties_edges_true_in_memory_cluster + - name: "ON DELETE Triggers Storage Properties On Edges True" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_delete" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *template_cluster + <<: *storage_properties_edges_true_in_memory_cluster + - name: "Triggers privilege check" binary: "tests/e2e/triggers/memgraph__e2e__triggers__privileges" args: ["--bolt-port", *bolt_port] - <<: *template_cluster - - name: "ON DELETE Triggers Storage Properties On Edges False" # should be the same as the python file + <<: *storage_properties_edges_true_in_memory_cluster + + - name: "ON DELETE Triggers Storage Properties On Edges False" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/triggers/procedures/" args: ["triggers/triggers_properties_false.py"] - <<: *storage_properties_edges_false + <<: *storage_properties_edges_false_in_memory_cluster - - name: "ON CREATE Triggers on disk" + - name: "ON CREATE Triggers for disk storage" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_create" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *disk_template_cluster - - name: "ON UPDATE Triggers on disk" + <<: *storage_properties_edges_true_disk_cluster + + - name: "ON UPDATE Triggers for disk storage" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_update" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *disk_template_cluster - - name: "ON DELETE Triggers Storage Properties On Edges True On Disk" + <<: *storage_properties_edges_true_disk_cluster + + - name: "ON DELETE Triggers Storage Properties On Edges True for disk storage" binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_delete" args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" - <<: *disk_template_cluster - - name: "Triggers privilege check on disk" + <<: *storage_properties_edges_true_disk_cluster + + - name: "Triggers privilege check for disk storage" binary: "tests/e2e/triggers/memgraph__e2e__triggers__privileges" args: ["--bolt-port", *bolt_port] - <<: *disk_template_cluster - - name: "ON DELETE Triggers Storage Properties On Edges False On Disk" # should be the same as the python file + <<: *storage_properties_edges_true_disk_cluster + + - name: "ON DELETE Triggers Storage Properties On Edges False for disk storage" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/triggers/procedures/" args: ["triggers/triggers_properties_false.py"] - <<: *disk_storage_properties_edges_false + <<: *storage_properties_edges_false_disk_cluster diff --git a/tests/e2e/write_procedures/workloads.yaml b/tests/e2e/write_procedures/workloads.yaml index be03d0cab..b0c411b8b 100644 --- a/tests/e2e/write_procedures/workloads.yaml +++ b/tests/e2e/write_procedures/workloads.yaml @@ -1,16 +1,24 @@ -template_cluster: &template_cluster +bolt_port: &bolt_port "7687" +args: &args + - "--bolt_port" + - *bolt_port + - "--log-level=TRACE" + +in_memory_cluster: &in_memory_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "write-procedures-e2e.log" setup_queries: [] validation_queries: [] -disk_template_cluster: &disk_template_cluster + +disk_cluster: &disk_cluster cluster: main: - args: ["--bolt-port", "7687", "--log-level=TRACE"] + args: *args log_file: "write-procedures-e2e.log" - setup_queries: ["STORAGE MODE ON_DISK_TRANSACTIONAL"] + setup_queries: + - "STORAGE MODE ON_DISK_TRANSACTIONAL" validation_queries: [] workloads: @@ -18,19 +26,22 @@ workloads: binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/write_procedures/procedures/" args: ["write_procedures/simple_write.py"] - <<: *template_cluster + <<: *in_memory_cluster + - name: "Graph projection procedures" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/write_procedures/procedures/" args: ["write_procedures/read_subgraph.py"] - <<: *template_cluster + <<: *in_memory_cluster + - name: "Write procedures simple on disk" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/write_procedures/procedures/" args: ["write_procedures/simple_write.py"] - <<: *disk_template_cluster + <<: *disk_cluster + - name: "Graph projection procedures on disk" binary: "tests/e2e/pytest_runner.sh" proc: "tests/e2e/write_procedures/procedures/" args: ["write_procedures/read_subgraph.py"] - <<: *disk_template_cluster + <<: *disk_cluster diff --git a/tests/mgbench/workloads/high_write_set_property.py b/tests/mgbench/workloads/high_write_set_property.py new file mode 100644 index 000000000..098aa01ea --- /dev/null +++ b/tests/mgbench/workloads/high_write_set_property.py @@ -0,0 +1,72 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +from workloads.base import Workload + + +class HighWriteSetProperty(Workload): + NAME = "high_write_set_property" + CARDINALITY = 100000 + + def indexes_generator(self): + return [ + ("CREATE INDEX ON :Node;", {}), + ("CREATE INDEX ON :Node(prop1);", {}), + ("CREATE INDEX ON :Node(prop2);", {}), + ("CREATE INDEX ON :Node(prop3);", {}), + ("CREATE INDEX ON :Node(prop4);", {}), + ("CREATE INDEX ON :Node(prop5);", {}), + ("CREATE INDEX ON :Node(prop6);", {}), + ("CREATE INDEX ON :Node(prop7);", {}), + ("CREATE INDEX ON :Node(prop8);", {}), + ("CREATE INDEX ON :Node(prop9);", {}), + ("CREATE INDEX ON :Node(prop10);", {}), + ("CREATE INDEX ON :Node(prop11);", {}), + ("CREATE INDEX ON :Node(prop12);", {}), + ("CREATE INDEX ON :Node(prop13);", {}), + ("CREATE INDEX ON :Node(prop14);", {}), + ("CREATE INDEX ON :Node(prop15);", {}), + ("CREATE INDEX ON :Node(prop16);", {}), + ("CREATE INDEX ON :Node(prop17);", {}), + ("CREATE INDEX ON :Node(prop18);", {}), + ("CREATE INDEX ON :Node(prop19);", {}), + ("CREATE INDEX ON :Node(prop20);", {}), + ] + + def dataset_generator(self): + queries = [] + for i in range(0, HighWriteSetProperty.CARDINALITY): + queries.append( + ( + """CREATE +(:Node {prop1: $id, prop2: $id, prop3: $id, +prop4: $id, prop5: $id, prop6: $id, prop7: $id, prop8: $id, +prop9: $id, prop10: $id, prop11: $id, prop12: $id, prop13: $id, +prop14: $id, prop15: $id, prop16: $id, prop17: $id, prop18: $id, +prop19: $id, prop20: $id}); +""", + {"id": i}, + ) + ) + + return queries + + def benchmark__test__write_20_attributes_set_property(self): + return ( + "MATCH (n:Node) SET n.prop1 = n.prop2, n.prop2 = n.prop3, n.prop3 = n.prop4, n.prop4 = n.prop5, n.prop5 = n.prop6, n.prop6 = n.prop7, n.prop7 = n.prop8, n.prop8 = n.prop9, n.prop9 = n.prop10, n.prop10 = n.prop11, n.prop11 = n.prop12, n.prop12 = n.prop13, n.prop13 = n.prop14, n.prop14 = n.prop15, n.prop15 = n.prop16, n.prop16 = n.prop17, n.prop17 = n.prop18, n.prop18 = n.prop19, n.prop19 = n.prop20, n.prop20 = n.prop1;", + {}, + ) + + def benchmark__test__write_20_attributes_set_properties(self): + return ( + "MATCH (n:Node) SET n += {prop1: n.prop2, prop2: n.prop3, prop3: n.prop4, prop4: n.prop5, prop5: n.prop6, prop6: n.prop7, prop7: n.prop8, prop8: n.prop9, prop9: n.prop10, prop10: n.prop11, prop11: n.prop12, prop12: n.prop13, prop13: n.prop14, prop14: n.prop15, prop15: n.prop16, prop16: n.prop17, prop17: n.prop18, prop18: n.prop19, prop19: n.prop20, prop20: n.prop1};", + {}, + ) diff --git a/tests/mgbench/workloads/supernode.py b/tests/mgbench/workloads/supernode.py new file mode 100644 index 000000000..17f476675 --- /dev/null +++ b/tests/mgbench/workloads/supernode.py @@ -0,0 +1,43 @@ +# Copyright 2023 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +from workloads.base import Workload + + +class Supernode(Workload): + NAME = "supernode" + CARDINALITY = 50000 + + def indexes_generator(self): + return [ + ("CREATE INDEX ON :Supernode;", {}), + ("CREATE INDEX ON :Supernode(id);", {}), + ("CREATE INDEX ON :Node;", {}), + ("CREATE INDEX ON :Node(id);", {}), + ] + + def dataset_generator(self): + queries = [] + queries.append(("CREATE (:Supernode {id: $id});", {"id": 1})) + for i in range(0, Supernode.CARDINALITY): + queries.append(("CREATE (:Node {id: $id});", {"id": i})) + queries.append(("MATCH (s:Supernode), (n:Node) CREATE (s)<-[:EDGE]-(n)", {})) + + return queries + + def benchmark__test__merge_supernode_edges(self): + return ("MATCH (s:Supernode), (n:Node) MERGE (s)<-[:EDGE]-(n);", {}) + + def benchmark__test__merge_supernode_edges_other_way(self): + return ("MATCH (s:Supernode), (n:Node) MERGE (n)-[:EDGE]->(s);", {}) + + def benchmark__test__unwind_supernode_with_writes(self): + return (f"UNWIND range(1, {Supernode.CARDINALITY}) AS x MATCH (s:Supernode) SET s.prop = x", {}) diff --git a/tests/unit/cpp_api.cpp b/tests/unit/cpp_api.cpp index 70e62110e..19852478e 100644 --- a/tests/unit/cpp_api.cpp +++ b/tests/unit/cpp_api.cpp @@ -615,3 +615,77 @@ TYPED_TEST(CppApiTestFixture, TestValuePrint) { std::string date_test = oss_date.str(); ASSERT_EQ("2020-12-12", date_test); } + +TYPED_TEST(CppApiTestFixture, TestValueToString) { + /*graph and node shared by multiple types*/ + mgp_graph raw_graph = this->CreateGraph(memgraph::storage::View::NEW); + auto graph = mgp::Graph(&raw_graph); + + /*null*/ + ASSERT_EQ(mgp::Value().ToString(), ""); + /*bool*/ + ASSERT_EQ(mgp::Value(false).ToString(), "false"); + /*int*/ + const int64_t int1 = 60; + ASSERT_EQ(mgp::Value(int1).ToString(), "60"); + /*double*/ + const double double1 = 2.567891; + ASSERT_EQ(mgp::Value(double1).ToString(), "2.567891"); + /*string*/ + const std::string str = "string"; + ASSERT_EQ(mgp::Value(str).ToString(), "string"); + /*list*/ + mgp::List list; + auto node_list = graph.CreateNode(); + node_list.AddLabel("Label_list"); + list.AppendExtend(mgp::Value("inside")); + list.AppendExtend(mgp::Value("2")); + list.AppendExtend(mgp::Value(node_list)); + ASSERT_EQ(mgp::Value(list).ToString(), "[inside, 2, (id: 0, :Label_list, properties: {})]"); + /*map*/ + mgp::Map map; + auto node_map = graph.CreateNode(); + node_map.AddLabel("Label_map"); + map.Insert("key", mgp::Value(int1)); + map.Insert("node", mgp::Value(node_map)); + ASSERT_EQ(mgp::Value(map).ToString(), "{key: 60, node: (id: 1, :Label_map, properties: {})}"); + /*date*/ + mgp::Date date_1{"2020-12-12"}; + ASSERT_EQ(mgp::Value(date_1).ToString(), "2020-12-12"); + + /*local time*/ + mgp::LocalTime local_time{"09:15:00.360"}; + ASSERT_EQ(mgp::Value(local_time).ToString(), "9:15:0,3600"); + + /*local date time*/ + mgp::LocalDateTime local_date_time{"2021-10-05T14:15:00"}; + ASSERT_EQ(mgp::Value(local_date_time).ToString(), "2021-10-5T14:15:0,00"); + + /*duration*/ + mgp::Duration duration{"P14DT17H2M45S"}; + ASSERT_EQ(mgp::Value(duration).ToString(), "1270965000000ms"); + + /*node and relationship*/ + auto node1 = graph.CreateNode(); + node1.AddLabel("Label1"); + node1.AddLabel("Label2"); + auto node2 = graph.CreateNode(); + node2.SetProperty("key", mgp::Value("node_property")); + node2.SetProperty("key2", mgp::Value("node_property2")); + auto rel = graph.CreateRelationship(node1, node2, "Loves"); + + rel.SetProperty("key", mgp::Value("property")); + ASSERT_EQ(mgp::Value(rel).ToString(), + "(id: 2, :Label1:Label2, properties: {})-[type: Loves, id: 0, properties: {key: property}]->(id: 3, " + "properties: {key: node_property, key2: node_property2})"); + /*path*/ + mgp::Path path = mgp::Path(node1); + path.Expand(rel); + auto node3 = graph.CreateNode(); + auto rel2 = graph.CreateRelationship(node2, node3, "Loves2"); + path.Expand(rel2); + ASSERT_EQ( + mgp::Value(path).ToString(), + "(id: 2, :Label1:Label2, properties: {})-[type: Loves, id: 0, properties: {key: property}]->(id: 3, properties: " + "{key: node_property, key2: node_property2})-[type: Loves2, id: 1, properties: {}]->(id: 4, properties: {})"); +} diff --git a/tests/unit/storage_v2_indices.cpp b/tests/unit/storage_v2_indices.cpp index 20037a4d9..f63ac5b2c 100644 --- a/tests/unit/storage_v2_indices.cpp +++ b/tests/unit/storage_v2_indices.cpp @@ -15,6 +15,8 @@ #include #include "disk_test_utils.hpp" +#include "storage/v2/disk/label_index.hpp" +#include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/disk/storage.hpp" #include "storage/v2/inmemory/storage.hpp" #include "storage/v2/property_value.hpp"