Merge branch 'project-pineapples' into add-gnuplot-example

[🍍 < MG] Fix mgbench access control benchmark

src/coordinator/coordinator.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -9,206 +9,21 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-#include "coordinator/coordinator.hpp"
-#include "query/v2/requests.hpp"
-#include "storage/v3/value_conversions.hpp"
+#include <coordinator/coordinator.hpp>
 
 namespace memgraph::coordinator {
 
-// 1. try to begin any suggested splits
-// 2. mark all initialized RSMs as INITIALIZED in the ShardMap
-// 3. assign any valid underreplicated shards to the Heartbeat sender
-// 4. send any split requests that the Heartbeat sender should be applying
 CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
   spdlog::info("Coordinator handling HeartbeatRequest");
 
-  HeartbeatResponse ret{};
+  // add this storage engine to any under-replicated shards that it is not already a part of
 
-  bool initiated_split = false;
+  auto initializing_rsms_for_shard_manager =
+      shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
 
-  // 1. try to begin any suggested splits
-  for (const auto &suggested_split_info : heartbeat_request.suggested_splits) {
-    const LabelId label_id = suggested_split_info.label_id;
-    auto &label_space = shard_map_.label_spaces.at(label_id);
-
-    const PrimaryKey splitting_shard_low_key =
-        storage::conversions::ConvertPropertyVector(suggested_split_info.splitting_shard_low_key);
-    const PrimaryKey split_key = storage::conversions::ConvertPropertyVector(suggested_split_info.split_key);
-
-    auto &shard = label_space.shards.at(splitting_shard_low_key);
-    const ShardId splitting_shard_id = std::make_pair(label_id, split_key);
-
-    if (shard.pending_split.has_value() || shard.version != suggested_split_info.shard_version) {
-      spdlog::info("skipping split, already splitting: {}, shard.version: {}, suggested shard_version: {}",
-                   shard.pending_split.has_value(), shard.version, suggested_split_info.shard_version);
-      continue;
-    }
-
-    // begin the split process for this shard
-    spdlog::info("Coordinator beginning new split process after receiving a pending split");
-
-    // bump current shard version and store pending split info
-    shard.version = shard_map_.GetHlc();
-    shard.pending_split = suggested_split_info;
-    splitting_shards_.insert(splitting_shard_id);
-    initiated_split = true;
-
-    // copy this shard and store it in the ShardMap
-    ShardMetadata duplicated_shard{shard};
-    std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
-    for (auto &peer_metadata : duplicated_shard.peers) {
-      peer_metadata.status = Status::PENDING_SPLIT;
-      peer_metadata.split_from = peer_metadata.address.unique_id;
-
-      const auto new_uuid = shard_map_.NewShardUuid();
-
-      spdlog::info("Coordinator allocating new rsm uuid: {}", new_uuid);
-
-      // store new uuid for the right side of each shard
-      rsm_split_from_.insert({new_uuid, splitting_shard_id});
-      split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
-
-      peer_metadata.address.unique_id = new_uuid;
-    }
-
-    // TODO(tyler) fix this good assertion
-    // if (high_key) {
-    //   MG_ASSERT(converted_pk < *high_key, "Split point is beyond low key of the next shard");
-    // }
-    label_space.shards.insert({split_key, duplicated_shard});
-
-    // we will send the heartbeater its split request in stage 4 below
-  }
-
-  // 2. mark all initialized RSMs as INITIALIZED in the ShardMap
-  for (const auto &[initialized_rsm, shard_id] : heartbeat_request.initialized_rsms) {
-    spdlog::info("looking at rsm {}", initialized_rsm);
-    auto [label_id, low_key] = shard_id;
-    auto &label_space = shard_map_.label_spaces.at(label_id);
-    auto &shard = label_space.shards.at(low_key);
-
-    // if even a single shard has been initialized after a split, its raft log has
-    // reached consensus and we can allow more splits again and remove the pending
-    // split state.
-    if (rsm_split_from_.contains(initialized_rsm)) {
-      auto split_shard_id = rsm_split_from_.at(initialized_rsm);
-      splitting_shards_.erase(split_shard_id);
-      rsm_split_from_.erase(initialized_rsm);
-      MG_ASSERT(false,
-                "TODO(tyler) remove this. but until it works once, this is removing rsm from split from before "
-                "actually used");
-    }
-
-    size_t initialized_count = 0;
-    for (auto &peer : shard.peers) {
-      if (peer.address.unique_id == initialized_rsm) {
-        spdlog::info("Coordinator marking rsm {} as initialized", initialized_rsm);
-        peer.status = Status::CONSENSUS_PARTICIPANT;
-      }
-
-      if (peer.status == Status::CONSENSUS_PARTICIPANT) {
-        initialized_count++;
-      }
-    }
-
-    if (initialized_count >= label_space.replication_factor) {
-      spdlog::info("clearing underreplicated shard with {} initialized peers", initialized_count);
-      underreplicated_shards_.erase(shard_id);
-    }
-  }
-
-  // 3. assign any valid underreplicated shards to the Heartbeat sender
-  for (auto &underreplicated_shard_id : underreplicated_shards_) {
-    auto [label_id, low_key] = underreplicated_shard_id;
-    auto &label_space = shard_map_.label_spaces.at(label_id);
-    auto &shard = label_space.shards.at(low_key);
-
-    // make sure we're not already a member
-    for (const auto &peer : shard.peers) {
-      if (peer.address.last_known_ip == heartbeat_request.from_storage_manager.last_known_ip &&
-          peer.address.last_known_port == heartbeat_request.from_storage_manager.last_known_port) {
-        // we are already a member of this shard. continue to next underreplicated shard to consider
-        continue;
-      }
-    }
-
-    // we are not already a member, so we can be assigned to this shard
-    Address address = heartbeat_request.from_storage_manager;
-    address.unique_id = shard_map_.NewShardUuid();
-
-    std::optional<PrimaryKey> high_key;
-    auto next = std::next(label_space.shards.find(low_key));
-    if (next != label_space.shards.end()) {
-      high_key = next->first;
-    }
-
-    spdlog::info("assigning shard manager to shard");
-
-    ret.shards_to_initialize.push_back(ShardToInitialize{
-        .new_shard_version = shard.version,
-        .uuid = address.unique_id,
-        .label_id = label_id,
-        .min_key = low_key,
-        .max_key = high_key,
-        .schema = shard_map_.schemas[label_id],
-        .config = Config{.split =
-                             Config::Split{
-                                 .max_shard_vertex_size = label_space.split_threshold,
-                             }},
-        .id_to_names = shard_map_.IdToNames(),
-    });
-
-    PeerMetadata peer_metadata = {
-        .address = address,
-        .status = Status::INITIALIZING,
-    };
-
-    shard.peers.emplace_back(peer_metadata);
-  }
-
-  // 4. send any split requests that the Heartbeat sender should be applying
-  spdlog::info("0");
-  for (const auto &[label_id, low_key] : splitting_shards_) {
-    // see if this machine is a PENDING_SPLIT peer for any of the splitting shards
-    const auto &label_space = shard_map_.label_spaces.at(label_id);
-    const auto &shard = label_space.shards.at(low_key);
-
-    spdlog::info("1");
-    for (const auto &peer : shard.peers) {
-      spdlog::info("2");
-      if (peer.status != Status::PENDING_SPLIT ||
-          peer.address.last_known_ip != heartbeat_request.from_storage_manager.last_known_ip ||
-          peer.address.last_known_port != heartbeat_request.from_storage_manager.last_known_port) {
-        spdlog::info("not splitting peer: status is PENDING_SPLIT: {}, peer address: {} storage manager address: {}",
-                     peer.status == Status::PENDING_SPLIT, peer.address, heartbeat_request.from_storage_manager);
-        // not splitting or not us
-        continue;
-      }
-
-      spdlog::info("Coordinator expecting heartbeating peer to split, so it will reply with a ShardToSplit message");
-
-      std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping{};
-
-      // need to iterate over all peers again to build the full uuid_mapping to
-      // send to this machine
-      for (const auto &peer_metadata2 : shard.peers) {
-        uuid_mapping.emplace(peer_metadata2.split_from, peer_metadata2.address.unique_id);
-      }
-
-      ret.shards_to_split.push_back(ShardToSplit{
-          .split_key = low_key,
-          // .old_shard_version = shard.previous_version,
-          .new_shard_version = shard.version,
-          .uuid_mapping = uuid_mapping,
-      });
-    }
-  }
-
-  if (initiated_split) {
-    MG_ASSERT(!ret.shards_to_split.empty(), "did not send a split request to the shard that suggested a split");
-  }
-
-  return ret;
+  return HeartbeatResponse{
+      .shards_to_initialize = initializing_rsms_for_shard_manager,
+  };
 }
 
 CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
@@ -222,7 +37,6 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
       .logical_id = ++highest_allocated_timestamp_,
       // TODO(tyler) probably pass some more context to the Coordinator here
       // so that we can use our wall clock and enforce monotonicity.
-      // Check it to ensure it's +/- 1 day of the coordinator's io_.Now()
       // .coordinator_wall_clock = io_.Now(),
   };
 
@@ -247,6 +61,22 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
   return res;
 }
 
+/// This splits the shard immediately beneath the provided
+/// split key, keeping the assigned peers identical for now,
+/// but letting them be gradually migrated over time.
+CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
+  SplitShardResponse res{};
+
+  if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
+    res.success = false;
+  } else {
+    res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
+                                        split_shard_request.split_key);
+  }
+
+  return res;
+}
+
 /// This adds the provided storage engine to the standby storage engine pool,
 /// which can be used to rebalance storage over time.
 CoordinatorWriteResponses Coordinator::ApplyWrite(
@@ -272,16 +102,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initi
 
   std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
       initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
-      initialize_label_request.split_threshold, initialize_label_request.last_shard_map_version);
+      initialize_label_request.last_shard_map_version);
 
   if (new_label_id) {
     res.new_label_id = new_label_id.value();
     res.fresher_shard_map = std::nullopt;
     res.success = true;
-
-    auto min_key = SchemaToMinKey(initialize_label_request.schema);
-    auto shard_id = std::make_pair(new_label_id.value(), min_key);
-    underreplicated_shards_.insert(shard_id);
   } else {
     res.fresher_shard_map = shard_map_;
     res.success = false;

src/coordinator/coordinator.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,14 +20,13 @@
 
 #include <boost/uuid/uuid.hpp>
 
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "coordinator/shard_map.hpp"
-#include "io/simulator/simulator.hpp"
-#include "io/time.hpp"
-#include "io/transport.hpp"
-#include "query/v2/requests.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/schemas.hpp"
+#include <coordinator/hybrid_logical_clock.hpp>
+#include <coordinator/shard_map.hpp>
+#include <io/simulator/simulator.hpp>
+#include <io/time.hpp>
+#include <io/transport.hpp>
+#include <storage/v3/id_types.hpp>
+#include <storage/v3/schemas.hpp>
 
 namespace memgraph::coordinator {
 
@@ -38,8 +37,6 @@ using memgraph::storage::v3::SchemaProperty;
 using SimT = memgraph::io::simulator::SimulatorTransport;
 using PrimaryKey = std::vector<PropertyValue>;
 
-using ShardId = std::pair<LabelId, PrimaryKey>;
-
 struct HlcRequest {
   Hlc last_shard_map_version;
 };
@@ -115,7 +112,6 @@ struct InitializeLabelRequest {
   std::string label_name;
   std::vector<SchemaProperty> schema;
   size_t replication_factor;
-  uint64_t split_threshold;
   Hlc last_shard_map_version;
 };
 
@@ -125,29 +121,28 @@ struct InitializeLabelResponse {
   std::optional<ShardMap> fresher_shard_map;
 };
 
+struct HeartbeatRequest {
+  Address from_storage_manager;
+  std::set<boost::uuids::uuid> initialized_rsms;
+};
+
+struct HeartbeatResponse {
+  std::vector<ShardToInitialize> shards_to_initialize;
+};
+
 using CoordinatorWriteRequests =
-    std::variant<HlcRequest, AllocateEdgeIdBatchRequest, RegisterStorageEngineRequest, DeregisterStorageEngineRequest,
-                 InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
-using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, RegisterStorageEngineResponse,
-                                               DeregisterStorageEngineResponse, InitializeLabelResponse,
-                                               AllocatePropertyIdsResponse, HeartbeatResponse>;
+    std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
+                 DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
+using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
+                                               RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
+                                               InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
 
 using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
 using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
 
 class Coordinator {
  public:
-  explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {
-    // Populate underreplicated_shards_
-    for (const auto &[label_id, label_space] : shard_map_.label_spaces) {
-      for (const auto &[low_key, shard] : label_space.shards) {
-        if (shard.peers.size() < label_space.replication_factor) {
-          ShardId shard_id = std::make_pair(label_id, low_key);
-          underreplicated_shards_.insert(shard_id);
-        }
-      }
-    }
-  }
+  explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {}
 
   // NOLINTNEXTLINE(readability-convert-member-functions-to-static
   CoordinatorReadResponses Read(CoordinatorReadRequests requests) {
@@ -165,11 +160,6 @@ class Coordinator {
   ShardMap shard_map_;
   uint64_t highest_allocated_timestamp_{0};
 
-  std::set<ShardId> underreplicated_shards_;
-  std::map<boost::uuids::uuid, ShardId> rsm_split_from_;
-  std::set<ShardId> splitting_shards_;
-  std::map<Address, std::set<ShardId>> assigned_shards_;
-
   /// Query engines need to periodically request batches of unique edge IDs.
   uint64_t highest_allocated_edge_id_{0};
 
@@ -185,6 +175,11 @@ class Coordinator {
 
   CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
 
+  /// This splits the shard immediately beneath the provided
+  /// split key, keeping the assigned peers identical for now,
+  /// but letting them be gradually migrated over time.
+  CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
+
   /// This adds the provided storage engine to the standby storage engine pool,
   /// which can be used to rebalance storage over time.
   static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);

src/coordinator/hybrid_logical_clock.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,19 +23,17 @@ namespace memgraph::coordinator {
 using Time = memgraph::io::Time;
 
 /// Hybrid-logical clock
-struct Hlc final {
-  uint64_t logical_id{0};
+struct Hlc {
+  uint64_t logical_id = 0;
   Time coordinator_wall_clock = Time::min();
 
   auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
 
-  bool operator==(const Hlc &other) const noexcept = default;
-  bool operator<(const Hlc &other) const noexcept = default;
-  bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
-  bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
-  bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
-
-  Hlc operator++() { return {.logical_id = ++logical_id, .coordinator_wall_clock = Time::min()}; }
+  bool operator==(const Hlc &other) const = default;
+  bool operator<(const Hlc &other) const = default;
+  bool operator==(const uint64_t other) const { return logical_id == other; }
+  bool operator<(const uint64_t other) const { return logical_id < other; }
+  bool operator>=(const uint64_t other) const { return logical_id >= other; }
 
   friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
     auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);

src/coordinator/shard_map.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -9,21 +9,15 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-#include <boost/uuid/uuid.hpp>
-#include <map>
 #include <optional>
 #include <unordered_map>
 #include <vector>
 
 #include "common/types.hpp"
-#include "coordinator/hybrid_logical_clock.hpp"
 #include "coordinator/shard_map.hpp"
-#include "query/v2/requests.hpp"
 #include "spdlog/spdlog.h"
-#include "storage/v3/config.hpp"
 #include "storage/v3/schemas.hpp"
 #include "storage/v3/temporal.hpp"
-#include "storage/v3/value_conversions.hpp"
 #include "utils/cast.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/string.hpp"
@@ -74,9 +68,6 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
 }
 
 ShardMap ShardMap::Parse(std::istream &input_stream) {
-  const uint64_t default_replication_factor = 1;
-  const uint64_t default_split_threshold = 1'000'000;
-
   ShardMap shard_map;
   const auto read_size = [&input_stream] {
     size_t size{0};
@@ -168,10 +159,33 @@ ShardMap ShardMap::Parse(std::istream &input_stream) {
       schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
     }
     const auto hlc = shard_map.GetHlc();
-    MG_ASSERT(
-        shard_map.InitializeNewLabel(primary_label, schema, default_replication_factor, default_split_threshold, hlc)
-            .has_value(),
-        "Cannot initialize new label: {}", primary_label);
+    MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
+              "Cannot initialize new label: {}", primary_label);
+
+    const auto number_of_split_points = read_size();
+    spdlog::debug("Reading {} split points", number_of_split_points);
+
+    [[maybe_unused]] const auto remainder_from_last_line = read_line();
+    for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
+      const auto line = read_line();
+      spdlog::debug("Read split point '{}'", line);
+      MG_ASSERT(line.front() == '[', "Invalid split file format!");
+      MG_ASSERT(line.back() == ']', "Invalid split file format!");
+      std::string_view line_view{line};
+      line_view.remove_prefix(1);
+      line_view.remove_suffix(1);
+      static constexpr std::string_view kDelimiter{","};
+      auto pk_values_as_text = utils::Split(line_view, kDelimiter);
+      std::vector<PropertyValue> pk;
+      pk.reserve(number_of_primary_properties);
+      MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
+                "Split point contains invalid number of values '{}'", line);
+
+      for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
+        pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
+      }
+      shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
+    }
   }
 
   return shard_map;
@@ -232,7 +246,9 @@ std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
   return id_to_names;
 }
 
-boost::uuids::uuid Uint64ToUuid(uint64_t u) {
+Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
+
+boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
   return boost::uuids::uuid{0,
                             0,
                             0,
@@ -241,37 +257,21 @@ boost::uuids::uuid Uint64ToUuid(uint64_t u) {
                             0,
                             0,
                             0,
-                            static_cast<unsigned char>(u >> 56U),
-                            static_cast<unsigned char>(u >> 48U),
-                            static_cast<unsigned char>(u >> 40U),
-                            static_cast<unsigned char>(u >> 32U),
-                            static_cast<unsigned char>(u >> 24U),
-                            static_cast<unsigned char>(u >> 16U),
-                            static_cast<unsigned char>(u >> 8U),
-                            static_cast<unsigned char>(u)};
+                            static_cast<unsigned char>(shard_id >> 56U),
+                            static_cast<unsigned char>(shard_id >> 48U),
+                            static_cast<unsigned char>(shard_id >> 40U),
+                            static_cast<unsigned char>(shard_id >> 32U),
+                            static_cast<unsigned char>(shard_id >> 24U),
+                            static_cast<unsigned char>(shard_id >> 16U),
+                            static_cast<unsigned char>(shard_id >> 8U),
+                            static_cast<unsigned char>(shard_id)};
 }
 
-boost::uuids::uuid ShardMap::NewShardUuid() {
-  uint64_t shard_id = GetHlc().logical_id;
-  return Uint64ToUuid(shard_id);
-}
-
-Hlc ShardMap::GetHlc() noexcept { return ++shard_map_version; }
-
-HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized,
-                                         std::set<msgs::SuggestedSplitInfo> pending_splits) {
-  HeartbeatResponse ret{};
+std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
+                                                      std::set<boost::uuids::uuid> initialized) {
+  std::vector<ShardToInitialize> ret{};
 
   bool mutated = false;
-  std::map<std::pair<boost::uuids::uuid, Hlc>, msgs::PrimaryKey> mapped_pending_splits;
-  for (const auto &pending_split : pending_splits) {
-    MG_ASSERT(pending_split.shard_to_split_uuid != Uint64ToUuid(0),
-              "a shard split for an impossible UUID is being requested");
-    spdlog::info("Coordinator adding pending split for shard version: {}, uuid: {} to attempt to initiate",
-                 pending_split.shard_version, pending_split.shard_to_split_uuid);
-    mapped_pending_splits.insert(
-        {{pending_split.shard_to_split_uuid, pending_split.shard_version}, pending_split.split_key});
-  }
 
   for (auto &[label_id, label_space] : label_spaces) {
     for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
@@ -281,159 +281,63 @@ HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost
         high_key = next_it->first;
       }
       // TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
-      bool shard_assigned_to_machine = false;
+      bool machine_contains_shard = false;
 
-      for (auto &peer_metadata : shard.peers) {
-        const bool same_machine = peer_metadata.address.last_known_ip == storage_manager.last_known_ip &&
-                                  peer_metadata.address.last_known_port == storage_manager.last_known_port;
-        spdlog::info("Coordinator split debug - shard version: {}", shard.version);
-        if (initialized.contains(peer_metadata.address.unique_id)) {
-          spdlog::info("1");
-          shard_assigned_to_machine = true;
-
-          if (!same_machine) {
-            // set the last known ip and port to the storage manager that has heartbeated it just now
+      for (auto &aas : shard.peers) {
+        if (initialized.contains(aas.address.unique_id)) {
+          machine_contains_shard = true;
+          if (aas.status != Status::CONSENSUS_PARTICIPANT) {
             mutated = true;
-            peer_metadata.address.last_known_ip = storage_manager.last_known_ip;
-            peer_metadata.address.last_known_port = storage_manager.last_known_port;
+            spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
+            aas.status = Status::CONSENSUS_PARTICIPANT;
           }
-
-          if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
-            mutated = true;
-            spdlog::info("marking shard as full consensus participant: {}", peer_metadata.address.unique_id);
-            peer_metadata.status = Status::CONSENSUS_PARTICIPANT;
-          }
-        } else if (same_machine && peer_metadata.status == Status::INITIALIZING) {
-          spdlog::info("2");
-          // we are expecting this shard to be initialized (rather than split) on this machine,
-          // so send it an initialization request
-
-          shard_assigned_to_machine = true;
-          spdlog::info("reminding shard manager that they should begin participating in shard");
-
-          ret.shards_to_initialize.push_back(ShardToInitialize{
-              .new_shard_version = shard.version,
-              .uuid = peer_metadata.address.unique_id,
-              .label_id = label_id,
-              .min_key = low_key,
-              .max_key = high_key,
-              .schema = schemas[label_id],
-              .config = Config{.split =
-                                   Config::Split{
-                                       .max_shard_vertex_size = label_space.split_threshold,
-                                   }},
-              .id_to_names = IdToNames(),
-          });
-        } else if (same_machine && peer_metadata.status == Status::PENDING_SPLIT) {
-          // we are expecting this shard to be split, so send it a split request
-          spdlog::info(
-              "Coordinator expecting heartbeating peer to split, so it will reply with a ShardToSplit message");
-
-          std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping{};
-
-          // need to iterate over all peers again to build the full uuid_mapping to
-          // send to this machine
-          for (const auto &peer_metadata2 : shard.peers) {
-            uuid_mapping.emplace(peer_metadata2.split_from, peer_metadata2.address.unique_id);
-          }
-
-          MG_ASSERT(false, "adding a shard to split :)");
-          ret.shards_to_split.push_back(ShardToSplit{
-              .split_key = low_key,
-              // .old_shard_version = shard.previous_version,
-              .new_shard_version = shard.version,
-              .uuid_mapping = uuid_mapping,
-          });
         } else {
-          spdlog::info("5");
-          MG_ASSERT(
-              !same_machine,
-              "failed to properly handle a new Status type in the heartbeat management and shard assignment code");
-        }
+          const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
+                                    aas.address.last_known_port == storage_manager.last_known_port;
+          if (same_machine) {
+            machine_contains_shard = true;
+            spdlog::info("reminding shard manager that they should begin participating in shard");
 
-        if (const auto pending_split = mapped_pending_splits.find({peer_metadata.address.unique_id, shard.version});
-            pending_split != mapped_pending_splits.end()) {
-          // Now we handle shard split
-          if (shard.pending_split) {
-            spdlog::info("Coordinator received split request while split is happening!");
-            continue;
+            ret.push_back(ShardToInitialize{
+                .uuid = aas.address.unique_id,
+                .label_id = label_id,
+                .min_key = low_key,
+                .max_key = high_key,
+                .schema = schemas[label_id],
+                .config = Config{},
+                .id_to_names = IdToNames(),
+            });
           }
-          MG_ASSERT(false, "beginning split from coordinator perspective :)");
-          spdlog::info("Coordinator beginning new split process after receiving a pending split");
-          shard.pending_split = msgs::SuggestedSplitInfo{.shard_to_split_uuid = pending_split->first.first,
-                                                         .split_key = pending_split->second,
-                                                         .shard_version = pending_split->first.second};
-
-          shard.version = GetHlc();
-          std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
-          ShardMetadata duplicated_shard{shard};
-          for (auto &peer_metadata3 : duplicated_shard.peers) {
-            peer_metadata3.status = Status::PENDING_SPLIT;
-            peer_metadata3.split_from = peer_metadata3.address.unique_id;
-
-            const auto new_uuid = NewShardUuid();
-
-            // store new uuid for the right side of each shard
-            split_mapping.emplace(peer_metadata3.address.unique_id, new_uuid);
-
-            peer_metadata3.address.unique_id = new_uuid;
-          }
-          const auto converted_pk = storage::conversions::ConvertPropertyVector(pending_split->second);
-          if (high_key) {
-            MG_ASSERT(converted_pk < *high_key, "Split point is beyond low key of the next shard");
-          }
-          label_space.shards.insert({converted_pk, duplicated_shard});
-        } else {
-          spdlog::info("Coordinator not attempting to split this shard: there is no pending split for {}:{}",
-                       peer_metadata.address.unique_id, shard.version);
         }
       }
 
-      if (shard.pending_split.has_value()) {
-        // if the split shard has any peers that are CONSENSUS_PARTICIPANT, we can clear our
-        // pending split because even if a single peer has split, it means that the split has
-        // reached a majority of the replicas.
-        bool split_applied_anywhere = false;
-        const auto split_key = storage::conversions::ConvertPropertyVector(shard.pending_split->split_key);
+      if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
+        // increment version for each new uuid for deterministic creation
+        IncrementShardMapVersion();
 
-        for (const auto &split_peer : label_space.shards.at(split_key).peers) {
-          split_applied_anywhere |= split_peer.status == Status::CONSENSUS_PARTICIPANT;
-        }
-
-        if (split_applied_anywhere) {
-          spdlog::info("now that the shard in label space {} is done splitting, we can clear its pending_split data",
-                       label_id);
-          shard.pending_split.reset();
-        }
-      }
-
-      if (!shard_assigned_to_machine && shard.peers.size() < label_space.replication_factor) {
         Address address = storage_manager;
 
-        address.unique_id = NewShardUuid();
+        // TODO(tyler) use deterministic UUID so that coordinators don't diverge here
+        address.unique_id = NewShardUuid(shard_map_version.logical_id);
 
         spdlog::info("assigning shard manager to shard");
 
-        ret.shards_to_initialize.push_back(ShardToInitialize{
-            .new_shard_version = shard.version,
+        ret.push_back(ShardToInitialize{
             .uuid = address.unique_id,
             .label_id = label_id,
             .min_key = low_key,
             .max_key = high_key,
             .schema = schemas[label_id],
-            .config = Config{.split =
-                                 Config::Split{
-                                     .max_shard_vertex_size = label_space.split_threshold,
-                                 }},
+            .config = Config{},
             .id_to_names = IdToNames(),
         });
 
-        PeerMetadata peer_metadata = {
+        AddressAndStatus aas = {
             .address = address,
             .status = Status::INITIALIZING,
         };
 
-        shard.peers.emplace_back(peer_metadata);
+        shard.peers.emplace_back(aas);
       }
     }
   }
@@ -441,15 +345,36 @@ HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost
   if (mutated) {
     IncrementShardMapVersion();
   }
-
   return ret;
 }
 
+bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
+  if (previous_shard_map_version != shard_map_version) {
+    return false;
+  }
+
+  auto &label_space = label_spaces.at(label_id);
+  auto &shards_in_map = label_space.shards;
+
+  MG_ASSERT(!shards_in_map.empty());
+  MG_ASSERT(!shards_in_map.contains(key));
+  MG_ASSERT(label_spaces.contains(label_id));
+
+  // Finding the ShardMetadata that the new PrimaryKey should map to.
+  auto prev = std::prev(shards_in_map.upper_bound(key));
+  ShardMetadata duplicated_shard = prev->second;
+
+  // Apply the split
+  shards_in_map[key] = duplicated_shard;
+
+  IncrementShardMapVersion();
+
+  return true;
+}
+
 std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
-                                                    size_t replication_factor, uint64_t split_threshold,
-                                                    Hlc last_shard_map_version) {
+                                                    size_t replication_factor, Hlc last_shard_map_version) {
   if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
-    MG_ASSERT(false, "failed to InitializeNewLabel");
     return std::nullopt;
   }
 
@@ -468,9 +393,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
       .schema = schema,
       .shards = shards,
       .replication_factor = replication_factor,
-      .split_threshold = split_threshold,
   };
-
   schemas[label_id] = std::move(schema);
 
   label_spaces.emplace(label_id, label_space);
@@ -638,8 +561,8 @@ bool ShardMap::ClusterInitialized() const {
         return false;
       }
 
-      for (const auto &peer_metadata : shard.peers) {
-        if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
+      for (const auto &aas : shard.peers) {
+        if (aas.status != Status::CONSENSUS_PARTICIPANT) {
           spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
           return false;
         }
@@ -650,26 +573,4 @@ bool ShardMap::ClusterInitialized() const {
   return true;
 }
 
-size_t ShardMap::InitializedShards() const {
-  size_t count = 0;
-
-  for (const auto &[label_id, label_space] : label_spaces) {
-    for (const auto &[low_key, shard] : label_space.shards) {
-      if (shard.peers.size() < label_space.replication_factor) {
-        continue;
-      }
-
-      for (const auto &peer_metadata : shard.peers) {
-        if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
-          continue;
-        }
-      }
-
-      count += 1;
-    }
-  }
-
-  return count;
-}
-
 }  // namespace memgraph::coordinator

src/coordinator/shard_map.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,6 @@
 #include <algorithm>
 #include <limits>
 #include <map>
-#include <optional>
 #include <set>
 #include <vector>
 
@@ -24,7 +23,6 @@
 #include "common/types.hpp"
 #include "coordinator/hybrid_logical_clock.hpp"
 #include "io/address.hpp"
-#include "query/v2/requests.hpp"
 #include "storage/v3/config.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/name_id_mapper.hpp"
@@ -49,59 +47,39 @@ using memgraph::storage::v3::SchemaProperty;
 enum class Status : uint8_t {
   CONSENSUS_PARTICIPANT,
   INITIALIZING,
-  PENDING_SPLIT,
   // TODO(tyler) this will possibly have more states,
   // depending on the reconfiguration protocol that we
   // implement.
 };
 
-struct PeerMetadata {
+struct AddressAndStatus {
   memgraph::io::Address address;
   Status status;
-  boost::uuids::uuid split_from;
 
-  friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
+  friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
 
-  friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
-    in << "PeerMetadata { address: ";
-    in << peer_metadata.address;
-
-    if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
-      in << ", status: CONSENSUS_PARTICIPANT";
-    } else if (peer_metadata.status == Status::INITIALIZING) {
-      in << ", status: INITIALIZING";
-    } else if (peer_metadata.status == Status::PENDING_SPLIT) {
-      in << ", status: PENDING_SPLIT";
+  friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
+    in << "AddressAndStatus { address: ";
+    in << address_and_status.address;
+    if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
+      in << ", status: CONSENSUS_PARTICIPANT }";
     } else {
-      MG_ASSERT(false, "failed to update the operator<< implementation for Status");
+      in << ", status: INITIALIZING }";
     }
 
-    in << ", split_from: " << peer_metadata.split_from << " }";
-
     return in;
   }
 
-  friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
+  friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
+    return lhs.address == rhs.address;
+  }
 };
 
 using PrimaryKey = std::vector<PropertyValue>;
-using ShardId = std::pair<LabelId, PrimaryKey>;
 
 struct ShardMetadata {
-  std::vector<PeerMetadata> peers;
-  Hlc version;
-  std::optional<msgs::SuggestedSplitInfo> pending_split;
-
-  bool Underreplicated(size_t replication_factor) { return peers.size() < replication_factor; }
-
-  bool ContainsPeer(io::PartialAddress machine) {
-    for (const auto &peer : peers) {
-      if (peer.address.last_known_ip == machine.ip && peer.address.last_known_port == machine.port) {
-        return true;
-      }
-    }
-    return false;
-  }
+  std::vector<AddressAndStatus> peers;
+  uint64_t version;
 
   friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
     using utils::print_helpers::operator<<;
@@ -115,9 +93,7 @@ struct ShardMetadata {
     return in;
   }
 
-  friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) {
-    return lhs.version == rhs.version && lhs.peers == rhs.peers;
-  };
+  friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
 
   friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
     if (lhs.peers != rhs.peers) {
@@ -136,7 +112,6 @@ using PropertyMap = std::map<PropertyName, PropertyId>;
 using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
 
 struct ShardToInitialize {
-  Hlc new_shard_version;
   boost::uuids::uuid uuid;
   LabelId label_id;
   PrimaryKey min_key;
@@ -146,25 +121,6 @@ struct ShardToInitialize {
   std::unordered_map<uint64_t, std::string> id_to_names;
 };
 
-struct ShardToSplit {
-  PrimaryKey split_key;
-  Hlc old_shard_version;
-  Hlc new_shard_version;
-  std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
-};
-
-struct HeartbeatRequest {
-  Address from_storage_manager;
-  std::map<boost::uuids::uuid, ShardId> initialized_rsms;
-  std::set<msgs::SuggestedSplitInfo> suggested_splits;
-};
-
-struct HeartbeatResponse {
-  std::vector<ShardToInitialize> shards_to_initialize;
-  std::vector<ShardToSplit> shards_to_split;
-  std::vector<boost::uuids::uuid> acknowledged_initialized_rsms;
-};
-
 PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
 
 struct LabelSpace {
@@ -172,7 +128,6 @@ struct LabelSpace {
   // Maps between the smallest primary key stored in the shard and the shard
   std::map<PrimaryKey, ShardMetadata> shards;
   size_t replication_factor;
-  uint64_t split_threshold;
 
   friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
     using utils::print_helpers::operator<<;
@@ -208,19 +163,17 @@ struct ShardMap {
   // TODO(gabor) later we will want to update the wallclock time with
   // the given Io<impl>'s time as well
   Hlc IncrementShardMapVersion() noexcept;
-  Hlc GetHlc() noexcept;
+  Hlc GetHlc() const noexcept;
 
   std::unordered_map<uint64_t, std::string> IdToNames();
 
   // Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
-  HeartbeatResponse AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized,
-                                 std::set<msgs::SuggestedSplitInfo> pending_splits);
+  std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
+
+  bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
 
   std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
-                                            size_t replication_factor, uint64_t split_threshold,
-                                            Hlc last_shard_map_version);
-
-  boost::uuids::uuid NewShardUuid();
+                                            size_t replication_factor, Hlc last_shard_map_version);
 
   void AddServer(Address server_address);
 
@@ -246,9 +199,6 @@ struct ShardMap {
   /// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
   /// there is also an active leader for each shard.
   bool ClusterInitialized() const;
-
-  /// Returns the current count of all shards that are fully initialized.
-  size_t InitializedShards() const;
 };
 
 }  // namespace memgraph::coordinator

src/io/local_transport/local_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,6 @@
 #include "io/local_transport/local_transport_handle.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
-#include "utils/concrete_msg_sender.hpp"
 
 namespace memgraph::io::local_transport {
 
@@ -31,19 +30,19 @@ class LocalTransport {
   explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
       : local_transport_handle_(std::move(local_transport_handle)) {}
 
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
                                     std::function<void()> fill_notifier, Duration timeout) {
     return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
         to_address, from_address, std::move(request), timeout, fill_notifier);
   }
 
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
     return local_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
   }
 
-  template <utils::Message M>
+  template <Message M>
   void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
     return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
   }

src/io/local_transport/local_transport_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,6 @@
 #include "io/message_histogram_collector.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
-#include "utils/concrete_msg_sender.hpp"
 
 namespace memgraph::io::local_transport {
 
@@ -31,7 +30,7 @@ class LocalTransportHandle {
   mutable std::condition_variable cv_;
   bool should_shut_down_ = false;
   MessageHistogramCollector histograms_;
-  RequestId request_id_counter_ = 1;
+  RequestId request_id_counter_ = 0;
 
   // the responses to requests that are being waited on
   std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
@@ -68,7 +67,7 @@ class LocalTransportHandle {
     return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
   }
 
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
     std::unique_lock lock(mu_);
 
@@ -104,7 +103,7 @@ class LocalTransportHandle {
     return std::move(m_opt).value();
   }
 
-  template <utils::Message M>
+  template <Message M>
   void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
     auto type_info = TypeInfoFor(message);
 
@@ -139,7 +138,7 @@ class LocalTransportHandle {
     cv_.notify_all();
   }
 
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
                                           Duration timeout, std::function<void()> fill_notifier) {
     auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
@@ -156,7 +155,7 @@ class LocalTransportHandle {
     const auto now = Now();
     const Time deadline = now + timeout;
 
-    RequestId request_id;
+    RequestId request_id = 0;
     {
       std::unique_lock<std::mutex> lock(mu_);
 

src/io/message_conversion.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,6 @@
 #include <boost/core/demangle.hpp>
 
 #include "io/transport.hpp"
-#include "utils/concrete_msg_sender.hpp"
 #include "utils/type_info_ref.hpp"
 
 namespace memgraph::io {
@@ -49,7 +48,7 @@ struct OpaqueMessage {
   /// Return is the full std::variant<Ts...> type that holds the
   /// full parameter pack without interfering with recursive
   /// narrowing expansion.
-  template <typename Return, utils::Message Head, utils::Message... Rest>
+  template <typename Return, Message Head, Message... Rest>
   std::optional<Return> Unpack(std::any &&a) {
     if (typeid(Head) == a.type()) {
       Head concrete = std::any_cast<Head>(std::move(a));
@@ -68,12 +67,12 @@ struct OpaqueMessage {
   /// parameter pack for the types that they want to compare
   /// with the any and potentially include in the returned
   /// variant.
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
     return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
   }
 
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
     std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
 

src/io/messages.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,8 +13,8 @@
 
 #include <variant>
 
-#include "coordinator/coordinator.hpp"
-#include "io/rsm/raft.hpp"
+#include <coordinator/coordinator.hpp>
+#include <io/rsm/raft.hpp>
 #include "query/v2/requests.hpp"
 #include "utils/concepts.hpp"
 
@@ -41,6 +41,6 @@ using CoordinatorMessages =
 using ShardMessages = std::variant<ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, AppendResponse,
                                    WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>;
 
-using ShardManagerMessages = std::variant<msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>;
+using ShardManagerMessages = std::variant<WriteResponse<CoordinatorWriteResponses>>;
 
 }  // namespace memgraph::io::messages

src/io/rsm/raft.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,23 +19,17 @@
 #include <map>
 #include <set>
 #include <thread>
-#include <type_traits>
 #include <vector>
 
 #include <boost/core/demangle.hpp>
 
 #include "io/message_conversion.hpp"
-#include "io/rsm/shard_rsm.hpp"
 #include "io/simulator/simulator.hpp"
 #include "io/transport.hpp"
-#include "query/v2/requests.hpp"
 #include "utils/concepts.hpp"
 
 namespace memgraph::io::rsm {
 
-template <typename Type>
-concept HasShouldSplit = std::is_member_function_pointer<decltype(&Type::ShouldSplit)>::value;
-
 /// Timeout and replication tunables
 using namespace std::chrono_literals;
 static constexpr auto kMinimumElectionTimeout = 100ms;
@@ -348,21 +342,15 @@ class Raft {
     }
   }
 
-  template <typename = void>
-  requires HasShouldSplit<ReplicatedState> std::optional<msgs::SuggestedSplitInfo> ShouldSplit() {
-    return replicated_state_.ShouldSplit();
-  }
-
- private :
-     // Raft paper - 5.3
-     // When the entry has been safely replicated, the leader applies the
-     // entry to its state machine and returns the result of that
-     // execution to the client.
-     //
-     // "Safely replicated" is defined as being known to be present
-     // on at least a majority of all peers (inclusive of the Leader).
-     void
-     BumpCommitIndexAndReplyToClients(Leader &leader) {
+ private:
+  // Raft paper - 5.3
+  // When the entry has been safely replicated, the leader applies the
+  // entry to its state machine and returns the result of that
+  // execution to the client.
+  //
+  // "Safely replicated" is defined as being known to be present
+  // on at least a majority of all peers (inclusive of the Leader).
+  void BumpCommitIndexAndReplyToClients(Leader &leader) {
     auto confirmed_log_sizes = std::vector<LogSize>{};
 
     // We include our own log size in the calculation of the log
@@ -414,17 +402,13 @@ class Raft {
         const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
         leader.pending_client_requests.erase(apply_index);
 
-        if (client_request.request_id == 0) {
-          Log("not replying to Raft request with explicit request ID of 0");
-        } else {
-          const WriteResponse<WriteResponseValue> resp{
-              .success = true,
-              .write_return = std::move(write_return),
-              .raft_index = apply_index,
-          };
+        const WriteResponse<WriteResponseValue> resp{
+            .success = true,
+            .write_return = std::move(write_return),
+            .raft_index = apply_index,
+        };
 
-          io_.Send(client_request.address, client_request.request_id, std::move(resp));
-        }
+        io_.Send(client_request.address, client_request.request_id, std::move(resp));
       }
     }
 
@@ -640,7 +624,7 @@ class Raft {
       MG_ASSERT(std::max(req.term, state_.term) == req.term);
     }
 
-    VoteResponse res{
+    const VoteResponse res{
         .term = std::max(req.term, state_.term),
         .committed_log_size = state_.committed_log_size,
         .vote_granted = new_leader,

src/io/simulator/simulator_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -32,7 +32,6 @@
 #include "io/simulator/simulator_stats.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
-#include "utils/concrete_msg_sender.hpp"
 
 namespace memgraph::io::simulator {
 
@@ -59,7 +58,7 @@ class SimulatorHandle {
   std::uniform_int_distribution<int> drop_distrib_{0, 99};
   SimulatorConfig config_;
   MessageHistogramCollector histograms_;
-  RequestId request_id_counter_{1};
+  RequestId request_id_counter_{0};
 
   bool TimeoutPromisesPastDeadline() {
     bool timed_anything_out = false;
@@ -106,7 +105,7 @@ class SimulatorHandle {
 
   bool ShouldShutDown() const;
 
-  template <utils::Message Request, utils::Message Response>
+  template <Message Request, Message Response>
   ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
                                          std::function<bool()> &&maybe_tick_simulator,
                                          std::function<void()> &&fill_notifier) {
@@ -156,7 +155,7 @@ class SimulatorHandle {
     return std::move(future);
   }
 
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
     std::unique_lock<std::mutex> lock(mu_);
 
@@ -194,7 +193,7 @@ class SimulatorHandle {
     return TimedOut{};
   }
 
-  template <utils::Message M>
+  template <Message M>
   void Send(Address to_address, Address from_address, RequestId request_id, M message) {
     spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
     auto type_info = TypeInfoFor(message);

src/io/simulator/simulator_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,6 @@
 #include "io/notifier.hpp"
 #include "io/simulator/simulator_handle.hpp"
 #include "io/time.hpp"
-#include "utils/concrete_msg_sender.hpp"
 
 namespace memgraph::io::simulator {
 
@@ -34,7 +33,7 @@ class SimulatorTransport {
   SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
       : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
 
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
                                     std::function<void()> notification, Duration timeout) {
     std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
@@ -45,12 +44,12 @@ class SimulatorTransport {
         to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
   }
 
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
     return simulator_handle_->template Receive<Ms...>(receiver_address, timeout);
   }
 
-  template <utils::Message M>
+  template <Message M>
   void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
     return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
   }

src/io/transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,16 +22,21 @@
 #include "io/message_histogram_collector.hpp"
 #include "io/notifier.hpp"
 #include "io/time.hpp"
-#include "utils/concrete_msg_sender.hpp"
 #include "utils/result.hpp"
 
 namespace memgraph::io {
 
 using memgraph::utils::BasicResult;
 
+// TODO(tyler) ensure that Message continues to represent
+// reasonable constraints around message types over time,
+// as we adapt things to use Thrift-generated message types.
+template <typename T>
+concept Message = std::same_as<T, std::decay_t<T>>;
+
 using RequestId = uint64_t;
 
-template <utils::Message M>
+template <Message M>
 struct ResponseEnvelope {
   M message;
   RequestId request_id;
@@ -40,16 +45,16 @@ struct ResponseEnvelope {
   Duration response_latency;
 };
 
-template <utils::Message M>
+template <Message M>
 using ResponseResult = BasicResult<TimedOut, ResponseEnvelope<M>>;
 
-template <utils::Message M>
-using ResponseFuture = Future<ResponseResult<M>>;
+template <Message M>
+using ResponseFuture = memgraph::io::Future<ResponseResult<M>>;
 
-template <utils::Message M>
-using ResponsePromise = Promise<ResponseResult<M>>;
+template <Message M>
+using ResponsePromise = memgraph::io::Promise<ResponseResult<M>>;
 
-template <utils::Message... Ms>
+template <Message... Ms>
 struct RequestEnvelope {
   std::variant<Ms...> message;
   RequestId request_id;
@@ -57,7 +62,7 @@ struct RequestEnvelope {
   Address from_address;
 };
 
-template <utils::Message... Ms>
+template <Message... Ms>
 using RequestResult = BasicResult<TimedOut, RequestEnvelope<Ms...>>;
 
 template <typename I>
@@ -77,7 +82,7 @@ class Io {
   Duration GetDefaultTimeout() { return default_timeout_; }
 
   /// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
     const Address from_address = address_;
     std::function<void()> fill_notifier = nullptr;
@@ -87,7 +92,7 @@ class Io {
 
   /// Issue a request that times out after the default timeout. This tends
   /// to be used by clients.
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
     const Duration timeout = default_timeout_;
     const Address from_address = address_;
@@ -97,7 +102,7 @@ class Io {
   }
 
   /// Issue a request that will notify a Notifier when it is filled or times out.
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
                                                     ReadinessToken readiness_token) {
     const Duration timeout = default_timeout_;
@@ -108,7 +113,7 @@ class Io {
   }
 
   /// Issue a request that will notify a Notifier when it is filled or times out.
-  template <utils::Message RequestT, utils::Message ResponseT>
+  template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
                                                               ReadinessToken readiness_token, Duration timeout) {
     const Address from_address = address_;
@@ -119,14 +124,14 @@ class Io {
 
   /// Wait for an explicit number of microseconds for a request of one of the
   /// provided types to arrive. This tends to be used by servers.
-  template <utils::Message... Ms>
+  template <Message... Ms>
   RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
     return implementation_.template Receive<Ms...>(address_, timeout);
   }
 
   /// Wait the default number of microseconds for a request of one of the
   /// provided types to arrive. This tends to be used by servers.
-  template <utils::Message... Ms>
+  template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
     const Duration timeout = default_timeout_;
     return implementation_.template Receive<Ms...>(address_, timeout);
@@ -135,7 +140,7 @@ class Io {
   /// Send a message in a best-effort fashion. This is used for messaging where
   /// responses are not necessarily expected, and for servers to respond to requests.
   /// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
-  template <utils::Message M>
+  template <Message M>
   void Send(Address to_address, RequestId request_id, M message) {
     Address from_address = address_;
     return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
@@ -156,7 +161,6 @@ class Io {
   }
 
   Address GetAddress() { return address_; }
-
   void SetAddress(Address address) { address_ = address; }
 
   Io<I> ForkLocal(boost::uuids::uuid uuid) {
@@ -169,16 +173,5 @@ class Io {
   }
 
   LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
-
-  template <utils::Message M>
-  utils::Sender<M> GetSender(Address address) {
-    Io<I> io_copy = Io(implementation_, address_);
-
-    std::function<void(M)> sender = [address, io_copy](M &&message) mutable {
-      io_copy.template Send<M>(address, 0, std::forward<M>(message));
-    };
-
-    return utils::Sender{sender};
-  }
 };
 };  // namespace memgraph::io

src/machine_manager/machine_manager.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -127,8 +127,7 @@ class MachineManager {
           std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
                        WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse,
                        WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
-                       AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>, msgs::SuggestedSplitInfo,
-                       msgs::InitializeSplitShard>;
+                       AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
 
       spdlog::info("MM waiting on Receive on address {}", io_.GetAddress().ToString());
 
@@ -136,8 +135,8 @@ class MachineManager {
       auto request_result = io_.template ReceiveWithTimeout<
           ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
           WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse, WriteResponse<CoordinatorWriteResponses>,
-          ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>,
-          msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>(receive_timeout);
+          ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>(
+          receive_timeout);
 
       if (request_result.HasError()) {
         // time to do Cron
@@ -176,8 +175,7 @@ class MachineManager {
       spdlog::info("smm: {}", shard_manager_.GetAddress().ToString());
       if (to_sm) {
         std::optional<ShardManagerMessages> conversion_attempt =
-            ConvertVariant<AllMessages, msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>(
-                std::move(request_envelope.message));
+            ConvertVariant<AllMessages, WriteResponse<CoordinatorWriteResponses>>(std::move(request_envelope.message));
 
         MG_ASSERT(conversion_attempt.has_value(), "shard manager message conversion failed");
 

src/query/v2/interpreter.cpp

@@ -64,7 +64,11 @@
 #include "utils/tsc.hpp"
 #include "utils/variant_helpers.hpp"
 
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
+
 namespace EventCounter {
+
 extern Event ReadQuery;
 extern Event WriteQuery;
 extern Event ReadWriteQuery;
@@ -74,6 +78,7 @@ extern const Event LabelPropertyIndexCreated;
 
 extern const Event StreamsCreated;
 extern const Event TriggersCreated;
+
 }  // namespace EventCounter
 
 namespace memgraph::query::v2 {
@@ -688,7 +693,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
     : plan_(plan),
       cursor_(plan->plan().MakeCursor(execution_memory)),
       frame_(plan->symbol_table().max_position(), execution_memory),
-      multi_frame_(plan->symbol_table().max_position(), kNumberOfFramesInMultiframe, execution_memory),
+      multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
       memory_limit_(memory_limit) {
   ctx_.db_accessor = dba;
   ctx_.symbol_table = plan->symbol_table();
@@ -812,8 +817,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStrea
 std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
                                                                 const std::vector<Symbol> &output_symbols,
                                                                 std::map<std::string, TypedValue> *summary) {
-  auto should_pull_multiple = false;  // TODO on the long term, we will only use PullMultiple
-  if (should_pull_multiple) {
+  if (FLAGS_use_multi_frame) {
     return PullMultiple(stream, n, output_symbols, summary);
   }
   // Set up temporary memory for a single Pull. Initial memory comes from the

src/query/v2/interpreter.hpp

@@ -295,7 +295,6 @@ class Interpreter final {
   void Abort();
 
   const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
-
   void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
     request_router_->InstallSimulatorTicker(tick_simulator);
   }

src/query/v2/multiframe.cpp

@@ -17,6 +17,9 @@
 #include "query/v2/bindings/frame.hpp"
 #include "utils/pmr/vector.hpp"
 
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
+
 namespace memgraph::query::v2 {
 
 static_assert(std::forward_iterator<ValidFramesReader::Iterator>);

src/query/v2/multiframe.hpp

@@ -13,10 +13,14 @@
 
 #include <iterator>
 
+#include <gflags/gflags.h>
+
 #include "query/v2/bindings/frame.hpp"
 
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint64(default_multi_frame_size);
+
 namespace memgraph::query::v2 {
-constexpr uint64_t kNumberOfFramesInMultiframe = 1000;  // TODO have it configurable
 
 class ValidFramesConsumer;
 class ValidFramesModifier;

src/query/v2/plan/operator.cpp

@@ -218,6 +218,7 @@ class DistributedCreateNodeCursor : public Cursor {
   }
 
   std::vector<msgs::NewVertex> NodeCreationInfoToRequest(ExecutionContext &context, Frame &frame) {
+    primary_keys_.clear();
     std::vector<msgs::NewVertex> requests;
     msgs::PrimaryKey pk;
     msgs::NewVertex rqst;
@@ -227,22 +228,27 @@ class DistributedCreateNodeCursor : public Cursor {
     ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr,
                                   storage::v3::View::NEW);
     if (const auto *node_info_properties = std::get_if<PropertiesMapList>(&node_info_.properties)) {
-      for (const auto &[key, value_expression] : *node_info_properties) {
+      for (const auto &[property, value_expression] : *node_info_properties) {
         TypedValue val = value_expression->Accept(evaluator);
-        if (context.request_router->IsPrimaryKey(primary_label, key)) {
-          rqst.primary_key.push_back(TypedValueToValue(val));
-          pk.push_back(TypedValueToValue(val));
+        auto msgs_value = TypedValueToValue(val);
+        if (context.request_router->IsPrimaryProperty(primary_label, property)) {
+          rqst.primary_key.push_back(msgs_value);
+          pk.push_back(std::move(msgs_value));
+        } else {
+          rqst.properties.emplace_back(property, std::move(msgs_value));
         }
       }
     } else {
       auto property_map = evaluator.Visit(*std::get<ParameterLookup *>(node_info_.properties)).ValueMap();
-      for (const auto &[key, value] : property_map) {
-        auto key_str = std::string(key);
-        auto property_id = context.request_router->NameToProperty(key_str);
-        if (context.request_router->IsPrimaryKey(primary_label, property_id)) {
-          rqst.primary_key.push_back(TypedValueToValue(value));
-          pk.push_back(TypedValueToValue(value));
-        }
+      for (const auto &[property, typed_value] : property_map) {
+        auto property_str = std::string(property);
+        auto property_id = context.request_router->NameToProperty(property_str);
+        auto msgs_value = TypedValueToValue(typed_value);
+        if (context.request_router->IsPrimaryProperty(primary_label, property_id)) {
+          rqst.primary_key.push_back(msgs_value);
+          pk.push_back(std::move(msgs_value));
+        } else
+          rqst.properties.emplace_back(property_id, std::move(msgs_value));
       }
     }
 
@@ -268,6 +274,7 @@ class DistributedCreateNodeCursor : public Cursor {
   }
 
   std::vector<msgs::NewVertex> NodeCreationInfoToRequests(ExecutionContext &context, MultiFrame &multi_frame) {
+    primary_keys_.clear();
     std::vector<msgs::NewVertex> requests;
     auto multi_frame_modifier = multi_frame.GetValidFramesModifier();
     for (auto &frame : multi_frame_modifier) {
@@ -280,22 +287,27 @@ class DistributedCreateNodeCursor : public Cursor {
       ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, nullptr,
                                     storage::v3::View::NEW);
       if (const auto *node_info_properties = std::get_if<PropertiesMapList>(&node_info_.properties)) {
-        for (const auto &[key, value_expression] : *node_info_properties) {
+        for (const auto &[property, value_expression] : *node_info_properties) {
           TypedValue val = value_expression->Accept(evaluator);
-          if (context.request_router->IsPrimaryKey(primary_label, key)) {
-            rqst.primary_key.push_back(TypedValueToValue(val));
-            pk.push_back(TypedValueToValue(val));
+          auto msgs_value = TypedValueToValue(val);
+          if (context.request_router->IsPrimaryProperty(primary_label, property)) {
+            rqst.primary_key.push_back(msgs_value);
+            pk.push_back(std::move(msgs_value));
+          } else {
+            rqst.properties.emplace_back(property, std::move(msgs_value));
           }
         }
       } else {
         auto property_map = evaluator.Visit(*std::get<ParameterLookup *>(node_info_.properties)).ValueMap();
-        for (const auto &[key, value] : property_map) {
-          auto key_str = std::string(key);
-          auto property_id = context.request_router->NameToProperty(key_str);
-          if (context.request_router->IsPrimaryKey(primary_label, property_id)) {
-            rqst.primary_key.push_back(TypedValueToValue(value));
-            pk.push_back(TypedValueToValue(value));
-          }
+        for (const auto &[property, typed_value] : property_map) {
+          auto property_str = std::string(property);
+          auto property_id = context.request_router->NameToProperty(property_str);
+          auto msgs_value = TypedValueToValue(typed_value);
+          if (context.request_router->IsPrimaryProperty(primary_label, property_id)) {
+            rqst.primary_key.push_back(msgs_value);
+            pk.push_back(std::move(msgs_value));
+          } else
+            rqst.properties.emplace_back(property_id, std::move(msgs_value));
         }
       }
 
@@ -497,7 +509,7 @@ class DistributedScanAllAndFilterCursor : public Cursor {
 
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
+                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -693,7 +705,7 @@ class DistributedScanByPrimaryKeyCursor : public Cursor {
   void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
+                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -761,7 +773,7 @@ class DistributedScanByPrimaryKeyCursor : public Cursor {
               output_frame[output_symbol_] = TypedValue(it->second);
               populated_any = true;
               ++output_frame_it;
-            }           
+            }
             own_frames_it_->MakeInvalid();
           }
           break;
@@ -1333,28 +1345,47 @@ bool ContainsSameEdge(const TypedValue &a, const TypedValue &b) {
 
   return a.ValueEdge() == b.ValueEdge();
 }
+
+bool IsExpansionOk(Frame &frame, const Symbol &expand_symbol, const std::vector<Symbol> &previous_symbols) {
+  // This shouldn't raise a TypedValueException, because the planner
+  // makes sure these are all of the expected type. In case they are not
+  // an error should be raised long before this code is executed.
+  return std::ranges::all_of(previous_symbols,
+                             [&frame, &expand_value = frame[expand_symbol]](const auto &previous_symbol) {
+                               const auto &previous_value = frame[previous_symbol];
+                               return !ContainsSameEdge(previous_value, expand_value);
+                             });
+}
+
 }  // namespace
 
 bool EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Pull(Frame &frame, ExecutionContext &context) {
   SCOPED_PROFILE_OP("EdgeUniquenessFilter");
-
-  auto expansion_ok = [&]() {
-    const auto &expand_value = frame[self_.expand_symbol_];
-    for (const auto &previous_symbol : self_.previous_symbols_) {
-      const auto &previous_value = frame[previous_symbol];
-      // This shouldn't raise a TypedValueException, because the planner
-      // makes sure these are all of the expected type. In case they are not
-      // an error should be raised long before this code is executed.
-      if (ContainsSameEdge(previous_value, expand_value)) return false;
-    }
-    return true;
-  };
-
   while (input_cursor_->Pull(frame, context))
-    if (expansion_ok()) return true;
+    if (IsExpansionOk(frame, self_.expand_symbol_, self_.previous_symbols_)) return true;
   return false;
 }
 
+bool EdgeUniquenessFilter::EdgeUniquenessFilterCursor::PullMultiple(MultiFrame &output_multi_frame,
+                                                                    ExecutionContext &context) {
+  SCOPED_PROFILE_OP("EdgeUniquenessFilterMF");
+  auto populated_any = false;
+
+  while (output_multi_frame.HasInvalidFrame()) {
+    if (!input_cursor_->PullMultiple(output_multi_frame, context)) {
+      return populated_any;
+    }
+    for (auto &frame : output_multi_frame.GetValidFramesConsumer()) {
+      if (IsExpansionOk(frame, self_.expand_symbol_, self_.previous_symbols_)) {
+        populated_any = true;
+      } else {
+        frame.MakeInvalid();
+      }
+    }
+  }
+  return populated_any;
+}
+
 void EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Shutdown() { input_cursor_->Shutdown(); }
 
 void EdgeUniquenessFilter::EdgeUniquenessFilterCursor::Reset() { input_cursor_->Reset(); }
@@ -1575,6 +1606,7 @@ class AggregateCursor : public Cursor {
         ExpressionEvaluator evaluator(&frame, context->symbol_table, context->evaluation_context,
                                       context->request_router, storage::v3::View::NEW);
         ProcessOne(frame, &evaluator);
+        frame.MakeInvalid();
       }
     }
 
@@ -2181,7 +2213,7 @@ class UnwindCursor : public Cursor {
 
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
+                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -2940,27 +2972,16 @@ class DistributedCreateExpandCursor : public Cursor {
       const auto &v1 = v1_value.ValueVertex();
       const auto &v2 = OtherVertex(frame);
 
-      // Set src and dest vertices
-      // TODO(jbajic) Currently we are only handling scenario where vertices
-      // are matched
-      const auto set_vertex = [&context](const auto &vertex, auto &vertex_id) {
-        vertex_id.first = vertex.PrimaryLabel();
-        for (const auto &[key, val] : vertex.Properties()) {
-          if (context.request_router->IsPrimaryKey(vertex_id.first.id, key)) {
-            vertex_id.second.push_back(val);
-          }
-        }
-      };
       std::invoke([&]() {
         switch (edge_info.direction) {
           case EdgeAtom::Direction::IN: {
-            set_vertex(v2, request.src_vertex);
-            set_vertex(v1, request.dest_vertex);
+            request.src_vertex = v2.Id();
+            request.dest_vertex = v1.Id();
             break;
           }
           case EdgeAtom::Direction::OUT: {
-            set_vertex(v1, request.src_vertex);
-            set_vertex(v2, request.dest_vertex);
+            request.src_vertex = v1.Id();
+            request.dest_vertex = v2.Id();
             break;
           }
           case EdgeAtom::Direction::BOTH:
@@ -3102,6 +3123,9 @@ class DistributedExpandCursor : public Cursor {
       auto &vertex = vertex_value.ValueVertex();
       msgs::ExpandOneRequest request;
       request.direction = DirectionToMsgsDirection(self_.common_.direction);
+      std::transform(self_.common_.edge_types.begin(), self_.common_.edge_types.end(),
+                     std::back_inserter(request.edge_types),
+                     [](const storage::v3::EdgeTypeId edge_type_id) { return msgs::EdgeType{edge_type_id}; });
       // to not fetch any properties of the edges
       request.edge_properties.emplace();
       request.src_vertices.push_back(vertex.Id());
@@ -3242,6 +3266,9 @@ class DistributedExpandCursor : public Cursor {
 
     msgs::ExpandOneRequest request;
     request.direction = DirectionToMsgsDirection(self_.common_.direction);
+    std::transform(self_.common_.edge_types.begin(), self_.common_.edge_types.end(),
+                   std::back_inserter(request.edge_types),
+                   [](const storage::v3::EdgeTypeId edge_type_id) { return msgs::EdgeType{edge_type_id}; });
     // to not fetch any properties of the edges
     request.edge_properties.emplace();
     for (const auto &frame : own_multi_frame_->GetValidFramesReader()) {
@@ -3355,7 +3382,7 @@ class DistributedExpandCursor : public Cursor {
   void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
+                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }

src/query/v2/plan/operator.lcp

@@ -1570,6 +1570,7 @@ edge lists).")
      EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
                                 utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 

src/query/v2/plan/rewrite/index_lookup.hpp

@@ -597,6 +597,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
                      [](const auto &schema_elem) { return schema_elem.property_id; });
 
       for (const auto &property_filter : property_filters) {
+        if (property_filter.property_filter->type_ != PropertyFilter::Type::EQUAL) {
+          continue;
+        }
         const auto &property_id = db_->NameToProperty(property_filter.property_filter->property_.name);
         if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) {
           pk_temp.emplace_back(std::make_pair(property_filter.expression, property_filter));

src/query/v2/request_router.hpp

@@ -117,7 +117,7 @@ class RequestRouterInterface {
   virtual std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const = 0;
   virtual std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const = 0;
   virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0;
-  virtual bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0;
+  virtual bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0;
 
   virtual std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) = 0;
   virtual void InstallSimulatorTicker(std::function<bool()> tick_simulator) = 0;
@@ -231,7 +231,7 @@ class RequestRouter : public RequestRouterInterface {
     return edge_types_.IdToName(id.AsUint());
   }
 
-  bool IsPrimaryKey(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
+  bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
     const auto schema_it = shards_map_.schemas.find(primary_label);
     MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
 
@@ -420,7 +420,7 @@ class RequestRouter : public RequestRouterInterface {
       const std::vector<msgs::NewVertex> &new_vertices) {
     std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
 
-    for (const auto &new_vertex : new_vertices) {
+    for (auto &new_vertex : new_vertices) {
       MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
       auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
                                               storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
@@ -463,7 +463,7 @@ class RequestRouter : public RequestRouterInterface {
 
       ensure_shard_exists_in_table(shard_src_vertex);
 
-      if (shard_src_vertex.peers != shard_dest_vertex.peers) {
+      if (shard_src_vertex != shard_dest_vertex) {
         ensure_shard_exists_in_table(shard_dest_vertex);
         per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
       }
@@ -800,19 +800,15 @@ class SimulatedRequestRouterFactory : public RequestRouterFactory {
     io::Address unique_local_addr_query;
 
     // The simulated RR should not introduce stochastic behavior.
-    random_uuid = boost::uuids::uuid{3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-    unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
+    random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
 
     auto io = simulator_->Register(unique_local_addr_query);
     auto query_io = io.ForkLocal(random_uuid);
 
-    auto ret = std::make_unique<RequestRouter<TransportType>>(
+    return std::make_unique<RequestRouter<TransportType>>(
         coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
         std::move(io));
-
-    ret->InstallSimulatorTicker(simulator_->GetSimulatorTickClosure());
-
-    return ret;
   }
 };
 

src/query/v2/requests.hpp

@@ -12,7 +12,6 @@
 #pragma once
 
 #include <chrono>
-#include <cstdint>
 #include <iostream>
 #include <map>
 #include <memory>
@@ -22,13 +21,10 @@
 #include <variant>
 #include <vector>
 
-#include <boost/uuid/uuid.hpp>
-
 #include "coordinator/hybrid_logical_clock.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
-#include "storage/v3/shard.hpp"
 #include "utils/fnv.hpp"
 
 namespace memgraph::msgs {
@@ -575,64 +571,13 @@ struct CommitResponse {
   std::optional<ShardError> error;
 };
 
-struct SuggestedSplitInfo {
-  boost::uuids::uuid shard_to_split_uuid;
-  LabelId label_id;
-  PrimaryKey splitting_shard_low_key;
-  PrimaryKey split_key;
-  Hlc shard_version;
-
-  friend bool operator<(const SuggestedSplitInfo &lhs, const SuggestedSplitInfo &rhs) {
-    if (lhs.shard_to_split_uuid != rhs.shard_to_split_uuid) {
-      return lhs.shard_to_split_uuid < rhs.shard_to_split_uuid;
-    }
-
-    /*
-    TODO(tyler) fix this
-    if (lhs.split_key != rhs.split_key) {
-      return lhs.split_key < rhs.split_key;
-    }
-    */
-
-    return lhs.shard_version < rhs.shard_version;
-  }
-};
-
-struct SplitInfo {
-  PrimaryKey split_key;
-  boost::uuids::uuid right_side_uuid;
-  Hlc shard_version;
-};
-
-struct SplitRequest {
-  PrimaryKey split_key;
-  Hlc old_shard_version;
-  Hlc new_shard_version;
-  std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
-};
-
-struct InitializeSplitShard {
-  // TODO(jbajic) Make it unique by solving that is not copyable int std::any
-  std::shared_ptr<storage::v3::Shard> shard;
-  std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
-};
-
-struct InitializeSplitShardByUUID {
-  std::shared_ptr<storage::v3::Shard> shard;
-  boost::uuids::uuid shard_uuid;
-};
-
-struct SplitResponse {};
-
 using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
 using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse>;
 
-using WriteRequests =
-    std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest, CreateExpandRequest,
-                 DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest, SplitRequest>;
-using WriteResponses =
-    std::variant<CreateVerticesResponse, DeleteVerticesResponse, UpdateVerticesResponse, CreateExpandResponse,
-                 DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse, SplitResponse>;
+using WriteRequests = std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest,
+                                   CreateExpandRequest, DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest>;
+using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesResponse, UpdateVerticesResponse,
+                                    CreateExpandResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>;
 
 }  // namespace memgraph::msgs
 

src/storage/v3/CMakeLists.txt

@@ -18,7 +18,6 @@ set(storage_v3_src_files
     bindings/typed_value.cpp
     expr.cpp
     vertex.cpp
-    splitter.cpp
     request_helper.cpp)
 
 # ######################

src/storage/v3/config.hpp

@@ -30,10 +30,6 @@ struct Config {
     io::Duration reclamation_interval{};
   } gc;
 
-  struct Split {
-    uint64_t max_shard_vertex_size{500'000};
-  } split;
-
   struct Items {
     bool properties_on_edges{true};
   } items;

src/storage/v3/delta.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,14 +13,12 @@
 
 #include <cstdint>
 #include <memory>
-
 #include "storage/v3/edge_ref.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_id.hpp"
 #include "utils/logging.hpp"
-#include "utils/synchronized.hpp"
 
 namespace memgraph::storage::v3 {
 
@@ -29,11 +27,6 @@ struct Edge;
 struct Delta;
 struct CommitInfo;
 
-inline uint64_t GetNextDeltaId() noexcept {
-  static utils::Synchronized<uint64_t, utils::SpinLock> delta_id{0};
-  return delta_id.WithLock([](auto &id) { return id++; });
-}
-
 // This class stores one of three pointers (`Delta`, `Vertex` and `Edge`)
 // without using additional memory for storing the type. The type is stored in
 // the pointer itself in the lower bits. All of those structures contain large
@@ -165,54 +158,46 @@ struct Delta {
   struct RemoveInEdgeTag {};
   struct RemoveOutEdgeTag {};
 
-  Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::DELETE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
+  Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
+      : action(Action::DELETE_OBJECT), commit_info(commit_info), command_id(command_id) {}
 
-  Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::RECREATE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
+  Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
+      : action(Action::RECREATE_OBJECT), commit_info(commit_info), command_id(command_id) {}
 
-  Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::ADD_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
+  Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
+      : action(Action::ADD_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
 
-  Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::REMOVE_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
+  Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
+      : action(Action::REMOVE_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
 
   Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, CommitInfo *commit_info,
-        uint64_t delta_id, uint64_t command_id)
-      : action(Action::SET_PROPERTY),
-        id(delta_id),
-        commit_info(commit_info),
-        command_id(command_id),
-        property({key, value}) {}
+        uint64_t command_id)
+      : action(Action::SET_PROPERTY), commit_info(commit_info), command_id(command_id), property({key, value}) {}
 
   Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t delta_id, uint64_t command_id)
+        uint64_t command_id)
       : action(Action::ADD_IN_EDGE),
-        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t delta_id, uint64_t command_id)
+        uint64_t command_id)
       : action(Action::ADD_OUT_EDGE),
-        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t delta_id, uint64_t command_id)
+        uint64_t command_id)
       : action(Action::REMOVE_IN_EDGE),
-        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t delta_id, uint64_t command_id)
+        uint64_t command_id)
       : action(Action::REMOVE_OUT_EDGE),
-        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
@@ -241,10 +226,8 @@ struct Delta {
     }
   }
 
-  friend bool operator==(const Delta &lhs, const Delta &rhs) noexcept { return lhs.id == rhs.id; }
-
   Action action;
-  uint64_t id;
+
   // TODO: optimize with in-place copy
   CommitInfo *commit_info;
   uint64_t command_id;

src/storage/v3/indices.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -325,7 +325,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
   }
 }
 
-LabelIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
+LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -353,7 +353,7 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
   }
 }
 
-LabelIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction,
+LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
                                Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
     : index_container_(&index_container),
       label_(label),
@@ -465,7 +465,7 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
   }
 }
 
-LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
+LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -526,7 +526,7 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
 const PropertyValue kSmallestTemporalData =
     PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
 
-LabelPropertyIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
+LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
                                        const std::optional<utils::Bound<PropertyValue>> &lower_bound,
                                        const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
                                        Transaction *transaction, Indices *indices, Config::Items config,

src/storage/v3/indices.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,8 +18,6 @@
 #include <utility>
 
 #include "storage/v3/config.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/transaction.hpp"
 #include "storage/v3/vertex_accessor.hpp"
@@ -42,18 +40,12 @@ class LabelIndex {
     bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
   };
 
-  using IndexType = LabelId;
-
  public:
-  using IndexContainer = std::set<Entry>;
+  using LabelIndexContainer = std::set<Entry>;
 
   LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
 
-  LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator,
-             std::map<LabelId, IndexContainer> &data)
-      : index_{std::move(data)}, indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
-
   /// @throw std::bad_alloc
   void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
 
@@ -71,12 +63,12 @@ class LabelIndex {
 
   class Iterable {
    public:
-    Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
+    Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
              Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, IndexContainer::iterator index_iterator);
+      Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -89,7 +81,7 @@ class LabelIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      IndexContainer::iterator index_iterator_;
+      LabelIndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
@@ -98,7 +90,7 @@ class LabelIndex {
     Iterator end() { return {this, index_container_->end()}; }
 
    private:
-    IndexContainer *index_container_;
+    LabelIndexContainer *index_container_;
     LabelId label_;
     View view_;
     Transaction *transaction_;
@@ -122,35 +114,8 @@ class LabelIndex {
 
   void Clear() { index_.clear(); }
 
-  std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
-    if (index_.empty()) {
-      return {};
-    }
-
-    // Cloned index entries will contain new index entry iterators, but old
-    // vertices address which need to be adjusted after extracting vertices
-    std::map<IndexType, IndexContainer> cloned_indices;
-    for (auto &[index_type_val, index] : index_) {
-      auto entry_it = index.begin();
-      auto &cloned_indices_container = cloned_indices[index_type_val];
-      while (entry_it != index.end()) {
-        // We need to save the next pointer since the current one will be
-        // invalidated after extract
-        auto next_entry_it = std::next(entry_it);
-        if (entry_it->vertex->first > split_key) {
-          [[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
-              cloned_indices_container.insert(index.extract(entry_it));
-          MG_ASSERT(inserted, "Failed to extract index entry!");
-        }
-        entry_it = next_entry_it;
-      }
-    }
-
-    return cloned_indices;
-  }
-
  private:
-  std::map<LabelId, IndexContainer> index_;
+  std::map<LabelId, LabelIndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;
@@ -168,10 +133,9 @@ class LabelPropertyIndex {
     bool operator<(const PropertyValue &rhs) const;
     bool operator==(const PropertyValue &rhs) const;
   };
-  using IndexType = std::pair<LabelId, PropertyId>;
 
  public:
-  using IndexContainer = std::set<Entry>;
+  using LabelPropertyIndexContainer = std::set<Entry>;
 
   LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
@@ -195,14 +159,14 @@ class LabelPropertyIndex {
 
   class Iterable {
    public:
-    Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
+    Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
              const std::optional<utils::Bound<PropertyValue>> &lower_bound,
              const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
              Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, IndexContainer::iterator index_iterator);
+      Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -215,7 +179,7 @@ class LabelPropertyIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      IndexContainer::iterator index_iterator_;
+      LabelPropertyIndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
@@ -224,7 +188,7 @@ class LabelPropertyIndex {
     Iterator end();
 
    private:
-    IndexContainer *index_container_;
+    LabelPropertyIndexContainer *index_container_;
     LabelId label_;
     PropertyId property_;
     std::optional<utils::Bound<PropertyValue>> lower_bound_;
@@ -265,35 +229,8 @@ class LabelPropertyIndex {
 
   void Clear() { index_.clear(); }
 
-  std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
-    if (index_.empty()) {
-      return {};
-    }
-
-    // Cloned index entries will contain new index entry iterators, but old
-    // vertices address which need to be adjusted after extracting vertices
-    std::map<IndexType, IndexContainer> cloned_indices;
-    for (auto &[index_type_val, index] : index_) {
-      auto entry_it = index.begin();
-      auto &cloned_index_container = cloned_indices[index_type_val];
-      while (entry_it != index.end()) {
-        // We need to save the next pointer since the current one will be
-        // invalidated after extract
-        auto next_entry_it = std::next(entry_it);
-        if (entry_it->vertex->first > split_key) {
-          [[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
-              cloned_index_container.insert(index.extract(entry_it));
-          MG_ASSERT(inserted, "Failed to extract index entry!");
-        }
-        entry_it = next_entry_it;
-      }
-    }
-
-    return cloned_indices;
-  }
-
  private:
-  std::map<std::pair<LabelId, PropertyId>, IndexContainer> index_;
+  std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;

src/storage/v3/mvcc.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -108,7 +108,7 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
 /// a `DELETE_OBJECT` delta).
 /// @throw std::bad_alloc
 inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
-  return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(), GetNextDeltaId(),
+  return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(),
                                            transaction->command_id);
 }
 
@@ -119,7 +119,7 @@ template <typename TObj, class... Args>
 requires utils::SameAsAnyOf<TObj, Edge, Vertex>
 inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
   auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
-                                                 GetNextDeltaId(), transaction->command_id);
+                                                 transaction->command_id);
   auto *delta_holder = GetDeltaHolder(object);
 
   // The operations are written in such order so that both `next` and `prev`

src/storage/v3/shard.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,14 +18,14 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <variant>
 
 #include <bits/ranges_algo.h>
+#include <gflags/gflags.h>
 #include <spdlog/spdlog.h>
 
 #include "io/network/endpoint.hpp"
 #include "io/time.hpp"
-#include "storage/v3/delta.hpp"
-#include "storage/v3/edge.hpp"
 #include "storage/v3/edge_accessor.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/indices.hpp"
@@ -36,7 +36,6 @@
 #include "storage/v3/result.hpp"
 #include "storage/v3/schema_validator.hpp"
 #include "storage/v3/transaction.hpp"
-#include "storage/v3/value_conversions.hpp"
 #include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "storage/v3/view.hpp"
@@ -324,80 +323,25 @@ bool VerticesIterable::Iterator::operator==(const Iterator &other) const {
 }
 
 Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key,
-             const std::optional<PrimaryKey> max_primary_key, std::vector<SchemaProperty> schema, Hlc shard_version,
-             Config config, std::unordered_map<uint64_t, std::string> id_to_name)
+             const std::optional<PrimaryKey> max_primary_key, std::vector<SchemaProperty> schema, Config config,
+             std::unordered_map<uint64_t, std::string> id_to_name)
     : primary_label_{primary_label},
       min_primary_key_{min_primary_key},
       max_primary_key_{max_primary_key},
-      shard_version_{shard_version},
       schema_validator_{schemas_, name_id_mapper_},
       vertex_validator_{schema_validator_, primary_label},
       indices_{config.items, vertex_validator_},
       isolation_level_{config.transaction.isolation_level},
       config_{config},
-      shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
-                      name_id_mapper_) {
+      uuid_{utils::GenerateUUID()},
+      epoch_id_{utils::GenerateUUID()},
+      global_locker_{file_retainer_.AddLocker()} {
   CreateSchema(primary_label_, schema);
   StoreMapping(std::move(id_to_name));
 }
 
-Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
-             std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
-             std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
-             const std::unordered_map<uint64_t, std::string> &id_to_name, const Hlc shard_version)
-    : primary_label_{primary_label},
-      min_primary_key_{min_primary_key},
-      max_primary_key_{max_primary_key},
-      vertices_(std::move(vertices)),
-      edges_(std::move(edges)),
-      shard_version_(shard_version),
-      schema_validator_{schemas_, name_id_mapper_},
-      vertex_validator_{schema_validator_, primary_label},
-      indices_{config.items, vertex_validator_},
-      isolation_level_{config.transaction.isolation_level},
-      config_{config},
-      start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)),
-      shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
-                      name_id_mapper_) {
-  CreateSchema(primary_label_, schema);
-  StoreMapping(id_to_name);
-}
-
-Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
-             std::vector<SchemaProperty> schema, VertexContainer &&vertices,
-             std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
-             const std::unordered_map<uint64_t, std::string> &id_to_name, const Hlc shard_version)
-    : primary_label_{primary_label},
-      min_primary_key_{min_primary_key},
-      max_primary_key_{max_primary_key},
-      vertices_(std::move(vertices)),
-      shard_version_(shard_version),
-      schema_validator_{schemas_, name_id_mapper_},
-      vertex_validator_{schema_validator_, primary_label},
-      indices_{config.items, vertex_validator_},
-      isolation_level_{config.transaction.isolation_level},
-      config_{config},
-      start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)),
-      shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
-                      name_id_mapper_) {
-  CreateSchema(primary_label_, schema);
-  StoreMapping(id_to_name);
-}
-
 Shard::~Shard() {}
 
-std::unique_ptr<Shard> Shard::FromSplitData(SplitData &&split_data) {
-  if (split_data.config.items.properties_on_edges) [[likely]] {
-    return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
-                                   split_data.schema, std::move(split_data.vertices), std::move(*split_data.edges),
-                                   std::move(split_data.transactions), split_data.config, split_data.id_to_name,
-                                   split_data.shard_version);
-  }
-  return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
-                                 split_data.schema, std::move(split_data.vertices), std::move(split_data.transactions),
-                                 split_data.config, split_data.id_to_name, split_data.shard_version);
-}
-
 Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
     : shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
 
@@ -492,7 +436,7 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
   }
 
   std::vector<EdgeAccessor> deleted_edges;
-  const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)};
+  const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)};  // TODO Replace
   for (const auto &item : in_edges) {
     auto [edge_type, from_vertex, edge] = item;
     EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
@@ -1104,42 +1048,6 @@ void Shard::StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name) {
   name_id_mapper_.StoreMapping(std::move(id_to_name));
 }
 
-std::optional<ShardSuggestedSplitInfo> Shard::ShouldSplit() const noexcept {
-  if (vertices_.size() >= config_.split.max_shard_vertex_size) {
-    spdlog::info("ShouldSplit is attempting to begin the split process");
-    auto mid_elem = vertices_.begin();
-    // TODO(tyler) the first time we calculate the split point, we should store it so that we don't have to
-    // iterate over half of the entire index each time Cron is run until the split succeeds.
-    std::ranges::advance(mid_elem, static_cast<VertexContainer::difference_type>(vertices_.size() / 2));
-    return ShardSuggestedSplitInfo{
-        .label_id = PrimaryLabel(),
-        .splitting_shard_low_key = min_primary_key_,
-        .split_key = mid_elem->first,
-        .shard_version = shard_version_,
-    };
-  }
-
-  spdlog::trace("not splitting because we have {} vertices, lower than split threshold of {}", vertices_.size(),
-                config_.split.max_shard_vertex_size);
-  return std::nullopt;
-}
-
-std::optional<SplitData> Shard::PerformSplit(const PrimaryKey &split_key, const Hlc old_shard_version,
-                                             const Hlc new_shard_version) {
-  if (old_shard_version < shard_version_) {
-    MG_ASSERT(false, "considering splitting 1");
-    spdlog::warn("Curent shard version {} is bigger than given {}", shard_version_, old_shard_version);
-    return std::nullopt;
-  }
-
-  MG_ASSERT(false, "actually splitting 2");
-
-  shard_version_ = new_shard_version;
-  const auto old_max_key = max_primary_key_;
-  max_primary_key_ = split_key;
-  return shard_splitter_.SplitShard(split_key, old_max_key, shard_version_);
-}
-
 bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const {
   return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ &&
          (!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_);

src/storage/v3/shard.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,15 +14,12 @@
 #include <cstdint>
 #include <filesystem>
 #include <map>
-#include <memory>
 #include <numeric>
 #include <optional>
 #include <shared_mutex>
 #include <variant>
 #include <vector>
 
-#include <boost/uuid/uuid.hpp>
-
 #include "coordinator/hybrid_logical_clock.hpp"
 #include "io/network/endpoint.hpp"
 #include "io/time.hpp"
@@ -40,13 +37,11 @@
 #include "storage/v3/result.hpp"
 #include "storage/v3/schema_validator.hpp"
 #include "storage/v3/schemas.hpp"
-#include "storage/v3/splitter.hpp"
 #include "storage/v3/transaction.hpp"
 #include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "storage/v3/vertex_id.hpp"
 #include "storage/v3/view.hpp"
-#include "utils/concrete_msg_sender.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/file_locker.hpp"
 #include "utils/on_scope_exit.hpp"
@@ -58,8 +53,6 @@
 
 namespace memgraph::storage::v3 {
 
-using coordinator::Hlc;
-
 // The storage is based on this paper:
 // https://db.in.tum.de/~muehlbau/papers/mvcc.pdf
 // The paper implements a fully serializable storage, in our implementation we
@@ -189,31 +182,13 @@ struct StorageInfo {
   uint64_t memory_usage;
 };
 
-struct ShardSuggestedSplitInfo {
-  boost::uuids::uuid shard_to_split_uuid;
-  LabelId label_id;
-  PrimaryKey splitting_shard_low_key;
-  PrimaryKey split_key;
-  Hlc shard_version;
-};
-
 class Shard final {
  public:
   /// @throw std::system_error
   /// @throw std::bad_alloc
-  Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
-        std::vector<SchemaProperty> schema, Hlc shard_version, Config config = Config(),
-        std::unordered_map<uint64_t, std::string> id_to_name = {});
-
-  Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
-        std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
-        std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
-        const std::unordered_map<uint64_t, std::string> &id_to_name, Hlc shard_version);
-
-  Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
-        std::vector<SchemaProperty> schema, VertexContainer &&vertices,
-        std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
-        const std::unordered_map<uint64_t, std::string> &id_to_name, Hlc shard_version);
+  explicit Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
+                 std::vector<SchemaProperty> schema, Config config = Config(),
+                 std::unordered_map<uint64_t, std::string> id_to_name = {});
 
   Shard(const Shard &) = delete;
   Shard(Shard &&) noexcept = delete;
@@ -221,8 +196,6 @@ class Shard final {
   Shard operator=(Shard &&) noexcept = delete;
   ~Shard();
 
-  static std::unique_ptr<Shard> FromSplitData(SplitData &&split_data);
-
   class Accessor final {
    private:
     friend class Shard;
@@ -356,8 +329,6 @@ class Shard final {
 
   LabelId PrimaryLabel() const;
 
-  PrimaryKey LowKey() const { return min_primary_key_; }
-
   [[nodiscard]] bool IsVertexBelongToShard(const VertexId &vertex_id) const;
 
   /// @throw std::bad_alloc
@@ -389,10 +360,6 @@ class Shard final {
 
   void StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name);
 
-  std::optional<ShardSuggestedSplitInfo> ShouldSplit() const noexcept;
-
-  std::optional<SplitData> PerformSplit(const PrimaryKey &split_key, Hlc old_shard_version, Hlc new_shard_version);
-
  private:
   Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level);
 
@@ -410,7 +377,6 @@ class Shard final {
   // list is used only when properties are enabled for edges. Because of that we
   // keep a separate count of edges that is always updated.
   uint64_t edge_count_{0};
-  Hlc shard_version_{};
 
   SchemaValidator schema_validator_;
   VertexValidator vertex_validator_;
@@ -430,10 +396,41 @@ class Shard final {
   // storage.
   std::list<Gid> deleted_edges_;
 
+  // UUID used to distinguish snapshots and to link snapshots to WALs
+  std::string uuid_;
+  // Sequence number used to keep track of the chain of WALs.
+  uint64_t wal_seq_num_{0};
+
+  // UUID to distinguish different main instance runs for replication process
+  // on SAME storage.
+  // Multiple instances can have same storage UUID and be MAIN at the same time.
+  // We cannot compare commit timestamps of those instances if one of them
+  // becomes the replica of the other so we use epoch_id_ as additional
+  // discriminating property.
+  // Example of this:
+  // We have 2 instances of the same storage, S1 and S2.
+  // S1 and S2 are MAIN and accept their own commits and write them to the WAL.
+  // At the moment when S1 commited a transaction with timestamp 20, and S2
+  // a different transaction with timestamp 15, we change S2's role to REPLICA
+  // and register it on S1.
+  // Without using the epoch_id, we don't know that S1 and S2 have completely
+  // different transactions, we think that the S2 is behind only by 5 commits.
+  std::string epoch_id_;
+  // History of the previous epoch ids.
+  // Each value consists of the epoch id along the last commit belonging to that
+  // epoch.
+  std::deque<std::pair<std::string, uint64_t>> epoch_history_;
+
+  uint64_t wal_unsynced_transactions_{0};
+
+  utils::FileRetainer file_retainer_;
+
+  // Global locker that is used for clients file locking
+  utils::FileRetainer::FileLocker global_locker_;
+
   // Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but
   // haven't been cleaned up yet
   std::map<uint64_t, std::unique_ptr<Transaction>> start_logical_id_to_transaction_{};
-  Splitter shard_splitter_;
   bool has_any_transaction_aborted_since_last_gc{false};
 };
 

src/storage/v3/shard_manager.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,9 +12,8 @@
 #pragma once
 
 #include <queue>
+#include <set>
 #include <unordered_map>
-#include <variant>
-#include <vector>
 
 #include <boost/functional/hash.hpp>
 #include <boost/uuid/uuid.hpp>
@@ -32,8 +31,6 @@
 #include "storage/v3/shard.hpp"
 #include "storage/v3/shard_rsm.hpp"
 #include "storage/v3/shard_worker.hpp"
-#include "storage/v3/value_conversions.hpp"
-#include "utils/variant_helpers.hpp"
 
 namespace memgraph::storage::v3 {
 
@@ -45,6 +42,7 @@ using coordinator::HeartbeatRequest;
 using coordinator::HeartbeatResponse;
 using io::Address;
 using io::Duration;
+using io::Message;
 using io::RequestId;
 using io::ResponseFuture;
 using io::Time;
@@ -176,39 +174,7 @@ class ShardManager {
   /// Returns the Address for our underlying Io implementation
   Address GetAddress() { return io_.GetAddress(); }
 
-  void InitializeSplitShard(msgs::InitializeSplitShard &&init_split_shard) {
-    spdlog::warn("ShardManager received InitializeSplitShard message");
-    MG_ASSERT(false, "ShardManager::InitializeSplitShard called :)");
-    for (const auto &[from_uuid, new_uuid] : init_split_shard.uuid_mapping) {
-      bool has_source = rsm_worker_mapping_.contains(from_uuid);
-      if (has_source) {
-        coordinator::ShardId new_shard_id =
-            std::make_pair(init_split_shard.shard->PrimaryLabel(), init_split_shard.shard->LowKey());
-
-        msgs::InitializeSplitShardByUUID msg{.shard = std::move(init_split_shard.shard), .shard_uuid = new_uuid};
-        SendToWorkerByUuid(new_uuid, std::move(msg));
-        spdlog::warn("ShardManager initialized split shard with uuid: {}", new_uuid);
-
-        initialized_but_not_confirmed_rsm_.emplace(new_uuid, new_shard_id);
-        break;
-      }
-    }
-  }
-
-  void Receive(ShardManagerMessages &&smm, RequestId request_id, Address from) {
-    std::visit(utils::Overloaded{[this](msgs::SuggestedSplitInfo &&split_info) {
-                                   spdlog::info(
-                                       "ShardManager adding new suggested split info to the pending_splits_ structure");
-                                   pending_splits_.emplace(std::move(split_info));
-                                 },
-                                 [this](msgs::InitializeSplitShard &&init_split_shard) {
-                                   // TODO(jbajic) remove pending split for this completed split
-                                   // TODO(jbajic) Add new shard to initialized but not confirmed rsm
-                                   spdlog::info("ShardManager received a new split shard that it will now initialize");
-                                   InitializeSplitShard(std::move(init_split_shard));
-                                 }},
-               std::move(smm));
-  }
+  void Receive(ShardManagerMessages &&smm, RequestId request_id, Address from) {}
 
   void Route(ShardMessages &&sm, RequestId request_id, Address to, Address from) {
     Address address = io_.GetAddress();
@@ -235,12 +201,14 @@ class ShardManager {
   std::vector<shard_worker::Queue> workers_;
   std::vector<std::jthread> worker_handles_;
   std::vector<size_t> worker_rsm_counts_;
-  std::set<msgs::SuggestedSplitInfo> pending_splits_;
   std::unordered_map<uuid, size_t, boost::hash<boost::uuids::uuid>> rsm_worker_mapping_;
   Time next_reconciliation_ = Time::min();
   Address coordinator_leader_;
   std::optional<ResponseFuture<WriteResponse<CoordinatorWriteResponses>>> heartbeat_res_;
-  std::map<boost::uuids::uuid, coordinator::ShardId> initialized_but_not_confirmed_rsm_;
+
+  // TODO(tyler) over time remove items from initialized_but_not_confirmed_rsm_
+  // after the Coordinator is clearly aware of them
+  std::set<boost::uuids::uuid> initialized_but_not_confirmed_rsm_;
 
   void Reconciliation() {
     if (heartbeat_res_.has_value()) {
@@ -274,7 +242,6 @@ class ShardManager {
     HeartbeatRequest req{
         .from_storage_manager = GetAddress(),
         .initialized_rsms = initialized_but_not_confirmed_rsm_,
-        .suggested_splits = std::move(pending_splits_),
     };
 
     CoordinatorWriteRequests cwr = req;
@@ -289,20 +256,14 @@ class ShardManager {
   }
 
   void EnsureShardsInitialized(HeartbeatResponse hr) {
-    for (const auto &acknowledged_rsm : hr.acknowledged_initialized_rsms) {
-      MG_ASSERT(false, "coordinator properly acking initialized rsms :)");
-      initialized_but_not_confirmed_rsm_.erase(acknowledged_rsm);
-    }
-
     for (const auto &to_init : hr.shards_to_initialize) {
-      coordinator::ShardId new_shard_id = std::make_pair(to_init.label_id, to_init.min_key);
-      spdlog::info("ShardManager has been told to initialize shard {}", to_init.uuid);
-      initialized_but_not_confirmed_rsm_.emplace(to_init.uuid, new_shard_id);
+      initialized_but_not_confirmed_rsm_.emplace(to_init.uuid);
 
       if (rsm_worker_mapping_.contains(to_init.uuid)) {
-        spdlog::info(
-            "ShardManager forwarding shard intialization request to worker despite a mapping already existing. This "
-            "can happen due to benign race conditions.");
+        // it's not a bug for the coordinator to send us UUIDs that we have
+        // already created, because there may have been lag that caused
+        // the coordinator not to hear back from us.
+        return;
       }
 
       size_t worker_index = UuidToWorkerIndex(to_init.uuid);
@@ -311,54 +272,6 @@ class ShardManager {
 
       rsm_worker_mapping_.emplace(to_init.uuid, worker_index);
     }
-
-    for (const auto &to_split : hr.shards_to_split) {
-      for (const auto &[source, destination] : to_split.uuid_mapping) {
-        if (rsm_worker_mapping_.contains(destination)) {
-          // it's not a bug for the coordinator to send us UUIDs that we have
-          // already created, because there may have been lag that caused
-          // the coordinator not to hear back from us.
-          // TODO(tyler) make this idempotent before it hits raft
-          // break;
-        }
-
-        if (rsm_worker_mapping_.contains(source)) {
-          MG_ASSERT(false, "sending split request from SM to shard rsm :)");
-          // Create the proper layered request providing a Raft write
-          // request to the local shard rsm, under the guess that it is
-          // the current leader. Most of the time this will be an incorrect
-          // guess, but it will eventually succeed when the right ShardManager
-          // happens to send the message to the leader shard that is local.
-          // This is done to avoid blocking on an RsmClient or maintaining
-          // complex async request logic. It's fine to fire-and-forget because
-          // it is rare and will eventually succeed.
-          msgs::WriteRequests split_request_1 =
-              msgs::SplitRequest{.split_key = conversions::ConvertValueVector(to_split.split_key),
-                                 .old_shard_version = to_split.old_shard_version,
-                                 .new_shard_version = to_split.new_shard_version,
-                                 .uuid_mapping = to_split.uuid_mapping};
-
-          WriteRequest<msgs::WriteRequests> split_request_2;
-          split_request_2.operation = split_request_1;
-          ShardMessages split_request_3 = split_request_2;
-
-          const Address our_address = io_.GetAddress();
-          Address shard_address = our_address;
-          shard_address.unique_id = source;
-
-          shard_worker::RouteMessage shard_worker_message = {
-              .message = split_request_3,
-              .request_id = 0,
-              .to = shard_address,
-              .from = our_address,
-          };
-
-          SendToWorkerByUuid(source, shard_worker_message);
-        } else {
-          MG_ASSERT(false, "bad split source: {}", source);
-        }
-      }
-    }
   }
 };
 

src/storage/v3/shard_rsm.cpp

@@ -61,8 +61,6 @@ using conversions::FromMap;
 using conversions::FromPropertyValueToValue;
 using conversions::ToMsgsVertexId;
 using conversions::ToPropertyValue;
-using io::messages::ShardManagerMessages;
-using msgs::SplitResponse;
 
 auto CreateErrorResponse(const ShardError &shard_error, const auto transaction_id, const std::string_view action) {
   msgs::ShardError message_shard_error{shard_error.code, shard_error.message};
@@ -317,22 +315,6 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::UpdateEdgesRequest &&req) {
   return msgs::UpdateEdgesResponse{std::move(shard_error)};
 }
 
-msgs::WriteResponses ShardRsm::ApplyWrite(msgs::SplitRequest &&req) {
-  MG_ASSERT(false, "ShardRsm::ApplyWrite 0");
-
-  auto converted_primary_key = conversions::ConvertPropertyVector(req.split_key);
-  auto new_shard_split_data = shard_->PerformSplit(converted_primary_key, req.old_shard_version, req.new_shard_version);
-
-  if (new_shard_split_data) {
-    msgs::InitializeSplitShard msg{.shard = Shard::FromSplitData(std::move(*new_shard_split_data)),
-                                   .uuid_mapping = req.uuid_mapping};
-    ShardManagerMessages msg_to_send{std::move(msg)};
-    shard_manager_sender_.Send(std::move(msg_to_send));
-  }
-
-  return SplitResponse{};
-}
-
 msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) {
   auto acc = shard_->Access(req.transaction_id);
   std::optional<msgs::ShardError> shard_error;

src/storage/v3/shard_rsm.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,21 +12,17 @@
 #pragma once
 
 #include <memory>
-#include <optional>
 #include <variant>
 
 #include <openssl/ec.h>
-#include "io/messages.hpp"
 #include "query/v2/requests.hpp"
 #include "storage/v3/shard.hpp"
-#include "storage/v3/value_conversions.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 
 namespace memgraph::storage::v3 {
 
 class ShardRsm {
-  std::shared_ptr<Shard> shard_;
-  utils::Sender<io::messages::ShardManagerMessages> shard_manager_sender_;
+  std::unique_ptr<Shard> shard_;
 
   msgs::ReadResponses HandleRead(msgs::ExpandOneRequest &&req);
   msgs::ReadResponses HandleRead(msgs::GetPropertiesRequest &&req);
@@ -40,26 +36,10 @@ class ShardRsm {
   msgs::WriteResponses ApplyWrite(msgs::DeleteEdgesRequest &&req);
   msgs::WriteResponses ApplyWrite(msgs::UpdateEdgesRequest &&req);
 
-  msgs::WriteResponses ApplyWrite(msgs::SplitRequest &&req);
-
   msgs::WriteResponses ApplyWrite(msgs::CommitRequest &&req);
 
  public:
-  ShardRsm(std::shared_ptr<Shard> &&shard, utils::Sender<io::messages::ShardManagerMessages> shard_manager_sender)
-      : shard_(std::move(shard)), shard_manager_sender_{std::move(shard_manager_sender)} {};
-
-  std::optional<msgs::SuggestedSplitInfo> ShouldSplit() const noexcept {
-    auto split_info = shard_->ShouldSplit();
-    if (split_info) {
-      return msgs::SuggestedSplitInfo{
-          .shard_to_split_uuid = split_info->shard_to_split_uuid,
-          .label_id = split_info->label_id,
-          .splitting_shard_low_key = conversions::ConvertValueVector(split_info->splitting_shard_low_key),
-          .split_key = conversions::ConvertValueVector(split_info->split_key),
-          .shard_version = split_info->shard_version};
-    }
-    return std::nullopt;
-  }
+  explicit ShardRsm(std::unique_ptr<Shard> &&shard) : shard_(std::move(shard)){};
 
   // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
   msgs::ReadResponses Read(msgs::ReadRequests requests) {

src/storage/v3/shard_worker.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,11 +17,9 @@
 #include <queue>
 #include <variant>
 
-#include <boost/core/demangle.hpp>
 #include <boost/uuid/uuid.hpp>
 
 #include "coordinator/coordinator.hpp"
-#include "coordinator/hybrid_logical_clock.hpp"
 #include "coordinator/shard_map.hpp"
 #include "io/address.hpp"
 #include "io/future.hpp"
@@ -48,7 +46,6 @@ using io::RequestId;
 using io::Time;
 using io::messages::ShardMessages;
 using io::rsm::Raft;
-using msgs::InitializeSplitShardByUUID;
 using msgs::ReadRequests;
 using msgs::ReadResponses;
 using msgs::WriteRequests;
@@ -69,7 +66,7 @@ struct RouteMessage {
   Address from;
 };
 
-using Message = std::variant<ShutDown, Cron, ShardToInitialize, RouteMessage, InitializeSplitShardByUUID>;
+using Message = std::variant<ShutDown, Cron, ShardToInitialize, RouteMessage>;
 
 struct QueueInner {
   std::mutex mu{};
@@ -158,55 +155,9 @@ class ShardWorker {
   }
 
   bool Process(RouteMessage &&route_message) {
-    if (rsm_map_.contains(route_message.to.unique_id)) {
-      spdlog::info("ShardWorker routing message to rsm {}", route_message.to.unique_id);
-      auto &rsm = rsm_map_.at(route_message.to.unique_id);
+    auto &rsm = rsm_map_.at(route_message.to.unique_id);
 
-      rsm.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
-    } else {
-      auto type_info = std::visit([&](const auto &msg) { return io::TypeInfoFor(msg); }, route_message.message);
-
-      std::string demangled_name = boost::core::demangle(type_info.get().name());
-      spdlog::warn(
-          "ShardWorker received {} message for rsm {} which does not exist on our system (possibly due to a split not "
-          "having been applied locally yet)",
-          demangled_name, route_message.to.unique_id);
-    }
-
-    return true;
-  }
-
-  bool Process(InitializeSplitShardByUUID &&initialize_split_shard) {
-    MG_ASSERT(false, "trying to initialize new split shard :)");
-    if (rsm_map_.contains(initialize_split_shard.shard_uuid)) {
-      // it's not a bug for the coordinator to send us UUIDs that we have
-      // already created, because there may have been lag that caused
-      // the coordinator not to hear back from us.
-      return true;
-    }
-
-    auto rsm_io = io_.ForkLocal(initialize_split_shard.shard_uuid);
-
-    // TODO(tyler) get peers from Coordinator in HeartbeatResponse
-    std::vector<Address> rsm_peers = {};
-
-    Address local_shard_manager_address = io_.GetAddress().ForkLocalShardManager();
-    utils::Sender<io::messages::ShardManagerMessages> local_shard_manager_sender =
-        io_.template GetSender<io::messages::ShardManagerMessages>(local_shard_manager_address);
-
-    // TODO(tyler) pass this local_shard_manager_sender to the Shard so that it can communicate back to the local
-    // manager from split code
-    ShardRsm rsm_state{std::move(initialize_split_shard.shard), std::move(local_shard_manager_sender)};
-
-    ShardRaft<IoImpl> rsm{std::move(rsm_io), rsm_peers, std::move(rsm_state)};
-
-    spdlog::info("SM created a new shard with UUID {}", initialize_split_shard.shard_uuid);
-
-    // perform an initial Cron call for the new RSM
-    Time next_cron = rsm.Cron();
-    cron_schedule_.push(std::make_pair(next_cron, initialize_split_shard.shard_uuid));
-
-    rsm_map_.emplace(initialize_split_shard.shard_uuid, std::move(rsm));
+    rsm.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
 
     return true;
   }
@@ -222,13 +173,6 @@ class ShardWorker {
         auto &rsm = rsm_map_.at(uuid);
         Time next_for_uuid = rsm.Cron();
 
-        // Check if shard should split
-        if (auto split_info = rsm.ShouldSplit(); split_info) {
-          split_info->shard_to_split_uuid = uuid;
-          const auto shard_manager_addr = io_.GetAddress().ForkLocalShardManager();
-          io_.Send(shard_manager_addr, 0, *split_info);
-        }
-
         cron_schedule_.pop();
         cron_schedule_.push(std::make_pair(next_for_uuid, uuid));
       } else {
@@ -252,18 +196,10 @@ class ShardWorker {
     // TODO(tyler) get peers from Coordinator in HeartbeatResponse
     std::vector<Address> rsm_peers = {};
 
-    Address local_shard_manager_address = io_.GetAddress().ForkLocalShardManager();
-    utils::Sender<io::messages::ShardManagerMessages> local_shard_manager_sender =
-        io_.template GetSender<io::messages::ShardManagerMessages>(local_shard_manager_address);
+    std::unique_ptr<Shard> shard = std::make_unique<Shard>(to_init.label_id, to_init.min_key, to_init.max_key,
+                                                           to_init.schema, to_init.config, to_init.id_to_names);
 
-    // TODO(tyler) pass this local_shard_manager_sender to the Shard so that it can communicate back to the local
-    // manager from split code
-
-    std::unique_ptr<Shard> shard =
-        std::make_unique<Shard>(to_init.label_id, to_init.min_key, to_init.max_key, to_init.schema,
-                                to_init.new_shard_version, to_init.config, to_init.id_to_names);
-
-    ShardRsm rsm_state{std::move(shard), std::move(local_shard_manager_sender)};
+    ShardRsm rsm_state{std::move(shard)};
 
     ShardRaft<IoImpl> rsm{std::move(rsm_io), rsm_peers, std::move(rsm_state)};
 

src/storage/v3/splitter.cpp

@@ -1,351 +0,0 @@
-// Copyright 2023 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "storage/v3/splitter.hpp"
-
-#include <algorithm>
-#include <cstdint>
-#include <map>
-#include <memory>
-#include <optional>
-#include <set>
-
-#include "storage/v3/config.hpp"
-#include "storage/v3/delta.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/indices.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/name_id_mapper.hpp"
-#include "storage/v3/schemas.hpp"
-#include "storage/v3/shard.hpp"
-#include "storage/v3/transaction.hpp"
-#include "storage/v3/vertex.hpp"
-#include "utils/logging.hpp"
-
-namespace memgraph::storage::v3 {
-
-Splitter::Splitter(const LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
-                   std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
-                   const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper)
-    : primary_label_(primary_label),
-      vertices_(vertices),
-      edges_(edges),
-      start_logical_id_to_transaction_(start_logical_id_to_transaction),
-      indices_(indices),
-      config_(config),
-      schema_(schema),
-      name_id_mapper_(name_id_mapper) {}
-
-SplitData Splitter::SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
-                               const Hlc shard_version) {
-  SplitData data{.primary_label = primary_label_,
-                 .min_primary_key = split_key,
-                 .max_primary_key = max_primary_key,
-                 .schema = schema_,
-                 .config = config_,
-                 .id_to_name = name_id_mapper_.GetIdToNameMap(),
-                 .shard_version = shard_version};
-
-  std::set<uint64_t> collected_transactions_;
-  data.vertices = CollectVertices(data, collected_transactions_, split_key);
-  data.edges = CollectEdges(collected_transactions_, data.vertices, split_key);
-  data.transactions = CollectTransactions(collected_transactions_, data.vertices, *data.edges, split_key);
-
-  return data;
-}
-
-void Splitter::ScanDeltas(std::set<uint64_t> &collected_transactions_, Delta *delta) {
-  while (delta != nullptr) {
-    collected_transactions_.insert(delta->commit_info->start_or_commit_timestamp.logical_id);
-    delta = delta->next;
-  }
-}
-
-VertexContainer Splitter::CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_,
-                                          const PrimaryKey &split_key) {
-  data.label_indices = indices_.label_index.SplitIndexEntries(split_key);
-  data.label_property_indices = indices_.label_property_index.SplitIndexEntries(split_key);
-
-  VertexContainer splitted_data;
-  auto split_key_it = vertices_.find(split_key);
-  while (split_key_it != vertices_.end()) {
-    // Go through deltas and pick up transactions start_id/commit_id
-    ScanDeltas(collected_transactions_, split_key_it->second.delta);
-
-    auto next_it = std::next(split_key_it);
-
-    const auto &[splitted_vertex_it, inserted, node] = splitted_data.insert(vertices_.extract(split_key_it->first));
-    MG_ASSERT(inserted, "Failed to extract vertex!");
-
-    split_key_it = next_it;
-  }
-  return splitted_data;
-}
-
-std::optional<EdgeContainer> Splitter::CollectEdges(std::set<uint64_t> &collected_transactions_,
-                                                    const VertexContainer &split_vertices,
-                                                    const PrimaryKey &split_key) {
-  if (!config_.items.properties_on_edges) {
-    return std::nullopt;
-  }
-  EdgeContainer splitted_edges;
-  const auto split_vertex_edges = [&](const auto &edges_ref) {
-    // This is safe since if properties_on_edges is true, the this must be a ptr
-    for (const auto &edge_ref : edges_ref) {
-      auto *edge = std::get<2>(edge_ref).ptr;
-      const auto &other_vtx = std::get<1>(edge_ref);
-      ScanDeltas(collected_transactions_, edge->delta);
-      // Check if src and dest edge are both on splitted shard so we know if we
-      // should remove orphan edge, or make a clone
-      if (other_vtx.primary_key >= split_key) {
-        // Remove edge from shard
-        splitted_edges.insert(edges_.extract(edge->gid));
-      } else {
-        splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}});
-      }
-    }
-  };
-
-  for (const auto &vertex : split_vertices) {
-    split_vertex_edges(vertex.second.in_edges);
-    split_vertex_edges(vertex.second.out_edges);
-  }
-  return splitted_edges;
-}
-
-std::map<uint64_t, std::unique_ptr<Transaction>> Splitter::CollectTransactions(
-    const std::set<uint64_t> &collected_transactions_, VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
-    const PrimaryKey &split_key) {
-  std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
-
-  for (const auto &[commit_start, transaction] : start_logical_id_to_transaction_) {
-    // We need all transaction whose deltas need to be resolved for any of the
-    // entities
-    if (collected_transactions_.contains(transaction->commit_info->start_or_commit_timestamp.logical_id)) {
-      transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()});
-    }
-  }
-
-  // It is necessary to clone all the transactions first so we have new addresses
-  // for deltas, before doing alignment of deltas and prev_ptr
-  AdjustClonedTransactions(transactions, cloned_vertices, cloned_edges, split_key);
-  return transactions;
-}
-
-void PruneDeltas(Transaction &cloned_transaction, std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                 const PrimaryKey &split_key) {
-  // Remove delta chains that don't point to objects on splitted shard
-  auto cloned_delta_it = cloned_transaction.deltas.begin();
-
-  while (cloned_delta_it != cloned_transaction.deltas.end()) {
-    const auto prev = cloned_delta_it->prev.Get();
-    switch (prev.type) {
-      case PreviousPtr::Type::DELTA:
-      case PreviousPtr::Type::NULLPTR:
-        ++cloned_delta_it;
-        break;
-      case PreviousPtr::Type::VERTEX: {
-        if (prev.vertex->first < split_key) {
-          // We can remove this delta chain
-          auto *current_next_delta = cloned_delta_it->next;
-          cloned_delta_it = cloned_transaction.deltas.erase(cloned_delta_it);
-
-          while (current_next_delta != nullptr) {
-            auto *next_delta = current_next_delta->next;
-            // Find next delta transaction delta list
-            auto current_transaction_it = std::ranges::find_if(
-                cloned_transactions,
-                [&start_or_commit_timestamp =
-                     current_next_delta->commit_info->start_or_commit_timestamp](const auto &transaction) {
-                  return transaction.second->start_timestamp == start_or_commit_timestamp ||
-                         transaction.second->commit_info->start_or_commit_timestamp == start_or_commit_timestamp;
-                });
-            MG_ASSERT(current_transaction_it != cloned_transactions.end(), "Error when pruning deltas!");
-            // Remove it
-            current_transaction_it->second->deltas.remove_if(
-                [&current_next_delta = *current_next_delta](const auto &delta) { return delta == current_next_delta; });
-
-            current_next_delta = next_delta;
-          }
-        } else {
-          ++cloned_delta_it;
-        }
-        break;
-      }
-      case PreviousPtr::Type::EDGE:
-        ++cloned_delta_it;
-        break;
-    }
-  }
-}
-
-void Splitter::AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                                        VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
-                                        const PrimaryKey &split_key) {
-  for (auto &[commit_start, cloned_transaction] : cloned_transactions) {
-    AdjustClonedTransaction(*cloned_transaction, *start_logical_id_to_transaction_[commit_start], cloned_transactions,
-                            cloned_vertices, cloned_edges, split_key);
-  }
-  // Prune deltas whose delta chain points to vertex/edge that should not belong on that shard
-  // Prune must be after ajdust, since next, and prev are not set and we cannot follow the chain
-  for (auto &[commit_start, cloned_transaction] : cloned_transactions) {
-    PruneDeltas(*cloned_transaction, cloned_transactions, split_key);
-  }
-}
-
-inline bool IsDeltaHeadOfChain(const PreviousPtr::Type &delta_type) {
-  return delta_type == PreviousPtr::Type::VERTEX || delta_type == PreviousPtr::Type::EDGE;
-}
-
-bool DoesPrevPtrPointsToSplittedData(const PreviousPtr::Pointer &prev_ptr, const PrimaryKey &split_key) {
-  return prev_ptr.type == PreviousPtr::Type::VERTEX && prev_ptr.vertex->first < split_key;
-}
-
-void Splitter::AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
-                                       std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                                       VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
-                                       const PrimaryKey & /*split_key*/) {
-  auto delta_it = transaction.deltas.begin();
-  auto cloned_delta_it = cloned_transaction.deltas.begin();
-
-  while (delta_it != transaction.deltas.end()) {
-    // We can safely ignore deltas which are not head of delta chain
-    // Dont' adjust delta chain that points to irrelevant data vertices/edges
-    if (const auto delta_prev = delta_it->prev.Get(); !IsDeltaHeadOfChain(delta_prev.type)) {
-      ++delta_it;
-      ++cloned_delta_it;
-      continue;
-    }
-
-    const auto *delta = &*delta_it;
-    auto *cloned_delta = &*cloned_delta_it;
-    Delta *cloned_delta_prev_ptr = cloned_delta;
-    while (delta->next != nullptr) {
-      AdjustEdgeRef(*cloned_delta, cloned_edges);
-
-      // Align next ptr
-      AdjustDeltaNext(*delta, *cloned_delta, cloned_transactions);
-
-      // Align prev ptr
-      if (cloned_delta_prev_ptr != nullptr) {
-        AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_vertices, cloned_edges);
-      }
-
-      // TODO Next delta might not belong to the cloned transaction and thats
-      // why we skip this delta of the delta chain
-      if (cloned_delta->next != nullptr) {
-        cloned_delta = cloned_delta->next;
-        cloned_delta_prev_ptr = cloned_delta;
-      } else {
-        cloned_delta_prev_ptr = nullptr;
-      }
-      delta = delta->next;
-    }
-    // Align prev ptr
-    if (cloned_delta_prev_ptr != nullptr) {
-      AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_vertices, cloned_edges);
-    }
-
-    ++delta_it;
-    ++cloned_delta_it;
-  }
-  MG_ASSERT(delta_it == transaction.deltas.end() && cloned_delta_it == cloned_transaction.deltas.end(),
-            "Both iterators must be exhausted!");
-}
-
-void Splitter::AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const {
-  switch (cloned_delta.action) {
-    case Delta::Action::ADD_IN_EDGE:
-    case Delta::Action::ADD_OUT_EDGE:
-    case Delta::Action::REMOVE_IN_EDGE:
-    case Delta::Action::REMOVE_OUT_EDGE: {
-      // Find edge
-      if (config_.items.properties_on_edges) {
-        // Only case when not finding is when the edge is not on splitted shard
-        // TODO Do this after prune an move condition into assert
-        if (const auto cloned_edge_it =
-                std::ranges::find_if(cloned_edges, [edge_ptr = cloned_delta.vertex_edge.edge.ptr](
-                                                       const auto &elem) { return elem.second.gid == edge_ptr->gid; });
-            cloned_edge_it != cloned_edges.end()) {
-          cloned_delta.vertex_edge.edge = EdgeRef{&cloned_edge_it->second};
-        }
-      }
-      break;
-    }
-    case Delta::Action::DELETE_OBJECT:
-    case Delta::Action::RECREATE_OBJECT:
-    case Delta::Action::SET_PROPERTY:
-    case Delta::Action::ADD_LABEL:
-    case Delta::Action::REMOVE_LABEL: {
-      // noop
-      break;
-    }
-  }
-}
-
-void Splitter::AdjustDeltaNext(const Delta &original, Delta &cloned,
-                               std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions) {
-  // Get cloned_delta->next transaction, using delta->next original transaction
-  auto cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&original](const auto &elem) {
-    return elem.second->start_timestamp == original.next->commit_info->start_or_commit_timestamp ||
-           elem.second->commit_info->start_or_commit_timestamp == original.next->commit_info->start_or_commit_timestamp;
-  });
-  // TODO(jbajic) What if next in delta chain does not belong to cloned transaction?
-  // MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
-  if (cloned_transaction_it == cloned_transactions.end()) return;
-  // Find cloned delta in delta list of cloned transaction
-  auto found_cloned_delta_it = std::ranges::find_if(
-      cloned_transaction_it->second->deltas, [&original](const auto &elem) { return elem.id == original.next->id; });
-  MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(), "Delta with given uuid must exist!");
-  cloned.next = &*found_cloned_delta_it;
-}
-
-void Splitter::AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
-                                  std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                                  VertexContainer & /*cloned_vertices*/, EdgeContainer &cloned_edges) {
-  auto ptr = original.prev.Get();
-  switch (ptr.type) {
-    case PreviousPtr::Type::NULLPTR: {
-      // noop
-      break;
-    }
-    case PreviousPtr::Type::DELTA: {
-      // Same as for deltas except don't align next but prev
-      auto cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&ptr](const auto &elem) {
-        return elem.second->start_timestamp == ptr.delta->commit_info->start_or_commit_timestamp ||
-               elem.second->commit_info->start_or_commit_timestamp == ptr.delta->commit_info->start_or_commit_timestamp;
-      });
-      MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
-      // Find cloned delta in delta list of cloned transaction
-      auto found_cloned_delta_it =
-          std::ranges::find_if(cloned_transaction_it->second->deltas,
-                               [delta = ptr.delta](const auto &elem) { return elem.id == delta->id; });
-      MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(),
-                "Delta with given id must exist!");
-
-      cloned.prev.Set(&*found_cloned_delta_it);
-      break;
-    }
-    case PreviousPtr::Type::VERTEX: {
-      // The vertex was extracted and it is safe to reuse address
-      cloned.prev.Set(ptr.vertex);
-      break;
-    }
-    case PreviousPtr::Type::EDGE: {
-      // We can never be here if we have properties on edge disabled
-      auto *cloned_edge = &*cloned_edges.find(ptr.edge->gid);
-      cloned.prev.Set(&cloned_edge->second);
-      break;
-    }
-  };
-}
-
-}  // namespace memgraph::storage::v3

src/storage/v3/splitter.hpp

@@ -1,108 +0,0 @@
-// Copyright 2023 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-#pragma once
-
-#include <cstdint>
-#include <map>
-#include <memory>
-#include <optional>
-#include <set>
-
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "storage/v3/config.hpp"
-#include "storage/v3/delta.hpp"
-#include "storage/v3/edge.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/indices.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/name_id_mapper.hpp"
-#include "storage/v3/schemas.hpp"
-#include "storage/v3/transaction.hpp"
-#include "storage/v3/vertex.hpp"
-#include "utils/concepts.hpp"
-
-namespace memgraph::storage::v3 {
-
-// If edge properties-on-edges is false then we don't need to send edges but
-// only vertices, since they will contain those edges
-struct SplitData {
-  LabelId primary_label;
-  PrimaryKey min_primary_key;
-  std::optional<PrimaryKey> max_primary_key;
-  std::vector<SchemaProperty> schema;
-  Config config;
-  std::unordered_map<uint64_t, std::string> id_to_name;
-  coordinator::Hlc shard_version;
-
-  VertexContainer vertices;
-  std::optional<EdgeContainer> edges;
-  std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
-  std::map<LabelId, LabelIndex::IndexContainer> label_indices;
-  std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndex::IndexContainer> label_property_indices;
-};
-
-class Splitter final {
- public:
-  Splitter(LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
-           std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
-           const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper_);
-
-  Splitter(const Splitter &) = delete;
-  Splitter(Splitter &&) noexcept = delete;
-  Splitter &operator=(const Splitter &) = delete;
-  Splitter operator=(Splitter &&) noexcept = delete;
-  ~Splitter() = default;
-
-  SplitData SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
-                       coordinator::Hlc shard_version);
-
- private:
-  VertexContainer CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_start_id,
-                                  const PrimaryKey &split_key);
-
-  std::optional<EdgeContainer> CollectEdges(std::set<uint64_t> &collected_transactions_start_id,
-                                            const VertexContainer &split_vertices, const PrimaryKey &split_key);
-
-  std::map<uint64_t, std::unique_ptr<Transaction>> CollectTransactions(
-      const std::set<uint64_t> &collected_transactions_start_id, VertexContainer &cloned_vertices,
-      EdgeContainer &cloned_edges, const PrimaryKey &split_key);
-
-  static void ScanDeltas(std::set<uint64_t> &collected_transactions_start_id, Delta *delta);
-
-  void AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
-                               std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                               VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
-                               const PrimaryKey &split_key);
-
-  void AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                                VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
-                                const PrimaryKey &split_key);
-
-  void AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const;
-
-  static void AdjustDeltaNext(const Delta &original, Delta &cloned,
-                              std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions);
-
-  static void AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
-                                 std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
-                                 VertexContainer &cloned_vertices, EdgeContainer &cloned_edges);
-
-  const LabelId primary_label_;
-  VertexContainer &vertices_;
-  EdgeContainer &edges_;
-  std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction_;
-  Indices &indices_;
-  const Config &config_;
-  const std::vector<SchemaProperty> schema_;
-  const NameIdMapper &name_id_mapper_;
-};
-
-}  // namespace memgraph::storage::v3

src/storage/v3/transaction.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -31,16 +31,6 @@ struct CommitInfo {
 };
 
 struct Transaction {
-  Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, std::list<Delta> deltas,
-              uint64_t command_id, bool must_abort, bool is_aborted, IsolationLevel isolation_level)
-      : start_timestamp{start_timestamp},
-        commit_info{std::make_unique<CommitInfo>(new_commit_info)},
-        command_id(command_id),
-        deltas(std::move(deltas)),
-        must_abort(must_abort),
-        is_aborted(is_aborted),
-        isolation_level(isolation_level){};
-
   Transaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level)
       : start_timestamp(start_timestamp),
         commit_info(std::make_unique<CommitInfo>(CommitInfo{false, {start_timestamp}})),
@@ -64,54 +54,6 @@ struct Transaction {
 
   ~Transaction() {}
 
-  std::list<Delta> CopyDeltas(CommitInfo *commit_info) const {
-    std::list<Delta> copied_deltas;
-    for (const auto &delta : deltas) {
-      switch (delta.action) {
-        case Delta::Action::DELETE_OBJECT:
-          copied_deltas.emplace_back(Delta::DeleteObjectTag{}, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::RECREATE_OBJECT:
-          copied_deltas.emplace_back(Delta::RecreateObjectTag{}, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::ADD_LABEL:
-          copied_deltas.emplace_back(Delta::AddLabelTag{}, delta.label, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::REMOVE_LABEL:
-          copied_deltas.emplace_back(Delta::RemoveLabelTag{}, delta.label, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::ADD_IN_EDGE:
-          copied_deltas.emplace_back(Delta::AddInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
-                                     delta.vertex_edge.edge, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::ADD_OUT_EDGE:
-          copied_deltas.emplace_back(Delta::AddOutEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
-                                     delta.vertex_edge.edge, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::REMOVE_IN_EDGE:
-          copied_deltas.emplace_back(Delta::RemoveInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
-                                     delta.vertex_edge.edge, commit_info, delta.id, command_id);
-          break;
-        case Delta::Action::REMOVE_OUT_EDGE:
-          copied_deltas.emplace_back(Delta::RemoveOutEdgeTag{}, delta.vertex_edge.edge_type,
-                                     delta.vertex_edge.vertex_id, delta.vertex_edge.edge, commit_info, delta.id,
-                                     command_id);
-          break;
-        case Delta::Action::SET_PROPERTY:
-          copied_deltas.emplace_back(Delta::SetPropertyTag{}, delta.property.key, delta.property.value, commit_info,
-                                     delta.id, command_id);
-          break;
-      }
-    }
-    return copied_deltas;
-  }
-
-  // This does not solve the whole problem of copying deltas
-  std::unique_ptr<Transaction> Clone() const {
-    return std::make_unique<Transaction>(start_timestamp, *commit_info, CopyDeltas(commit_info.get()), command_id,
-                                         must_abort, is_aborted, isolation_level);
-  }
-
   coordinator::Hlc start_timestamp;
   std::unique_ptr<CommitInfo> commit_info;
   uint64_t command_id;

src/utils/concrete_msg_sender.hpp

@@ -1,42 +0,0 @@
-// Copyright 2023 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <concepts>
-#include <functional>
-#include <type_traits>
-
-namespace memgraph::utils {
-
-// TODO(tyler) ensure that Message continues to represent
-// reasonable constraints around message types over time,
-// as we adapt things to use Thrift-generated message types.
-template <typename T>
-concept Message = std::movable<T> && std::copyable<T>;
-
-/// This is a concrete type that allows one message type to be
-/// sent to a single address. Initially intended to be used by the
-/// Shard to send messages to the local ShardManager.
-template <Message M>
-class Sender {
- public:
-  Sender() = default;
-
-  explicit Sender(std::function<void(M)> sender) : sender_(sender) {}
-
-  void Send(M &&message) { sender_(std::forward<M>(message)); }
-
- private:
-  std::function<void(M)> sender_;
-};
-
-}  // namespace memgraph::utils

tests/benchmark/CMakeLists.txt

@@ -79,6 +79,3 @@ target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage
 
 add_benchmark(data_structures_remove.cpp)
 target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3)
-
-add_benchmark(storage_v3_split.cpp)
-target_link_libraries(${test_prefix}storage_v3_split mg-storage-v3 mg-query-v2)

tests/benchmark/storage_v3_split.cpp

@@ -1,194 +0,0 @@
-// Copyright 2023 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <cstdint>
-#include <optional>
-#include <vector>
-
-#include <benchmark/benchmark.h>
-#include <gflags/gflags.h>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/property_value.hpp"
-#include "storage/v3/shard.hpp"
-#include "storage/v3/vertex.hpp"
-#include "storage/v3/vertex_id.hpp"
-
-namespace memgraph::benchmark {
-
-class ShardSplitBenchmark : public ::benchmark::Fixture {
- protected:
-  using PrimaryKey = storage::v3::PrimaryKey;
-  using PropertyId = storage::v3::PropertyId;
-  using PropertyValue = storage::v3::PropertyValue;
-  using LabelId = storage::v3::LabelId;
-  using EdgeTypeId = storage::v3::EdgeTypeId;
-  using Shard = storage::v3::Shard;
-  using VertexId = storage::v3::VertexId;
-  using Gid = storage::v3::Gid;
-
-  void SetUp(const ::benchmark::State &state) override {
-    storage.emplace(primary_label, min_pk, std::nullopt, schema_property_vector, last_hlc);
-    storage->StoreMapping(
-        {{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
-  }
-
-  void TearDown(const ::benchmark::State &) override { storage = std::nullopt; }
-
-  const PropertyId primary_property{PropertyId::FromUint(2)};
-  const PropertyId secondary_property{PropertyId::FromUint(5)};
-  std::vector<storage::v3::SchemaProperty> schema_property_vector = {
-      storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
-  const std::vector<PropertyValue> min_pk{PropertyValue{0}};
-  const LabelId primary_label{LabelId::FromUint(1)};
-  const LabelId secondary_label{LabelId::FromUint(4)};
-  const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
-  std::optional<Shard> storage;
-
-  coordinator::Hlc last_hlc{0, io::Time{}};
-
-  coordinator::Hlc GetNextHlc() {
-    ++last_hlc.logical_id;
-    last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
-    return last_hlc;
-  }
-};
-
-BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplit)(::benchmark::State &state) {
-  const auto number_of_vertices{state.range(0)};
-  std::random_device r;
-  std::default_random_engine e1(r());
-  std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
-
-  for (int64_t i{0}; i < number_of_vertices; ++i) {
-    auto acc = storage->Access(GetNextHlc());
-    MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
-                                          {{secondary_property, PropertyValue(i)}})
-                  .HasValue(),
-              "Failed creating with pk {}", i);
-    if (i > 1) {
-      const auto vtx1 = uniform_dist(e1) % i;
-      const auto vtx2 = uniform_dist(e1) % i;
-
-      MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
-                               VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
-                    .HasValue(),
-                "Failed on {} and {}", vtx1, vtx2);
-    }
-    acc.Commit(GetNextHlc());
-  }
-  for (auto _ : state) {
-    auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
-  }
-}
-
-BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithGc)(::benchmark::State &state) {
-  const auto number_of_vertices{state.range(0)};
-  std::random_device r;
-  std::default_random_engine e1(r());
-  std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
-
-  for (int64_t i{0}; i < number_of_vertices; ++i) {
-    auto acc = storage->Access(GetNextHlc());
-    MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
-                                          {{secondary_property, PropertyValue(i)}})
-                  .HasValue(),
-              "Failed creating with pk {}", i);
-    if (i > 1) {
-      const auto vtx1 = uniform_dist(e1) % i;
-      const auto vtx2 = uniform_dist(e1) % i;
-
-      MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
-                               VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
-                    .HasValue(),
-                "Failed on {} and {}", vtx1, vtx2);
-    }
-    acc.Commit(GetNextHlc());
-  }
-  storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
-  for (auto _ : state) {
-    auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
-  }
-}
-
-BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithFewTransactions)(::benchmark::State &state) {
-  const auto number_of_vertices = state.range(0);
-  const auto number_of_edges = state.range(1);
-  const auto number_of_transactions = state.range(2);
-  std::random_device r;
-  std::default_random_engine e1(r());
-  std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
-
-  const auto max_transactions_needed = std::max(number_of_vertices, number_of_edges);
-  for (int64_t vertex_counter{number_of_vertices}, edge_counter{number_of_edges}, i{0};
-       vertex_counter > 0 || edge_counter > 0; --vertex_counter, --edge_counter, ++i) {
-    auto acc = storage->Access(GetNextHlc());
-    if (vertex_counter > 0) {
-      MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
-                                            {{secondary_property, PropertyValue(i)}})
-                    .HasValue(),
-                "Failed creating with pk {}", i);
-    }
-    if (edge_counter > 0 && i > 1) {
-      const auto vtx1 = uniform_dist(e1) % std::min(i, number_of_vertices);
-      const auto vtx2 = uniform_dist(e1) % std::min(i, number_of_vertices);
-
-      MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
-                               VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
-                    .HasValue(),
-                "Failed on {} and {}", vtx1, vtx2);
-    }
-
-    acc.Commit(GetNextHlc());
-    if (i == max_transactions_needed - number_of_transactions) {
-      storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
-    }
-  }
-
-  for (auto _ : state) {
-    // Don't create shard since shard deallocation can take some time as well
-    auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
-  }
-}
-
-// Range:
-// Number of vertices
-// This run is pessimistic, number of vertices corresponds with number if transactions
-BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplit)
-    ->RangeMultiplier(10)
-    ->Range(100'000, 100'000)
-    ->Unit(::benchmark::kMillisecond);
-
-// Range:
-// Number of vertices
-// This run is optimistic, in this run there are no transactions
-BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithGc)
-    ->RangeMultiplier(10)
-    ->Range(100'000, 1'000'000)
-    ->Unit(::benchmark::kMillisecond);
-
-// Args:
-// Number of vertices
-// Number of edges
-// Number of transaction
-BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactions)
-    ->Args({100'000, 100'000, 1'000})
-    ->Args({100'000, 100'000, 10'000})
-    ->Args({1'000'000, 100'000, 1'000})
-    ->Args({1'000'000, 100'000, 10'000})
-    ->Args({100'000, 1'000'000, 1'000})
-    ->Args({1'000'000, 1'00'000, 10'000})
-    ->Unit(::benchmark::kMillisecond);
-
-}  // namespace memgraph::benchmark
-
-BENCHMARK_MAIN();

tests/mgbench/compare_results.py

@@ -14,7 +14,6 @@
 import argparse
 import json
 
-
 FIELDS = [
     {
         "name": "throughput",
@@ -85,39 +84,32 @@ def compare_results(results_from, results_to, fields):
                 if group == "__import__":
                     continue
                 for scenario, summary_to in scenarios.items():
-                    summary_from = recursive_get(
-                        results_from, dataset, variant, group, scenario,
-                        value={})
-                    if len(summary_from) > 0 and \
-                            summary_to["count"] != summary_from["count"] or \
-                            summary_to["num_workers"] != \
-                            summary_from["num_workers"]:
+                    summary_from = recursive_get(results_from, dataset, variant, group, scenario, value={})
+                    if (
+                        len(summary_from) > 0
+                        and summary_to["count"] != summary_from["count"]
+                        or summary_to["num_workers"] != summary_from["num_workers"]
+                    ):
                         raise Exception("Incompatible results!")
-                    testcode = "/".join([dataset, variant, group, scenario,
-                                         "{:02d}".format(
-                                             summary_to["num_workers"])])
+                    testcode = "/".join([dataset, variant, group, scenario, "{:02d}".format(summary_to["num_workers"])])
                     row = {}
                     performance_changed = False
                     for field in fields:
                         key = field["name"]
                         if key in summary_to:
-                            row[key] = compute_diff(
-                                summary_from.get(key, None),
-                                summary_to[key])
+                            row[key] = compute_diff(summary_from.get(key, None), summary_to[key])
                         elif key in summary_to["database"]:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "database", key,
-                                              value=None),
-                                summary_to["database"][key])
+                                recursive_get(summary_from, "database", key, value=None), summary_to["database"][key]
+                            )
                         else:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "metadata", key,
-                                              "average", value=None),
-                                summary_to["metadata"][key]["average"])
-                        if "diff" not in row[key] or \
-                                ("diff_treshold" in field and
-                                 abs(row[key]["diff"]) >=
-                                 field["diff_treshold"]):
+                                recursive_get(summary_from, "metadata", key, "average", value=None),
+                                summary_to["metadata"][key]["average"],
+                            )
+                        if "diff" not in row[key] or (
+                            "diff_treshold" in field and abs(row[key]["diff"]) >= field["diff_treshold"]
+                        ):
                             performance_changed = True
                     if performance_changed:
                         ret[testcode] = row
@@ -130,29 +122,36 @@ def generate_remarkup(fields, data):
         ret += "<table>\n"
         ret += "  <tr>\n"
         ret += "    <th>Testcode</th>\n"
-        ret += "\n".join(map(lambda x: "    <th>{}</th>".format(
-            x["name"].replace("_", " ").capitalize()), fields)) + "\n"
+        ret += (
+            "\n".join(
+                map(
+                    lambda x: "    <th>{}</th>".format(x["name"].replace("_", " ").capitalize()),
+                    fields,
+                )
+            )
+            + "\n"
+        )
         ret += "  </tr>\n"
         for testcode in sorted(data.keys()):
             ret += "  <tr>\n"
             ret += "    <td>{}</td>\n".format(testcode)
             for field in fields:
-                result = data[testcode][field["name"]]
-                value = result["value"] * field["scaling"]
-                if "diff" in result:
-                    diff = result["diff"]
-                    arrow = "arrow-up" if diff >= 0 else "arrow-down"
-                    if not (field["positive_diff_better"] ^ (diff >= 0)):
-                        color = "green"
+                result = data[testcode].get(field["name"])
+                if result != None:
+                    value = result["value"] * field["scaling"]
+                    if "diff" in result:
+                        diff = result["diff"]
+                        arrow = "arrow-up" if diff >= 0 else "arrow-down"
+                        if not (field["positive_diff_better"] ^ (diff >= 0)):
+                            color = "green"
+                        else:
+                            color = "red"
+                        sign = "{{icon {} color={}}}".format(arrow, color)
+                        ret += '    <td bgcolor="{}">{:.3f}{} ({:+.2%})</td>\n'.format(
+                            color, value, field["unit"], diff
+                        )
                     else:
-                        color = "red"
-                    sign = "{{icon {} color={}}}".format(arrow, color)
-                    ret += "    <td>{:.3f}{} //({:+.2%})// {}</td>\n".format(
-                        value, field["unit"], diff, sign)
-                else:
-                    ret += "    <td>{:.3f}{} //(new)// " \
-                           "{{icon plus color=blue}}</td>\n".format(
-                               value, field["unit"])
+                        ret += '<td bgcolor="blue">{:.3f}{} //(new)// </td>\n'.format(value, field["unit"])
             ret += "  </tr>\n"
         ret += "</table>\n"
     else:
@@ -161,11 +160,14 @@ def generate_remarkup(fields, data):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Compare results of multiple benchmark runs.")
-    parser.add_argument("--compare", action="append", nargs=2,
-                        metavar=("from", "to"),
-                        help="compare results between `from` and `to` files")
+    parser = argparse.ArgumentParser(description="Compare results of multiple benchmark runs.")
+    parser.add_argument(
+        "--compare",
+        action="append",
+        nargs=2,
+        metavar=("from", "to"),
+        help="compare results between `from` and `to` files",
+    )
     parser.add_argument("--output", default="", help="output file name")
     args = parser.parse_args()
 

tests/mgbench/dataset_creator_unwind.py

@@ -0,0 +1,139 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import argparse
+import random
+
+import helpers
+
+# Explaination of datasets:
+#   - empty_only_index: contains index; contains no data
+#   - small: contains index; contains data (small dataset)
+#
+# Datamodel is as follow:
+#
+#                               ┌──────────────┐
+#                               │ Permission   │
+#         ┌────────────────┐    │  Schema:uuid │   ┌────────────┐
+#         │:IS_FOR_IDENTITY├────┤  Index:name  ├───┤:IS_FOR_FILE│
+#         └┬───────────────┘    └──────────────┘   └────────────┤
+#          │                                                    │
+#   ┌──────▼──────────────┐                                  ┌──▼────────────────┐
+#   │  Identity           │                                  │ File              │
+#   │   Schema:uuid       │                                  │  Schema:uuid      │
+#   │   Index:email       │                                  │  Index:name       │
+#   └─────────────────────┘                                  │  Index:platformId │
+#                                                            └───────────────────┘
+#
+#   - File: attributes: ["uuid", "name", "platformId"]
+#   - Permission: attributes: ["uuid", "name"]
+#   - Identity: attributes: ["uuid", "email"]
+#
+# Indexes:
+#   - File: [File(uuid), File(platformId), File(name)]
+#   - Permission: [Permission(uuid), Permission(name)]
+#   - Identity: [Identity(uuid), Identity(email)]
+#
+# Edges:
+#   - (:Permission)-[:IS_FOR_FILE]->(:File)
+#   - (:Permission)-[:IS_FOR_IDENTITYR]->(:Identity)
+#
+# AccessControl specific: uuid is the schema
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--number_of_identities", type=int, default=10)
+    parser.add_argument("--number_of_files", type=int, default=10)
+    parser.add_argument("--percentage_of_permissions", type=float, default=1.0)
+    parser.add_argument("--filename", default="dataset.cypher")
+
+    args = parser.parse_args()
+
+    number_of_identities = args.number_of_identities
+    number_of_files = args.number_of_files
+    percentage_of_permissions = args.percentage_of_permissions
+    filename = args.filename
+
+    assert number_of_identities >= 0
+    assert number_of_files >= 0
+    assert percentage_of_permissions > 0.0 and percentage_of_permissions <= 1.0
+    assert filename != ""
+
+    with open(filename, "w") as f:
+        f.write("MATCH (n) DETACH DELETE n;\n")
+
+        # Create the indexes
+        f.write("CREATE INDEX ON :File;\n")
+        f.write("CREATE INDEX ON :Permission;\n")
+        f.write("CREATE INDEX ON :Identity;\n")
+        f.write("CREATE INDEX ON :File(platformId);\n")
+        f.write("CREATE INDEX ON :File(name);\n")
+        f.write("CREATE INDEX ON :Permission(name);\n")
+        f.write("CREATE INDEX ON :Identity(email);\n")
+
+        # Create extra index: in distributed, this will be the schema
+        f.write("CREATE INDEX ON :File(uuid);\n")
+        f.write("CREATE INDEX ON :Permission(uuid);\n")
+        f.write("CREATE INDEX ON :Identity(uuid);\n")
+
+        uuid = 1
+
+        # Create the nodes File
+        f.write("UNWIND [")
+        for index in range(0, number_of_files):
+            if index != 0:
+                f.write(",")
+            f.write(f' {{uuid: {uuid}, platformId: "platform_id", name: "name_file_{uuid}"}}')
+            uuid += 1
+        f.write("] AS props CREATE (:File {uuid: props.uuid, platformId: props.platformId, name: props.name});\n")
+
+        identities = []
+        f.write("UNWIND [")
+        # Create the nodes Identity
+        for index in range(0, number_of_identities):
+            if index != 0:
+                f.write(",")
+            f.write(f' {{uuid: {uuid}, name: "mail_{uuid}@something.com"}}')
+            uuid += 1
+        f.write("] AS props CREATE (:Identity {uuid: props.uuid, name: props.name});\n")
+
+        f.write("UNWIND [")
+        created = 0
+        for outer_index in range(0, number_of_files):
+            for inner_index in range(0, number_of_identities):
+
+                file_uuid = outer_index + 1
+                identity_uuid = number_of_files + inner_index + 1
+
+                if random.random() <= percentage_of_permissions:
+
+                    if created > 0:
+                        f.write(",")
+
+                    f.write(
+                        f' {{permUuid: {uuid}, permName: "name_permission_{uuid}", fileUuid: {file_uuid}, identityUuid: {identity_uuid}}}'
+                    )
+                    created += 1
+                    uuid += 1
+
+                    if created == 5000:
+                        f.write(
+                            "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);\nUNWIND ["
+                        )
+                        created = 0
+        f.write(
+            "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);\n"
+        )
+
+
+if __name__ == "__main__":
+    main()

tests/mgbench/datasets.py

@@ -353,7 +353,7 @@ class AccessControl(Dataset):
 
     def benchmark__create__vertex(self):
         self.next_value_idx += 1
-        query = (f"CREATE (:File {{uuid: {self.next_value_idx}}});", {})
+        query = ("CREATE (:File {uuid: $uuid})", {"uuid": self.next_value_idx})
         return query
 
     def benchmark__create__edges(self):
@@ -379,6 +379,24 @@ class AccessControl(Dataset):
         return query
 
     def benchmark__match__match_all_vertices_with_edges(self):
-        self.next_value_idx += 1
         query = ("MATCH (permission:Permission)-[e:IS_FOR_FILE]->(file:File) RETURN *", {})
         return query
+
+    def benchmark__match__match_users_with_permission_for_files(self):
+        file_uuid_1 = self._get_random_uuid("File")
+        file_uuid_2 = self._get_random_uuid("File")
+        min_file_uuid = min(file_uuid_1, file_uuid_2)
+        max_file_uuid = max(file_uuid_1, file_uuid_2)
+        query = (
+            "MATCH (f:File)<-[ff:IS_FOR_FILE]-(p:Permission)-[fi:IS_FOR_IDENTITY]->(i:Identity) WHERE f.uuid >= $min_file_uuid  AND f.uuid <= $max_file_uuid RETURN *",
+            {"min_file_uuid": min_file_uuid, "max_file_uuid": max_file_uuid},
+        )
+        return query
+
+    def benchmark__match__match_users_with_permission_for_specific_file(self):
+        file_uuid = self._get_random_uuid("File")
+        query = (
+            "MATCH (f:File {uuid: $file_uuid})<-[ff:IS_FOR_FILE]-(p:Permission)-[fi:IS_FOR_IDENTITY]->(i:Identity) RETURN *",
+            {"file_uuid": file_uuid},
+        )
+        return query

tests/mgbench/runners.py

@@ -68,6 +68,15 @@ class Memgraph:
         self._cleanup()
         atexit.unregister(self._cleanup)
 
+    # Returns None if string_value is not true or false, casing doesn't matter
+    def _get_bool_value(self, string_value):
+        lower_string_value = string_value.lower()
+        if lower_string_value == "true":
+            return True
+        if lower_string_value == "false":
+            return False
+        return None
+
     def _get_args(self, **kwargs):
         data_directory = os.path.join(self._directory.name, "memgraph")
         if self._memgraph_version >= (0, 50, 0):
@@ -83,7 +92,13 @@ class Memgraph:
             args_list = self._extra_args.split(" ")
             assert len(args_list) % 2 == 0
             for i in range(0, len(args_list), 2):
-                kwargs[args_list[i]] = args_list[i + 1]
+                key = args_list[i]
+                value = args_list[i + 1]
+                maybe_bool_value = self._get_bool_value(value)
+                if maybe_bool_value is not None:
+                    kwargs[key] = maybe_bool_value
+                else:
+                    kwargs[key] = value
 
         return _convert_args_to_flags(self._memgraph_binary, **kwargs)
 

tests/mgbench/splitfiles/accesscontrol_large.shard_configuration

@@ -1,8 +1,12 @@
-4
+8
 uuid
 email
 name
 platformId
+permUuid
+permName
+fileUuid
+identityUuid
 2
 IS_FOR_IDENTITY
 IS_FOR_FILE

tests/mgbench/splitfiles/accesscontrol_medium.shard_configuration

@@ -1,8 +1,12 @@
-4
+8
 uuid
 email
 name
 platformId
+permUuid
+permName
+fileUuid
+identityUuid
 2
 IS_FOR_IDENTITY
 IS_FOR_FILE

tests/mgbench/splitfiles/accesscontrol_small.shard_configuration

@@ -1,8 +1,12 @@
-4
+8
 uuid
 email
 name
 platformId
+permUuid
+permName
+fileUuid
+identityUuid
 2
 IS_FOR_IDENTITY
 IS_FOR_FILE

tests/simulation/CMakeLists.txt

@@ -32,4 +32,5 @@ add_simulation_test(trial_query_storage/query_storage_test.cpp)
 add_simulation_test(sharded_map.cpp)
 add_simulation_test(shard_rsm.cpp)
 add_simulation_test(cluster_property_test.cpp)
+add_simulation_test(cluster_property_test_cypher_queries.cpp)
 add_simulation_test(request_router.cpp)

tests/simulation/cluster_config.hpp

@@ -20,11 +20,11 @@ namespace memgraph::tests::simulation {
 struct ClusterConfig {
   int servers;
   int replication_factor;
-  int split_threshold;
+  int shards;
 
   friend std::ostream &operator<<(std::ostream &in, const ClusterConfig &cluster) {
     in << "ClusterConfig { servers: " << cluster.servers << ", replication_factor: " << cluster.replication_factor
-       << " }";
+       << ", shards: " << cluster.shards << " }";
     return in;
   }
 };
@@ -44,7 +44,7 @@ struct Arbitrary<ClusterConfig> {
         gen::set(&ClusterConfig::servers, gen::inRange(kMinimumServers, kMaximumServers)),
         gen::set(&ClusterConfig::replication_factor,
                  gen::inRange(kMinimumReplicationFactor, kMaximumReplicationFactor)),
-        gen::set(&ClusterConfig::split_threshold, gen::inRange(kMinimumSplitThreshold, kMaximumSplitThreshold)));
+        gen::set(&ClusterConfig::shards, gen::inRange(kMinimumShards, kMaximumShards)));
   }
 };
 

tests/simulation/cluster_property_test_cypher_queries.cpp

@@ -0,0 +1,64 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+// This test serves as an example of a property-based model test.
+// It generates a cluster configuration and a set of operations to
+// apply against both the real system and a greatly simplified model.
+
+#include <chrono>
+
+#include <gtest/gtest.h>
+#include <rapidcheck.h>
+#include <rapidcheck/gtest.h>
+#include <spdlog/cfg/env.h>
+
+#include "generated_operations.hpp"
+#include "io/simulator/simulator_config.hpp"
+#include "io/time.hpp"
+#include "storage/v3/shard_manager.hpp"
+#include "test_cluster.hpp"
+
+namespace memgraph::tests::simulation {
+
+using io::Duration;
+using io::Time;
+using io::simulator::SimulatorConfig;
+using storage::v3::kMaximumCronInterval;
+
+RC_GTEST_PROP(RandomClusterConfig, HappyPath, (ClusterConfig cluster_config, NonEmptyOpVec ops, uint64_t rng_seed)) {
+  spdlog::cfg::load_env_levels();
+
+  SimulatorConfig sim_config{
+      .drop_percent = 0,
+      .perform_timeouts = false,
+      .scramble_messages = true,
+      .rng_seed = rng_seed,
+      .start_time = Time::min(),
+      .abort_time = Time::max(),
+  };
+
+  std::vector<std::string> queries = {"CREATE (n:test_label{property_1: 0, property_2: 0});", "MATCH (n) RETURN n;"};
+
+  auto [sim_stats_1, latency_stats_1] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries);
+  auto [sim_stats_2, latency_stats_2] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries);
+
+  if (latency_stats_1 != latency_stats_2) {
+    spdlog::error("simulator stats diverged across runs");
+    spdlog::error("run 1 simulator stats: {}", sim_stats_1);
+    spdlog::error("run 2 simulator stats: {}", sim_stats_2);
+    spdlog::error("run 1 latency:\n{}", latency_stats_1.SummaryTable());
+    spdlog::error("run 2 latency:\n{}", latency_stats_2.SummaryTable());
+    RC_ASSERT(latency_stats_1 == latency_stats_2);
+    RC_ASSERT(sim_stats_1 == sim_stats_2);
+  }
+}
+
+}  // namespace memgraph::tests::simulation

tests/simulation/common.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -130,7 +130,6 @@ class MockedShardRsm {
   msgs::DeleteEdgesResponse ApplyImpl(msgs::DeleteEdgesRequest rqst) { return {}; }
   msgs::UpdateEdgesResponse ApplyImpl(msgs::UpdateEdgesRequest rqst) { return {}; }
   msgs::CommitResponse ApplyImpl(msgs::CommitRequest rqst) { return {}; }
-  msgs::CommitResponse ApplyImpl(msgs::SplitRequest rqst) { return {}; }
 
   WriteResponses Apply(WriteRequests write_requests) {
     return {std::visit([this]<typename T>(T &&request) { return WriteResponses{ApplyImpl(std::forward<T>(request))}; },

tests/simulation/generated_operations.hpp

@@ -35,20 +35,13 @@ struct CreateVertex {
 };
 
 struct ScanAll {
-  friend std::ostream &operator<<(std::ostream &in, const ScanAll & /* unused */) {
+  friend std::ostream &operator<<(std::ostream &in, const ScanAll &get) {
     in << "ScanAll {}";
     return in;
   }
 };
 
-struct AssertShardsSplit {
-  friend std::ostream &operator<<(std::ostream &in, const AssertShardsSplit & /* unused */) {
-    in << "AssertShardsSplit {}";
-    return in;
-  }
-};
-
-using OpVariant = std::variant<CreateVertex, ScanAll, AssertShardsSplit>;
+using OpVariant = std::variant<CreateVertex, ScanAll>;
 
 struct Op {
   OpVariant inner;
@@ -88,8 +81,8 @@ using namespace memgraph::tests::simulation;
 template <>
 struct Arbitrary<CreateVertex> {
   static Gen<CreateVertex> arbitrary() {
-    return gen::build<CreateVertex>(gen::set(&CreateVertex::first, gen::inRange(0, kMaximumKey)),
-                                    gen::set(&CreateVertex::second, gen::inRange(0, kMaximumKey)));
+    return gen::build<CreateVertex>(gen::set(&CreateVertex::first, gen::inRange(0, kMaximumShards + 1)),
+                                    gen::set(&CreateVertex::second, gen::inRange(0, kMaximumShards + 1)));
   }
 };
 
@@ -98,23 +91,15 @@ struct Arbitrary<ScanAll> {
   static Gen<ScanAll> arbitrary() { return gen::just(ScanAll{}); }
 };
 
-template <>
-struct Arbitrary<AssertShardsSplit> {
-  static Gen<AssertShardsSplit> arbitrary() { return gen::just(AssertShardsSplit{}); }
-};
-
 OpVariant opHoist(ScanAll op) { return op; }
 OpVariant opHoist(CreateVertex op) { return op; }
-OpVariant opHoist(AssertShardsSplit op) { return op; }
 
 template <>
 struct ::rc::Arbitrary<Op> {
   static Gen<Op> arbitrary() {
     return gen::build<Op>(gen::set(
-        &Op::inner,
-        gen::oneOf(gen::map(gen::arbitrary<CreateVertex>(), [](CreateVertex op) { return opHoist(op); }),
-                   gen::map(gen::arbitrary<ScanAll>(), [](ScanAll op) { return opHoist(op); }),
-                   gen::map(gen::arbitrary<AssertShardsSplit>(), [](AssertShardsSplit op) { return opHoist(op); }))));
+        &Op::inner, gen::oneOf(gen::map(gen::arbitrary<CreateVertex>(), [](CreateVertex op) { return opHoist(op); }),
+                               gen::map(gen::arbitrary<ScanAll>(), [](ScanAll op) { return opHoist(op); }))));
   }
 };
 

tests/simulation/request_router.cpp

@@ -24,7 +24,6 @@
 #include "common/types.hpp"
 #include "coordinator/coordinator_client.hpp"
 #include "coordinator/coordinator_rsm.hpp"
-#include "coordinator/shard_map.hpp"
 #include "io/address.hpp"
 #include "io/errors.hpp"
 #include "io/rsm/raft.hpp"
@@ -40,7 +39,7 @@
 #include "utils/result.hpp"
 
 namespace memgraph::query::v2::tests {
-using coordinator::PeerMetadata;
+using coordinator::AddressAndStatus;
 using CompoundKey = coordinator::PrimaryKey;
 using coordinator::Coordinator;
 using coordinator::CoordinatorClient;
@@ -110,11 +109,11 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
   shards_for_label.clear();
 
   // add first shard at [0, 0]
-  PeerMetadata aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = {1}};
+  ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
 
   auto key1 = storage::v3::PropertyValue(0);
   auto key2 = storage::v3::PropertyValue(0);
@@ -122,11 +121,11 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
   shards_for_label[compound_key_1] = shard1;
 
   // add second shard at [12, 13]
-  PeerMetadata aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = {1}};
+  ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
 
   auto key3 = storage::v3::PropertyValue(12);
   auto key4 = storage::v3::PropertyValue(13);

tests/simulation/shard_rsm.cpp

@@ -1496,13 +1496,9 @@ int TestMessages() {
 
   std::vector<SchemaProperty> schema_prop = {get_schema_property()};
 
-  const coordinator::Hlc shard_version{0};
-  auto shard_ptr1 =
-      std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
-  auto shard_ptr2 =
-      std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
-  auto shard_ptr3 =
-      std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
+  auto shard_ptr1 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
+  auto shard_ptr2 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
+  auto shard_ptr3 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
 
   shard_ptr1->StoreMapping(
       {{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}, {7, "e_prop"}});
@@ -1515,13 +1511,9 @@ int TestMessages() {
   std::vector<Address> address_for_2{shard_server_1_address, shard_server_3_address};
   std::vector<Address> address_for_3{shard_server_1_address, shard_server_2_address};
 
-  utils::Sender<io::messages::ShardManagerMessages> local_shard_manager_sender{[](const auto &elem) {}};
-  ConcreteShardRsm shard_server1(std::move(shard_server_io_1), address_for_1,
-                                 ShardRsm(std::move(shard_ptr1), local_shard_manager_sender));
-  ConcreteShardRsm shard_server2(std::move(shard_server_io_2), address_for_2,
-                                 ShardRsm(std::move(shard_ptr2), local_shard_manager_sender));
-  ConcreteShardRsm shard_server3(std::move(shard_server_io_3), address_for_3,
-                                 ShardRsm(std::move(shard_ptr3), local_shard_manager_sender));
+  ConcreteShardRsm shard_server1(std::move(shard_server_io_1), address_for_1, ShardRsm(std::move(shard_ptr1)));
+  ConcreteShardRsm shard_server2(std::move(shard_server_io_2), address_for_2, ShardRsm(std::move(shard_ptr2)));
+  ConcreteShardRsm shard_server3(std::move(shard_server_io_3), address_for_3, ShardRsm(std::move(shard_ptr3)));
 
   auto server_thread1 = std::jthread([&shard_server1]() { shard_server1.Run(); });
   simulator.IncrementServerCountAndWaitForQuiescentState(shard_server_1_address);

tests/simulation/sharded_map.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,12 +33,12 @@
 #include "utils/result.hpp"
 
 using memgraph::common::SchemaType;
+using memgraph::coordinator::AddressAndStatus;
 using memgraph::coordinator::Coordinator;
 using memgraph::coordinator::CoordinatorClient;
 using memgraph::coordinator::CoordinatorRsm;
 using memgraph::coordinator::HlcRequest;
 using memgraph::coordinator::HlcResponse;
-using memgraph::coordinator::PeerMetadata;
 using memgraph::coordinator::PrimaryKey;
 using memgraph::coordinator::ShardMap;
 using memgraph::coordinator::ShardMetadata;
@@ -105,9 +105,9 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
   shards_for_label.clear();
 
   // add first shard at [0, 0]
-  PeerMetadata aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
   ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
 
@@ -117,9 +117,9 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
   shards_for_label.emplace(compound_key_1, shard1);
 
   // add second shard at [12, 13]
-  PeerMetadata aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
-  PeerMetadata aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
+  AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
   ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
 

tests/simulation/simulation_interpreter.hpp

@@ -27,18 +27,9 @@
 
 namespace memgraph::io::simulator {
 
-class FlatStream {
-  std::vector<query::v2::TypedValue> all_results_;
-
- public:
-  void Result(const std::vector<query::v2::TypedValue> &new_values) {
-    all_results_.insert(all_results_.end(), new_values.begin(), new_values.end());
-  }
-
-  std::vector<query::v2::TypedValue> &&TakeResults() { return std::move(all_results_); }
-};
-
 class SimulatedInterpreter {
+  using ResultStream = query::v2::DiscardValueResultStream;
+
  public:
   explicit SimulatedInterpreter(std::unique_ptr<query::v2::InterpreterContext> interpreter_context)
       : interpreter_context_(std::move(interpreter_context)) {
@@ -51,8 +42,23 @@ class SimulatedInterpreter {
   SimulatedInterpreter &operator=(SimulatedInterpreter &&) = delete;
   ~SimulatedInterpreter() = default;
 
-  std::vector<query::v2::TypedValue> RunQuery(const std::string &query) {
-    FlatStream stream;
+  void InstallSimulatorTicker(Simulator &simulator) {
+    interpreter_->InstallSimulatorTicker(simulator.GetSimulatorTickClosure());
+  }
+
+  std::vector<ResultStream> RunQueries(const std::vector<std::string> &queries) {
+    std::vector<ResultStream> results;
+    results.reserve(queries.size());
+
+    for (const auto &query : queries) {
+      results.emplace_back(RunQuery(query));
+    }
+    return results;
+  }
+
+ private:
+  ResultStream RunQuery(const std::string &query) {
+    ResultStream stream;
 
     std::map<std::string, memgraph::storage::v3::PropertyValue> params;
     const std::string *username = nullptr;
@@ -60,10 +66,9 @@ class SimulatedInterpreter {
     interpreter_->Prepare(query, params, username);
     interpreter_->PullAll(&stream);
 
-    return stream.TakeResults();
+    return stream;
   }
 
- private:
   std::unique_ptr<query::v2::InterpreterContext> interpreter_context_;
   std::unique_ptr<query::v2::Interpreter> interpreter_;
 };

tests/simulation/test_cluster.hpp

@@ -36,7 +36,6 @@
 #include "utils/print_helpers.hpp"
 #include "utils/variant_helpers.hpp"
 
-#include "generated_operations.hpp"
 #include "simulation_interpreter.hpp"
 
 namespace memgraph::tests::simulation {
@@ -72,13 +71,6 @@ using storage::v3::SchemaProperty;
 using CompoundKey = std::pair<int, int>;
 using ShardClient = RsmClient<SimulatorTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
 
-struct SimClientContext {
-  CoordinatorClient<SimulatorTransport> coordinator_client;
-  const ClusterConfig &cluster_config;
-  io::simulator::SimulatedInterpreter interpreter;
-  std::set<CompoundKey> correctness_model;
-};
-
 MachineManager<SimulatorTransport> MkMm(Simulator &simulator, std::vector<Address> coordinator_addresses, Address addr,
                                         ShardMap shard_map) {
   MachineConfig config{
@@ -124,8 +116,9 @@ void WaitForShardsToInitialize(CoordinatorClient<SimulatorTransport> &coordinato
   }
 }
 
-ShardMap TestShardMap(const ClusterConfig &cluster_config) {
+ShardMap TestShardMap(int n_splits, int replication_factor) {
   ShardMap sm{};
+
   const std::string label_name = std::string("test_label");
 
   // register new properties
@@ -142,83 +135,67 @@ ShardMap TestShardMap(const ClusterConfig &cluster_config) {
       SchemaProperty{.property_id = property_id_2, .type = type_2},
   };
 
-  std::optional<LabelId> label_id = sm.InitializeNewLabel(label_name, schema, cluster_config.replication_factor,
-                                                          cluster_config.split_threshold, sm.shard_map_version);
+  std::optional<LabelId> label_id = sm.InitializeNewLabel(label_name, schema, replication_factor, sm.shard_map_version);
   RC_ASSERT(label_id.has_value());
 
+  // split the shard at N split points
+  for (int64_t i = 1; i < n_splits; ++i) {
+    const auto key1 = memgraph::storage::v3::PropertyValue(i);
+    const auto key2 = memgraph::storage::v3::PropertyValue(0);
+
+    const auto split_point = {key1, key2};
+
+    const bool split_success = sm.SplitShard(sm.shard_map_version, label_id.value(), split_point);
+
+    RC_ASSERT(split_success);
+  }
+
   return sm;
 }
 
-void ExecuteOp(SimClientContext &context, CreateVertex create_vertex) {
+void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std::set<CompoundKey> &correctness_model,
+               CreateVertex create_vertex) {
   const auto key1 = memgraph::storage::v3::PropertyValue(create_vertex.first);
   const auto key2 = memgraph::storage::v3::PropertyValue(create_vertex.second);
 
   std::vector<msgs::Value> primary_key = {msgs::Value(int64_t(create_vertex.first)),
                                           msgs::Value(int64_t(create_vertex.second))};
 
-  if (context.correctness_model.contains(std::make_pair(create_vertex.first, create_vertex.second))) {
+  if (correctness_model.contains(std::make_pair(create_vertex.first, create_vertex.second))) {
     // TODO(tyler) remove this early-return when we have properly handled setting non-unique vertexes
     return;
   }
 
-  std::string query = fmt::format("CREATE (n:test_label{{property_1: {}, property_2: {}}});", create_vertex.first,
-                                  create_vertex.second);
+  auto label_id = request_router.NameToLabel("test_label");
 
-  auto result_stream = context.interpreter.RunQuery(query);
+  msgs::NewVertex nv{.primary_key = primary_key};
+  nv.label_ids.push_back({label_id});
 
-  // TODO(tyler) is there a better way to assert the actual success of the operation?
-  RC_ASSERT(result_stream.size() == 0);
+  std::vector<msgs::NewVertex> new_vertices;
+  new_vertices.push_back(std::move(nv));
 
-  // RC_ASSERT(!result_stream[0].error.has_value());
+  auto result = request_router.CreateVertices(std::move(new_vertices));
 
-  context.correctness_model.emplace(std::make_pair(create_vertex.first, create_vertex.second));
+  RC_ASSERT(result.size() == 1);
+  RC_ASSERT(!result[0].error.has_value());
+
+  correctness_model.emplace(std::make_pair(create_vertex.first, create_vertex.second));
 }
 
-void ExecuteOp(SimClientContext &context, ScanAll scan_all) {
-  auto results = context.interpreter.RunQuery("MATCH (n) RETURN n;");
+void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std::set<CompoundKey> &correctness_model,
+               ScanAll scan_all) {
+  auto results = request_router.ScanVertices("test_label");
 
-  RC_ASSERT(results.size() == context.correctness_model.size());
+  RC_ASSERT(results.size() == correctness_model.size());
 
-  for (const auto &typed_value : results) {
-    // TODO(tyler) assert on actual values returned
-    // const auto properties = vertex_accessor.Properties();
-    // const auto primary_key = vertex_accessor.Id().second;
-    // const CompoundKey model_key = std::make_pair(primary_key[0].int_v, primary_key[1].int_v);
-    // RC_ASSERT(context.correctness_model.contains(model_key));
+  for (const auto &vertex_accessor : results) {
+    const auto properties = vertex_accessor.Properties();
+    const auto primary_key = vertex_accessor.Id().second;
+    const CompoundKey model_key = std::make_pair(primary_key[0].int_v, primary_key[1].int_v);
+    RC_ASSERT(correctness_model.contains(model_key));
   }
 }
 
-void ExecuteOp(SimClientContext &context, AssertShardsSplit assert_shards_split) {
-  const int minimum_expected_shards = (context.correctness_model.size() / context.cluster_config.split_threshold) + 1;
-  // TODO(tyler) make this a higher number of retries
-  const int maximum_attempts = 10'000;
-  size_t initialized_shards;
-
-  for (int i = 0; i < maximum_attempts; i++) {
-    GetShardMapRequest req{};
-    CoordinatorReadRequests read_req = req;
-    auto read_res = context.coordinator_client.SendReadRequest(read_req);
-    if (read_res.HasError()) {
-      // timed out
-      continue;
-    }
-    auto response_result = read_res.GetValue();
-    auto response = std::get<GetShardMapResponse>(response_result);
-    auto shard_map = response.shard_map;
-
-    initialized_shards = shard_map.InitializedShards();
-
-    if (initialized_shards >= minimum_expected_shards) {
-      MG_ASSERT(initialized_shards < 3, "just kidding, this is great, we now have {} initialized shards",
-                initialized_shards);
-      return;
-    }
-  }
-
-  spdlog::error("expected {} shards, but we only have {}", minimum_expected_shards, initialized_shards);
-  MG_ASSERT(false, "exceeded maximum attempts for waiting for the expected number of shard splits to occur");
-}
-
 /// This struct exists as a way of detaching
 /// a thread if something causes an uncaught
 /// exception - because that thread would not
@@ -245,17 +222,15 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
   auto machine_1_addr = Address::TestAddress(1);
   auto cli_addr = Address::TestAddress(2);
   auto cli_addr_2 = Address::TestAddress(3);
-  auto cli_addr_3 = Address::TestAddress(4);
 
   Io<SimulatorTransport> cli_io = simulator.Register(cli_addr);
   Io<SimulatorTransport> cli_io_2 = simulator.Register(cli_addr_2);
-  Io<SimulatorTransport> cli_io_3 = simulator.Register(cli_addr_3);
 
   auto coordinator_addresses = std::vector{
       machine_1_addr,
   };
 
-  ShardMap initialization_sm = TestShardMap(cluster_config);
+  ShardMap initialization_sm = TestShardMap(cluster_config.shards - 1, cluster_config.replication_factor);
 
   auto mm_1 = MkMm(simulator, coordinator_addresses, machine_1_addr, initialization_sm);
   Address coordinator_address = mm_1.CoordinatorAddress();
@@ -267,19 +242,19 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
 
   // TODO(tyler) clarify addresses of coordinator etc... as it's a mess
 
+  CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, coordinator_address, {coordinator_address});
+  WaitForShardsToInitialize(coordinator_client);
+
+  query::v2::RequestRouter<SimulatorTransport> request_router(std::move(coordinator_client), std::move(cli_io));
+  std::function<bool()> tick_simulator = simulator.GetSimulatorTickClosure();
+  request_router.InstallSimulatorTicker(tick_simulator);
+
+  request_router.StartTransaction();
+
   auto correctness_model = std::set<CompoundKey>{};
 
-  SimClientContext context{
-      .coordinator_client = CoordinatorClient<SimulatorTransport>(cli_io, coordinator_address, {coordinator_address}),
-      .cluster_config = cluster_config,
-      .interpreter = io::simulator::SetUpInterpreter(coordinator_address, simulator),
-      .correctness_model = correctness_model,
-  };
-
-  WaitForShardsToInitialize(context.coordinator_client);
-
   for (const Op &op : ops) {
-    std::visit([&](auto &o) { ExecuteOp(context, o); }, op.inner);
+    std::visit([&](auto &o) { ExecuteOp(request_router, correctness_model, o); }, op.inner);
   }
 
   // We have now completed our workload without failing any assertions, so we can
@@ -306,4 +281,65 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
   return std::make_pair(stats, histo);
 }
 
+std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulationWithQueries(
+    const SimulatorConfig &sim_config, const ClusterConfig &cluster_config, const std::vector<std::string> &queries) {
+  spdlog::info("========================== NEW SIMULATION ==========================");
+
+  auto simulator = Simulator(sim_config);
+
+  auto machine_1_addr = Address::TestAddress(1);
+  auto cli_addr = Address::TestAddress(2);
+  auto cli_addr_2 = Address::TestAddress(3);
+
+  Io<SimulatorTransport> cli_io = simulator.Register(cli_addr);
+  Io<SimulatorTransport> cli_io_2 = simulator.Register(cli_addr_2);
+
+  auto coordinator_addresses = std::vector{
+      machine_1_addr,
+  };
+
+  ShardMap initialization_sm = TestShardMap(cluster_config.shards - 1, cluster_config.replication_factor);
+
+  auto mm_1 = MkMm(simulator, coordinator_addresses, machine_1_addr, initialization_sm);
+  Address coordinator_address = mm_1.CoordinatorAddress();
+
+  auto mm_thread_1 = std::jthread(RunMachine, std::move(mm_1));
+  simulator.IncrementServerCountAndWaitForQuiescentState(machine_1_addr);
+
+  auto detach_on_error = DetachIfDropped{.handle = mm_thread_1};
+
+  // TODO(tyler) clarify addresses of coordinator etc... as it's a mess
+
+  CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, coordinator_address, {coordinator_address});
+  WaitForShardsToInitialize(coordinator_client);
+
+  auto simulated_interpreter = io::simulator::SetUpInterpreter(coordinator_address, simulator);
+  simulated_interpreter.InstallSimulatorTicker(simulator);
+
+  auto query_results = simulated_interpreter.RunQueries(queries);
+
+  // We have now completed our workload without failing any assertions, so we can
+  // disable detaching the worker thread, which will cause the mm_thread_1 jthread
+  // to be joined when this function returns.
+  detach_on_error.detach = false;
+
+  simulator.ShutDown();
+
+  mm_thread_1.join();
+
+  SimulatorStats stats = simulator.Stats();
+
+  spdlog::info("total messages:     {}", stats.total_messages);
+  spdlog::info("dropped messages:   {}", stats.dropped_messages);
+  spdlog::info("timed out requests: {}", stats.timed_out_requests);
+  spdlog::info("total requests:     {}", stats.total_requests);
+  spdlog::info("total responses:    {}", stats.total_responses);
+  spdlog::info("simulator ticks:    {}", stats.simulator_ticks);
+
+  auto histo = cli_io_2.ResponseLatencies();
+
+  spdlog::info("========================== SUCCESS :) ==========================");
+  return std::make_pair(stats, histo);
+}
+
 }  // namespace memgraph::tests::simulation

tests/simulation/testing_constants.hpp

@@ -13,7 +13,9 @@
 
 namespace memgraph::tests::simulation {
 
-static constexpr auto kMaximumKey = 50;
+// TODO(tyler) increase this when we start standing up multiple machines in cluster tests
+static constexpr auto kMinimumShards = 1;
+static constexpr auto kMaximumShards = kMinimumShards + 10;
 
 // TODO(tyler) increase this when we start standing up multiple machines in cluster tests
 static constexpr auto kMinimumServers = 1;
@@ -23,7 +25,4 @@ static constexpr auto kMaximumServers = kMinimumServers + 1;
 static constexpr auto kMinimumReplicationFactor = 1;
 static constexpr auto kMaximumReplicationFactor = kMinimumReplicationFactor + 1;
 
-static constexpr auto kMinimumSplitThreshold = 3;
-static constexpr auto kMaximumSplitThreshold = 10;
-
 }  // namespace memgraph::tests::simulation

tests/unit/CMakeLists.txt

@@ -294,9 +294,6 @@ target_link_libraries(${test_prefix}storage_v3_expr mg-storage-v3 mg-expr)
 add_unit_test(storage_v3_schema.cpp)
 target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3)
 
-add_unit_test(storage_v3_shard_split.cpp)
-target_link_libraries(${test_prefix}storage_v3_shard_split mg-storage-v3  mg-query-v2)
-
 # Test mg-query-v2
 # These are commented out because of the new TypedValue in the query engine
 # add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp)

tests/unit/mock_helpers.hpp

@@ -41,7 +41,7 @@ class MockedRequestRouter : public RequestRouterInterface {
   MOCK_METHOD(std::optional<storage::v3::EdgeTypeId>, MaybeNameToEdgeType, (const std::string &), (const));
   MOCK_METHOD(std::optional<storage::v3::LabelId>, MaybeNameToLabel, (const std::string &), (const));
   MOCK_METHOD(bool, IsPrimaryLabel, (storage::v3::LabelId), (const));
-  MOCK_METHOD(bool, IsPrimaryKey, (storage::v3::LabelId, storage::v3::PropertyId), (const));
+  MOCK_METHOD(bool, IsPrimaryProperty, (storage::v3::LabelId, storage::v3::PropertyId), (const));
   MOCK_METHOD((std::optional<std::pair<uint64_t, uint64_t>>), AllocateInitialEdgeIds, (io::Address));
   MOCK_METHOD(void, InstallSimulatorTicker, (std::function<bool()>));
   MOCK_METHOD(const std::vector<coordinator::SchemaProperty> &, GetSchemaForLabel, (storage::v3::LabelId), (const));

tests/unit/query_v2_create_node_multiframe.cpp

@@ -58,7 +58,7 @@ TEST(CreateNodeTest, CreateNodeCursor) {
   MockedRequestRouter router;
   EXPECT_CALL(router, CreateVertices(_)).Times(1).WillOnce(Return(std::vector<msgs::CreateVerticesResponse>{}));
   EXPECT_CALL(router, IsPrimaryLabel(_)).WillRepeatedly(Return(true));
-  EXPECT_CALL(router, IsPrimaryKey(_, _)).WillRepeatedly(Return(true));
+  EXPECT_CALL(router, IsPrimaryProperty(_, _)).WillRepeatedly(Return(true));
   auto context = MakeContext(ast, symbol_table, &router, &id_alloc);
   auto multi_frame = CreateMultiFrame(context.symbol_table.max_position());
   cursor->PullMultiple(multi_frame, context);

tests/unit/query_v2_expression_evaluator.cpp

@@ -123,7 +123,7 @@ class MockedRequestRouter : public RequestRouterInterface {
 
   bool IsPrimaryLabel(LabelId label) const override { return true; }
 
-  bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override { return true; }
+  bool IsPrimaryProperty(LabelId primary_label, PropertyId property) const override { return true; }
 
   std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) override {
     return {};

tests/unit/storage_v3.cpp

@@ -78,9 +78,9 @@ class StorageV3 : public ::testing::TestWithParam<bool> {
   const PropertyId primary_property{PropertyId::FromUint(2)};
   std::vector<storage::v3::SchemaProperty> schema_property_vector = {
       storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
+  Shard store{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector,
+              Config{.gc = {.reclamation_interval = reclamation_interval}}};
   coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard store{primary_label,          min_pk,   std::nullopt /*max_primary_key*/,
-              schema_property_vector, last_hlc, Config{.gc = {.reclamation_interval = reclamation_interval}}};
 };
 INSTANTIATE_TEST_SUITE_P(WithGc, StorageV3, ::testing::Values(true));
 INSTANTIATE_TEST_SUITE_P(WithoutGc, StorageV3, ::testing::Values(false));
@@ -2650,7 +2650,7 @@ TEST_P(StorageV3, TestCreateVertexAndValidate) {
               (std::map<PropertyId, PropertyValue>{{prop1, PropertyValue(111)}}));
   }
   {
-    Shard store(primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc);
+    Shard store(primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector);
     auto acc = store.Access(GetNextHlc());
     auto vertex1 = acc.CreateVertexAndValidate({}, {PropertyValue{0}}, {});
     auto vertex2 = acc.CreateVertexAndValidate({}, {PropertyValue{0}}, {});

tests/unit/storage_v3_edge.cpp

@@ -54,9 +54,10 @@ class StorageEdgeTest : public ::testing::TestWithParam<bool> {
   const PropertyId primary_property{PropertyId::FromUint(2)};
   std::vector<storage::v3::SchemaProperty> schema_property_vector = {
       storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
+  Shard store{primary_label, min_pk, max_pk, schema_property_vector,
+              Config{.items = {.properties_on_edges = GetParam()}}};
+
   coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard store{primary_label,          min_pk,   max_pk,
-              schema_property_vector, last_hlc, Config{.items = {.properties_on_edges = GetParam()}}};
 };
 
 INSTANTIATE_TEST_SUITE_P(EdgesWithProperties, StorageEdgeTest, ::testing::Values(true));

tests/unit/storage_v3_expr.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -136,8 +136,7 @@ class ExpressionEvaluatorTest : public ::testing::Test {
 
   std::vector<storage::v3::SchemaProperty> schema_property_vector = {
       storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
-  coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard db{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
+  Shard db{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
 
   Shard::Accessor storage_dba{db.Access(GetNextHlc())};
   DbAccessor dba{&storage_dba};
@@ -150,6 +149,8 @@ class ExpressionEvaluatorTest : public ::testing::Test {
   Frame frame{128};
   ExpressionEvaluator eval{&frame, symbol_table, ctx, &dba, View::OLD};
 
+  coordinator::Hlc last_hlc{0, io::Time{}};
+
   void SetUp() override { db.StoreMapping({{1, "label"}, {2, "property"}}); }
 
   std::vector<PropertyId> NamesToProperties(const std::vector<std::string> &property_names) {

tests/unit/storage_v3_indices.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -44,8 +44,7 @@ class IndexTest : public testing::Test {
       storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
   const std::vector<PropertyValue> min_pk{PropertyValue{0}};
   const LabelId primary_label{LabelId::FromUint(1)};
-  coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
+  Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
 
   const PropertyId prop_id{PropertyId::FromUint(5)};
   const PropertyId prop_val{PropertyId::FromUint(6)};
@@ -56,6 +55,7 @@ class IndexTest : public testing::Test {
   static constexpr io::Duration one_time_unit{1};
   int primary_key_id{0};
   int vertex_id{0};
+  coordinator::Hlc last_hlc{0, io::Time{}};
 
   LabelId NameToLabelId(std::string_view label_name) { return storage.NameToLabel(label_name); }
 

tests/unit/storage_v3_isolation_level.cpp

@@ -82,9 +82,8 @@ TEST_P(StorageIsolationLevelTest, Visibility) {
 
   for (auto override_isolation_level_index{0U}; override_isolation_level_index < isolation_levels.size();
        ++override_isolation_level_index) {
-    coordinator::Hlc last_hlc{0, io::Time{}};
-    Shard store{primary_label,          min_pk,   max_pk,
-                schema_property_vector, last_hlc, Config{.transaction = {.isolation_level = default_isolation_level}}};
+    Shard store{primary_label, min_pk, max_pk, schema_property_vector,
+                Config{.transaction = {.isolation_level = default_isolation_level}}};
     const auto override_isolation_level = isolation_levels[override_isolation_level_index];
     auto creator = store.Access(GetNextHlc());
     auto default_isolation_level_reader = store.Access(GetNextHlc());

tests/unit/storage_v3_shard_rsm.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -58,20 +58,15 @@ class ShardRSMTest : public testing::Test {
     PropertyValue max_pk(static_cast<int64_t>(10000000));
     std::vector<PropertyValue> max_prim_key = {max_pk};
 
-    coordinator::Hlc shard_version{GetTransactionId()};
-    auto shard_ptr1 =
-        std::make_unique<Shard>(primary_label, min_prim_key, max_prim_key, std::vector{schema_prop}, shard_version);
+    auto shard_ptr1 = std::make_unique<Shard>(primary_label, min_prim_key, max_prim_key, std::vector{schema_prop});
     shard_ptr1->StoreMapping({{1, "primary_label"},
                               {2, "primary_label2"},
                               {3, "label"},
                               {4, "primary_prop1"},
                               {5, "primary_prop2"},
                               {6, "prop"}});
-    coordinator::Address local_shard_manager_address = coordinator::Address();
-    utils::Sender<io::messages::ShardManagerMessages> local_shard_manager_sender{[](const auto &elem) {}};
-
     shard_ptr1->CreateSchema(primary_label2, {{primary_property2, SchemaType::INT}});
-    shard_rsm = std::make_unique<ShardRsm>(std::move(shard_ptr1), std::move(local_shard_manager_sender));
+    shard_rsm = std::make_unique<ShardRsm>(std::move(shard_ptr1));
   }
 
   LabelId NameToLabel(const std::string &name) { return LabelId::FromUint(id_mapper_.NameToId(name)); }

tests/unit/storage_v3_shard_split.cpp

@@ -1,500 +0,0 @@
-// Copyright 2023 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <cstdint>
-#include <memory>
-
-#include <gmock/gmock-matchers.h>
-#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "query/v2/requests.hpp"
-#include "storage/v3/delta.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/mvcc.hpp"
-#include "storage/v3/property_value.hpp"
-#include "storage/v3/shard.hpp"
-#include "storage/v3/vertex.hpp"
-#include "storage/v3/vertex_id.hpp"
-
-using testing::Pair;
-using testing::UnorderedElementsAre;
-
-namespace memgraph::storage::v3::tests {
-
-class ShardSplitTest : public testing::Test {
- protected:
-  void SetUp() override {
-    storage.StoreMapping(
-        {{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
-  }
-
-  const PropertyId primary_property{PropertyId::FromUint(2)};
-  const PropertyId secondary_property{PropertyId::FromUint(5)};
-  std::vector<storage::v3::SchemaProperty> schema_property_vector = {
-      storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
-  const std::vector<PropertyValue> min_pk{PropertyValue{0}};
-  const LabelId primary_label{LabelId::FromUint(1)};
-  const LabelId secondary_label{LabelId::FromUint(4)};
-  const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
-  coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
-
-  coordinator::Hlc GetNextHlc() {
-    ++last_hlc.logical_id;
-    last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
-    return last_hlc;
-  }
-
-  void AssertShardState(auto &shard, const int split_min, const int split_max) {
-    auto acc = shard.Access(GetNextHlc());
-    for (int i{0}; i < split_min; ++i) {
-      EXPECT_FALSE(acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD).has_value());
-    }
-    for (int i{split_min}; i < split_max; ++i) {
-      const auto vtx = acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD);
-      ASSERT_TRUE(vtx.has_value());
-      EXPECT_TRUE(vtx->InEdges(View::OLD)->size() == 1 || vtx->OutEdges(View::OLD)->size() == 1);
-    }
-  }
-};
-
-void AssertEqVertexContainer(const VertexContainer &actual, const VertexContainer &expected) {
-  ASSERT_EQ(actual.size(), expected.size());
-
-  auto expected_it = expected.begin();
-  auto actual_it = actual.begin();
-  while (expected_it != expected.end()) {
-    EXPECT_EQ(actual_it->first, expected_it->first);
-    EXPECT_EQ(actual_it->second.deleted, expected_it->second.deleted);
-    EXPECT_EQ(actual_it->second.labels, expected_it->second.labels);
-
-    auto *expected_delta = expected_it->second.delta;
-    auto *actual_delta = actual_it->second.delta;
-    // This asserts delta chain
-    while (expected_delta != nullptr) {
-      EXPECT_EQ(actual_delta->action, expected_delta->action);
-      EXPECT_EQ(actual_delta->id, expected_delta->id);
-
-      switch (expected_delta->action) {
-        case Delta::Action::ADD_LABEL:
-        case Delta::Action::REMOVE_LABEL: {
-          EXPECT_EQ(actual_delta->label, expected_delta->label);
-          break;
-        }
-        case Delta::Action::SET_PROPERTY: {
-          EXPECT_EQ(actual_delta->property.key, expected_delta->property.key);
-          EXPECT_EQ(actual_delta->property.value, expected_delta->property.value);
-          break;
-        }
-        case Delta::Action::ADD_IN_EDGE:
-        case Delta::Action::ADD_OUT_EDGE:
-        case Delta::Action::REMOVE_IN_EDGE:
-        case Delta::Action::RECREATE_OBJECT:
-        case Delta::Action::DELETE_OBJECT:
-        case Delta::Action::REMOVE_OUT_EDGE: {
-          break;
-        }
-      }
-
-      const auto expected_prev = expected_delta->prev.Get();
-      const auto actual_prev = actual_delta->prev.Get();
-      switch (expected_prev.type) {
-        case PreviousPtr::Type::NULLPTR: {
-          ASSERT_EQ(actual_prev.type, PreviousPtr::Type::NULLPTR) << "Expected type is nullptr!";
-          break;
-        }
-        case PreviousPtr::Type::DELTA: {
-          ASSERT_EQ(actual_prev.type, PreviousPtr::Type::DELTA) << "Expected type is delta!";
-          EXPECT_EQ(actual_prev.delta->action, expected_prev.delta->action);
-          EXPECT_EQ(actual_prev.delta->id, expected_prev.delta->id);
-          break;
-        }
-        case v3::PreviousPtr::Type::EDGE: {
-          ASSERT_EQ(actual_prev.type, PreviousPtr::Type::EDGE) << "Expected type is edge!";
-          EXPECT_EQ(actual_prev.edge->gid, expected_prev.edge->gid);
-          break;
-        }
-        case v3::PreviousPtr::Type::VERTEX: {
-          ASSERT_EQ(actual_prev.type, PreviousPtr::Type::VERTEX) << "Expected type is vertex!";
-          EXPECT_EQ(actual_prev.vertex->first, expected_prev.vertex->first);
-          break;
-        }
-      }
-
-      expected_delta = expected_delta->next;
-      actual_delta = actual_delta->next;
-    }
-    EXPECT_EQ(expected_delta, nullptr);
-    EXPECT_EQ(actual_delta, nullptr);
-    ++expected_it;
-    ++actual_it;
-  }
-}
-
-void AssertEqDeltaLists(const std::list<Delta> &actual, const std::list<Delta> &expected) {
-  EXPECT_EQ(actual.size(), expected.size());
-  auto actual_it = actual.begin();
-  auto expected_it = expected.begin();
-  while (actual_it != actual.end()) {
-    EXPECT_EQ(actual_it->id, expected_it->id);
-    EXPECT_EQ(actual_it->action, expected_it->action);
-    ++actual_it;
-    ++expected_it;
-  }
-}
-
-void AddDeltaToDeltaChain(Vertex *object, Delta *new_delta) {
-  auto *delta_holder = GetDeltaHolder(object);
-
-  new_delta->next = delta_holder->delta;
-  new_delta->prev.Set(object);
-  if (delta_holder->delta) {
-    delta_holder->delta->prev.Set(new_delta);
-  }
-  delta_holder->delta = new_delta;
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitWithVertices) {
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(
-      acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(121)}})
-          .HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-  auto current_hlc = GetNextHlc();
-  acc.Commit(current_hlc);
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 0);
-  EXPECT_EQ(splitted_data.transactions.size(), 1);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
-
-  CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
-  Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 4, 1};
-  Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 5, 2};
-  Delta delta_remove_label{Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4};
-  Delta delta_set_property{Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4};
-  Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 8, 3};
-
-  VertexContainer expected_vertices;
-  auto [it_4, inserted1] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
-  delta_delete1.prev.Set(&*it_4);
-  auto [it_5, inserted2] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
-  delta_delete2.prev.Set(&*it_5);
-  auto [it_6, inserted3] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
-  delta_delete3.prev.Set(&*it_6);
-  it_5->second.labels.push_back(secondary_label);
-  AddDeltaToDeltaChain(&*it_5, &delta_set_property);
-  AddDeltaToDeltaChain(&*it_5, &delta_remove_label);
-
-  AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
-
-  // This is to ensure that the transaction that we have don't point to invalid
-  // object on the other shard
-  std::list<Delta> expected_deltas;
-  expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 4, 1);
-  expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 5, 2);
-  expected_deltas.emplace_back(Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4);
-  expected_deltas.emplace_back(Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4);
-  expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 8, 3);
-  AssertEqDeltaLists(splitted_data.transactions.begin()->second->deltas, expected_deltas);
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) {
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
-                   .HasError());
-
-  auto current_hlc = GetNextHlc();
-  acc.Commit(current_hlc);
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 2);
-  EXPECT_EQ(splitted_data.transactions.size(), 1);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
-
-  CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
-  Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 12, 1};
-  Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 13, 1};
-  Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 14, 1};
-  Delta delta_add_in_edge1{Delta::RemoveInEdgeTag{},
-                           edge_type_id,
-                           VertexId{primary_label, {PropertyValue(1)}},
-                           EdgeRef{Gid::FromUint(1)},
-                           &commit_info,
-                           17,
-                           1};
-  Delta delta_add_out_edge2{Delta::RemoveOutEdgeTag{},
-                            edge_type_id,
-                            VertexId{primary_label, {PropertyValue(6)}},
-                            EdgeRef{Gid::FromUint(2)},
-                            &commit_info,
-                            19,
-                            1};
-  Delta delta_add_in_edge2{Delta::RemoveInEdgeTag{},
-                           edge_type_id,
-                           VertexId{primary_label, {PropertyValue(4)}},
-                           EdgeRef{Gid::FromUint(2)},
-                           &commit_info,
-                           20,
-                           1};
-  VertexContainer expected_vertices;
-  auto [vtx4, inserted4] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
-  auto [vtx5, inserted5] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
-  auto [vtx6, inserted6] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
-  AddDeltaToDeltaChain(&*vtx4, &delta_add_out_edge2);
-  AddDeltaToDeltaChain(&*vtx5, &delta_add_in_edge1);
-  AddDeltaToDeltaChain(&*vtx6, &delta_add_in_edge2);
-
-  AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitBeforeCommit) {
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
-                   .HasError());
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 2);
-  EXPECT_EQ(splitted_data.transactions.size(), 1);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitWithCommitedAndOngoingTransactions) {
-  {
-    auto acc = storage.Access(GetNextHlc());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-
-    acc.Commit(GetNextHlc());
-  }
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
-                   .HasError());
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 2);
-  EXPECT_EQ(splitted_data.transactions.size(), 2);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitWithLabelIndex) {
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {}).HasError());
-  acc.Commit(GetNextHlc());
-  storage.CreateIndex(secondary_label);
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 0);
-  EXPECT_EQ(splitted_data.transactions.size(), 1);
-  EXPECT_EQ(splitted_data.label_indices.size(), 1);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
-}
-
-TEST_F(ShardSplitTest, TestBasicSplitWithLabelPropertyIndex) {
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(
-      acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {{secondary_property, PropertyValue(1)}})
-          .HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(
-      acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(21)}})
-          .HasError());
-  EXPECT_FALSE(
-      acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {{secondary_property, PropertyValue(22)}})
-          .HasError());
-  acc.Commit(GetNextHlc());
-  storage.CreateIndex(secondary_label, secondary_property);
-
-  auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-
-  EXPECT_EQ(splitted_data.vertices.size(), 3);
-  EXPECT_EQ(splitted_data.edges->size(), 0);
-  EXPECT_EQ(splitted_data.transactions.size(), 1);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
-}
-
-TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroyOriginalShard) {
-  const auto split_value{4};
-  PrimaryKey splitted_value{{PropertyValue(4)}};
-  std::unique_ptr<Shard> splitted_shard;
-
-  {
-    Shard storage2{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, GetNextHlc()};
-    auto acc = storage2.Access(GetNextHlc());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-
-    EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                                VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
-                     .HasError());
-    EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
-                                VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
-                     .HasError());
-    EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
-                                VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
-                     .HasError());
-    acc.Commit(GetNextHlc());
-
-    auto splitted_data = storage2.PerformSplit({PropertyValue(split_value)}, last_hlc, GetNextHlc());
-    EXPECT_EQ(splitted_data->vertices.size(), 3);
-    EXPECT_EQ(splitted_data->edges->size(), 2);
-    EXPECT_EQ(splitted_data->transactions.size(), 1);
-    EXPECT_EQ(splitted_data->label_indices.size(), 0);
-    EXPECT_EQ(splitted_data->label_property_indices.size(), 0);
-
-    // Create a new shard
-    splitted_shard = Shard::FromSplitData(std::move(*splitted_data));
-    // Call gc on old shard
-    storage2.CollectGarbage(GetNextHlc().coordinator_wall_clock);
-    // Destroy original
-  }
-
-  splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
-  AssertShardState(*splitted_shard, 4, 6);
-}
-
-TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroySplittedShard) {
-  PrimaryKey splitted_value{{PropertyValue(4)}};
-
-  auto acc = storage.Access(GetNextHlc());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
-  EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
-
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
-                   .HasError());
-  EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
-                              VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
-                   .HasError());
-  acc.Commit(GetNextHlc());
-
-  auto splitted_data = storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
-  EXPECT_EQ(splitted_data->vertices.size(), 3);
-  EXPECT_EQ(splitted_data->edges->size(), 2);
-  EXPECT_EQ(splitted_data->transactions.size(), 1);
-  EXPECT_EQ(splitted_data->label_indices.size(), 0);
-  EXPECT_EQ(splitted_data->label_property_indices.size(), 0);
-
-  {
-    // Create a new shard
-    auto splitted_shard = Shard::FromSplitData(std::move(*splitted_data));
-    // Call gc on new shard
-    splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
-    // Destroy splitted shard
-  }
-
-  storage.CollectGarbage(GetNextHlc().coordinator_wall_clock);
-  AssertShardState(storage, 1, 3);
-}
-
-TEST_F(ShardSplitTest, TestBigSplit) {
-  int pk{0};
-  for (int64_t i{0}; i < 10'000; ++i) {
-    auto acc = storage.Access(GetNextHlc());
-    EXPECT_FALSE(
-        acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(pk++)}, {{secondary_property, PropertyValue(i)}})
-            .HasError());
-    EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(pk++)}, {}).HasError());
-
-    EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(pk - 2)}},
-                                VertexId{primary_label, PrimaryKey{PropertyValue(pk - 1)}}, edge_type_id,
-                                Gid::FromUint(pk))
-                     .HasError());
-    acc.Commit(GetNextHlc());
-  }
-  storage.CreateIndex(secondary_label, secondary_property);
-
-  const auto split_value = pk / 2;
-  auto splitted_data = *storage.PerformSplit({PropertyValue(split_value)}, last_hlc, GetNextHlc());
-
-  EXPECT_EQ(splitted_data.vertices.size(), 10000);
-  EXPECT_EQ(splitted_data.edges->size(), 5000);
-  EXPECT_EQ(splitted_data.transactions.size(), 5000);
-  EXPECT_EQ(splitted_data.label_indices.size(), 0);
-  EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
-
-  auto shard = Shard::FromSplitData(std::move(splitted_data));
-  AssertShardState(*shard, split_value, split_value * 2);
-}
-
-}  // namespace memgraph::storage::v3::tests

tests/unit/storage_v3_vertex_accessors.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -57,8 +57,9 @@ class StorageV3Accessor : public ::testing::Test {
   const PropertyId primary_property{PropertyId::FromUint(2)};
   std::vector<storage::v3::SchemaProperty> schema_property_vector = {
       storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
+  Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
+
   coordinator::Hlc last_hlc{0, io::Time{}};
-  Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
 };
 
 TEST_F(StorageV3Accessor, TestPrimaryLabel) {

tools/plot/.gitignore

@@ -0,0 +1,2 @@
+*.dat
+*.out

tools/plot/pool_size_batch_size_query_latency.gnuplot

@@ -0,0 +1,7 @@
+set dgrid3d 30,30
+set hidden3d
+set label "pool size"   at 8, 11000, 0
+set label "multiframe size"  at 20,6000,0
+set label "execution time (ms)"  at 14,0,5000
+splot "frames1.dat" u 1:2:3 with lines
+pause mouse close

tools/plot/thread_shard_scanall_latency.gnuplot

@@ -0,0 +1,6 @@
+set dgrid3d 30,30
+set hidden3d
+set label "shards"   at 20,-1,10000
+set label "threads"  at -10,5,10000
+splot "data.dat" u 1:2:3 with lines
+pause mouse close