Try to add simplest mgbench dataset (#746)

Cherry pick commit 11300960.

.github/workflows/diff.yaml

@@ -205,7 +205,7 @@ jobs:
 
           # Run simulation tests.
           cd tests/mgbench
-          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
+          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1" --no-authorization
 
   release_build:
     name: "Release build"
@@ -261,7 +261,7 @@ jobs:
 
           # Run simulation tests.
           cd tests/mgbench
-          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
+          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1" --no-authorization
 
       - name: Run e2e tests
         run: |

src/coordinator/shard_map.cpp

@@ -283,7 +283,7 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
       // TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
       bool machine_contains_shard = false;
 
-      for (auto &aas : shard) {
+      for (auto &aas : shard.peers) {
         if (initialized.contains(aas.address.unique_id)) {
           machine_contains_shard = true;
           if (aas.status != Status::CONSENSUS_PARTICIPANT) {
@@ -311,7 +311,7 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
         }
       }
 
-      if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
+      if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
         // increment version for each new uuid for deterministic creation
         IncrementShardMapVersion();
 
@@ -337,7 +337,7 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
             .status = Status::INITIALIZING,
         };
 
-        shard.emplace_back(aas);
+        shard.peers.emplace_back(aas);
       }
     }
   }
@@ -360,9 +360,9 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons
   MG_ASSERT(!shards_in_map.contains(key));
   MG_ASSERT(label_spaces.contains(label_id));
 
-  // Finding the Shard that the new PrimaryKey should map to.
+  // Finding the ShardMetadata that the new PrimaryKey should map to.
   auto prev = std::prev(shards_in_map.upper_bound(key));
-  Shard duplicated_shard = prev->second;
+  ShardMetadata duplicated_shard = prev->second;
 
   // Apply the split
   shards_in_map[key] = duplicated_shard;
@@ -383,7 +383,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
   labels.emplace(std::move(label_name), label_id);
 
   PrimaryKey initial_key = SchemaToMinKey(schema);
-  Shard empty_shard = {};
+  ShardMetadata empty_shard = {};
 
   Shards shards = {
       {initial_key, empty_shard},
@@ -479,7 +479,7 @@ Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey
   return shards;
 }
 
-Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
+ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
   MG_ASSERT(labels.contains(label_name));
 
   LabelId label_id = labels.at(label_name);
@@ -492,7 +492,7 @@ Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &ke
   return std::prev(label_space.shards.upper_bound(key))->second;
 }
 
-Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
+ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
   MG_ASSERT(label_spaces.contains(label_id));
 
   const auto &label_space = label_spaces.at(label_id);
@@ -556,12 +556,12 @@ EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new
 bool ShardMap::ClusterInitialized() const {
   for (const auto &[label_id, label_space] : label_spaces) {
     for (const auto &[low_key, shard] : label_space.shards) {
-      if (shard.size() < label_space.replication_factor) {
+      if (shard.peers.size() < label_space.replication_factor) {
         spdlog::info("label_space below desired replication factor");
         return false;
       }
 
-      for (const auto &aas : shard) {
+      for (const auto &aas : shard.peers) {
         if (aas.status != Status::CONSENSUS_PARTICIPANT) {
           spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
           return false;

src/coordinator/shard_map.hpp

@@ -76,8 +76,35 @@ struct AddressAndStatus {
 };
 
 using PrimaryKey = std::vector<PropertyValue>;
-using Shard = std::vector<AddressAndStatus>;
-using Shards = std::map<PrimaryKey, Shard>;
+
+struct ShardMetadata {
+  std::vector<AddressAndStatus> peers;
+  uint64_t version;
+
+  friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
+    using utils::print_helpers::operator<<;
+
+    in << "ShardMetadata { peers: ";
+    in << shard.peers;
+    in << " version: ";
+    in << shard.version;
+    in << " }";
+
+    return in;
+  }
+
+  friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
+
+  friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
+    if (lhs.peers != rhs.peers) {
+      return lhs.peers < rhs.peers;
+    }
+
+    return lhs.version < rhs.version;
+  }
+};
+
+using Shards = std::map<PrimaryKey, ShardMetadata>;
 using LabelName = std::string;
 using PropertyName = std::string;
 using EdgeTypeName = std::string;
@@ -99,7 +126,7 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
 struct LabelSpace {
   std::vector<SchemaProperty> schema;
   // Maps between the smallest primary key stored in the shard and the shard
-  std::map<PrimaryKey, Shard> shards;
+  std::map<PrimaryKey, ShardMetadata> shards;
   size_t replication_factor;
 
   friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
@@ -160,9 +187,9 @@ struct ShardMap {
 
   Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
 
-  Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
+  ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
 
-  Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
+  ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
 
   PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
 

src/expr/interpret/eval.hpp

@@ -36,8 +36,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
           typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
 class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
  public:
-  ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
-                      DbAccessor *dba, StorageView view)
+  ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
+                      StorageView view)
       : frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
 
   using ExpressionVisitor<TypedValue>::Visit;
@@ -782,7 +782,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
 
   LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
 
-  Frame<TypedValue> *frame_;
+  Frame *frame_;
   const SymbolTable *symbol_table_;
   const EvaluationContext *ctx_;
   DbAccessor *dba_;

src/expr/interpret/frame.hpp

@@ -20,7 +20,6 @@
 
 namespace memgraph::expr {
 
-template <typename TypedValue>
 class Frame {
  public:
   /// Create a Frame of given size backed by a utils::NewDeleteResource()
@@ -42,4 +41,18 @@ class Frame {
   utils::pmr::vector<TypedValue> elems_;
 };
 
+class FrameWithValidity final : public Frame {
+ public:
+  explicit FrameWithValidity(int64_t size) : Frame(size), is_valid_(false) {}
+
+  FrameWithValidity(int64_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
+
+  bool IsValid() const noexcept { return is_valid_; }
+  void MakeValid() noexcept { is_valid_ = true; }
+  void MakeInvalid() noexcept { is_valid_ = false; }
+
+ private:
+  bool is_valid_;
+};
+
 }  // namespace memgraph::expr

src/io/future.hpp

@@ -35,10 +35,13 @@ class Shared {
   std::optional<T> item_;
   bool consumed_ = false;
   bool waiting_ = false;
-  std::function<bool()> simulator_notifier_ = nullptr;
+  bool filled_ = false;
+  std::function<bool()> wait_notifier_ = nullptr;
+  std::function<void()> fill_notifier_ = nullptr;
 
  public:
-  explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
+  explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
+      : wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
   Shared() = default;
   Shared(Shared &&) = delete;
   Shared &operator=(Shared &&) = delete;
@@ -64,7 +67,7 @@ class Shared {
     waiting_ = true;
 
     while (!item_) {
-      if (simulator_notifier_) [[unlikely]] {
+      if (wait_notifier_) [[unlikely]] {
         // We can't hold our own lock while notifying
         // the simulator because notifying the simulator
         // involves acquiring the simulator's mutex
@@ -76,7 +79,7 @@ class Shared {
         // so we have to get out of its way to avoid
         // a cyclical deadlock.
         lock.unlock();
-        std::invoke(simulator_notifier_);
+        std::invoke(wait_notifier_);
         lock.lock();
         if (item_) {
           // item may have been filled while we
@@ -115,11 +118,19 @@ class Shared {
       std::unique_lock<std::mutex> lock(mu_);
 
       MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
-      MG_ASSERT(!item_, "Promise filled twice!");
+      MG_ASSERT(!filled_, "Promise filled twice!");
 
       item_ = item;
+      filled_ = true;
     }  // lock released before condition variable notification
 
+    if (fill_notifier_) {
+      spdlog::trace("calling fill notifier");
+      std::invoke(fill_notifier_);
+    } else {
+      spdlog::trace("not calling fill notifier");
+    }
+
     cv_.notify_all();
   }
 
@@ -251,8 +262,9 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
 }
 
 template <typename T>
-std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
-  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
+std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
+                                                                    std::function<void()> fill_notifier) {
+  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
 
   Future<T> future = Future<T>(shared);
   Promise<T> promise = Promise<T>(shared);

src/io/local_transport/local_transport.hpp

@@ -31,9 +31,10 @@ class LocalTransport {
       : local_transport_handle_(std::move(local_transport_handle)) {}
 
   template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
-    return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address,
-                                                                                std::move(request), timeout);
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
+                                    std::function<void()> fill_notifier, Duration timeout) {
+    return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
+        to_address, from_address, std::move(request), timeout, fill_notifier);
   }
 
   template <Message... Ms>

src/io/local_transport/local_transport_handle.hpp

@@ -140,8 +140,12 @@ class LocalTransportHandle {
 
   template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
-                                          Duration timeout) {
-    auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
+                                          Duration timeout, std::function<void()> fill_notifier) {
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
+        // set null notifier for when the Future::Wait is called
+        nullptr,
+        // set notifier for when Promise::Fill is called
+        std::forward<std::function<void()>>(fill_notifier));
 
     const bool port_matches = to_address.last_known_port == from_address.last_known_port;
     const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;

src/io/notifier.hpp

@@ -0,0 +1,93 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <optional>
+#include <vector>
+
+namespace memgraph::io {
+
+class ReadinessToken {
+  size_t id_;
+
+ public:
+  explicit ReadinessToken(size_t id) : id_(id) {}
+  size_t GetId() const { return id_; }
+};
+
+class Inner {
+  std::condition_variable cv_;
+  std::mutex mu_;
+  std::vector<ReadinessToken> ready_;
+  std::optional<std::function<bool()>> tick_simulator_;
+
+ public:
+  void Notify(ReadinessToken readiness_token) {
+    {
+      std::unique_lock<std::mutex> lock(mu_);
+      ready_.emplace_back(readiness_token);
+    }  // mutex dropped
+
+    cv_.notify_all();
+  }
+
+  ReadinessToken Await() {
+    std::unique_lock<std::mutex> lock(mu_);
+
+    while (ready_.empty()) {
+      if (tick_simulator_) [[unlikely]] {
+        // This avoids a deadlock in a similar way that
+        // Future::Wait will release its mutex while
+        // interacting with the simulator, due to
+        // the fact that the simulator may cause
+        // notifications that we are interested in.
+        lock.unlock();
+        std::invoke(tick_simulator_.value());
+        lock.lock();
+      } else {
+        cv_.wait(lock);
+      }
+    }
+
+    ReadinessToken ret = ready_.back();
+    ready_.pop_back();
+    return ret;
+  }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
+    std::unique_lock<std::mutex> lock(mu_);
+    tick_simulator_ = tick_simulator;
+  }
+};
+
+class Notifier {
+  std::shared_ptr<Inner> inner_;
+
+ public:
+  Notifier() : inner_(std::make_shared<Inner>()) {}
+  Notifier(const Notifier &) = default;
+  Notifier &operator=(const Notifier &) = default;
+  Notifier(Notifier &&old) = default;
+  Notifier &operator=(Notifier &&old) = default;
+  ~Notifier() = default;
+
+  void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
+
+  ReadinessToken Await() const { return inner_->Await(); }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
+};
+
+}  // namespace memgraph::io

src/io/rsm/raft.hpp

@@ -91,33 +91,43 @@ struct ReadResponse {
 };
 
 template <class... ReadReturn>
-utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &read_response) {
-  return TypeInfoForVariant(read_response.read_return);
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
+  return TypeInfoForVariant(response.read_return);
 }
 
 template <class ReadReturn>
-utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* read_response */) {
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
   return typeid(ReadReturn);
 }
 
+template <class ReadOperation>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
+  return typeid(ReadOperation);
+}
+
+template <class... ReadOperations>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
+  return TypeInfoForVariant(request.operation);
+}
+
 template <class... WriteReturn>
-utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &write_response) {
-  return TypeInfoForVariant(write_response.write_return);
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
+  return TypeInfoForVariant(response.write_return);
 }
 
 template <class WriteReturn>
-utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* write_response */) {
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
   return typeid(WriteReturn);
 }
 
 template <class WriteOperation>
-utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* write_request */) {
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
   return typeid(WriteOperation);
 }
 
 template <class... WriteOperations>
-utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &write_request) {
-  return TypeInfoForVariant(write_request.operation);
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
+  return TypeInfoForVariant(request.operation);
 }
 
 /// AppendRequest is a raft-level message that the Leader
@@ -846,7 +856,9 @@ class Raft {
   // Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
   std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
                              Address from_address) {
-    Log("handling ReadOperation");
+    auto type_info = TypeInfoFor(req);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    Log("handling ReadOperation<" + demangled_name + ">");
     ReadOperation read_operation = req.operation;
 
     ReadResponseValue read_return = replicated_state_.Read(read_operation);

src/io/rsm/rsm_client.hpp

@@ -19,6 +19,7 @@
 
 #include "io/address.hpp"
 #include "io/errors.hpp"
+#include "io/notifier.hpp"
 #include "io/rsm/raft.hpp"
 #include "utils/result.hpp"
 
@@ -37,18 +38,11 @@ using memgraph::io::rsm::WriteRequest;
 using memgraph::io::rsm::WriteResponse;
 using memgraph::utils::BasicResult;
 
-class AsyncRequestToken {
-  size_t id_;
-
- public:
-  explicit AsyncRequestToken(size_t id) : id_(id) {}
-  size_t GetId() const { return id_; }
-};
-
 template <typename RequestT, typename ResponseT>
 struct AsyncRequest {
   Time start_time;
   RequestT request;
+  Notifier notifier;
   ResponseFuture<ResponseT> future;
 };
 
@@ -66,8 +60,6 @@ class RsmClient {
   std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
   std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
 
-  size_t async_token_generator_ = 0;
-
   void SelectRandomLeader() {
     std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
     size_t addr_index = io_.Rand(addr_distrib);
@@ -101,61 +93,63 @@ class RsmClient {
   ~RsmClient() = default;
 
   BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
-    auto token = SendAsyncWriteRequest(req);
-    auto poll_result = AwaitAsyncWriteRequest(token);
+    Notifier notifier;
+    const ReadinessToken readiness_token{0};
+    SendAsyncWriteRequest(req, notifier, readiness_token);
+    auto poll_result = AwaitAsyncWriteRequest(readiness_token);
     while (!poll_result) {
-      poll_result = AwaitAsyncWriteRequest(token);
+      poll_result = AwaitAsyncWriteRequest(readiness_token);
     }
     return poll_result.value();
   }
 
   BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
-    auto token = SendAsyncReadRequest(req);
-    auto poll_result = AwaitAsyncReadRequest(token);
+    Notifier notifier;
+    const ReadinessToken readiness_token{0};
+    SendAsyncReadRequest(req, notifier, readiness_token);
+    auto poll_result = AwaitAsyncReadRequest(readiness_token);
     while (!poll_result) {
-      poll_result = AwaitAsyncReadRequest(token);
+      poll_result = AwaitAsyncReadRequest(readiness_token);
     }
     return poll_result.value();
   }
 
   /// AsyncRead methods
-  AsyncRequestToken SendAsyncReadRequest(const ReadRequestT &req) {
-    size_t token = async_token_generator_++;
-
+  void SendAsyncReadRequest(const ReadRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
     ReadRequest<ReadRequestT> read_req = {.operation = req};
 
     AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
         .start_time = io_.Now(),
         .request = std::move(req),
-        .future = io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req),
+        .notifier = notifier,
+        .future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
+            leader_, read_req, notifier, readiness_token),
     };
 
-    async_reads_.emplace(token, std::move(async_request));
-
-    return AsyncRequestToken{token};
+    async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
   }
 
-  void ResendAsyncReadRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_reads_.at(token.GetId());
+  void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
 
     ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
 
-    async_request.future =
-        io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
+    async_request.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
+        leader_, read_req, async_request.notifier, readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_reads_.at(token.GetId());
+  std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
 
     if (!async_request.future.IsReady()) {
       return std::nullopt;
     }
 
-    return AwaitAsyncReadRequest();
+    return AwaitAsyncReadRequest(readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_reads_.at(token.GetId());
+  std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
     ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
 
     const Duration overall_timeout = io_.GetDefaultTimeout();
@@ -165,7 +159,7 @@ class RsmClient {
     if (result_has_error && past_time_out) {
       // TODO static assert the exact type of error.
       spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-      async_reads_.erase(token.GetId());
+      async_reads_.erase(readiness_token.GetId());
       return TimedOut{};
     }
 
@@ -176,7 +170,7 @@ class RsmClient {
       PossiblyRedirectLeader(read_get_response);
 
       if (read_get_response.success) {
-        async_reads_.erase(token.GetId());
+        async_reads_.erase(readiness_token.GetId());
         spdlog::debug("returning read_return for RSM request");
         return std::move(read_get_response.read_return);
       }
@@ -184,49 +178,48 @@ class RsmClient {
       SelectRandomLeader();
     }
 
-    ResendAsyncReadRequest(token);
+    ResendAsyncReadRequest(readiness_token);
 
     return std::nullopt;
   }
 
   /// AsyncWrite methods
-  AsyncRequestToken SendAsyncWriteRequest(const WriteRequestT &req) {
-    size_t token = async_token_generator_++;
-
+  void SendAsyncWriteRequest(const WriteRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
     WriteRequest<WriteRequestT> write_req = {.operation = req};
 
     AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
         .start_time = io_.Now(),
         .request = std::move(req),
-        .future = io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, write_req),
+        .notifier = notifier,
+        .future = io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
+            leader_, write_req, notifier, readiness_token),
     };
 
-    async_writes_.emplace(token, std::move(async_request));
-
-    return AsyncRequestToken{token};
+    async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
   }
 
-  void ResendAsyncWriteRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_writes_.at(token.GetId());
+  void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
 
     WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
 
     async_request.future =
-        io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, write_req);
+        io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
+            leader_, write_req, async_request.notifier, readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_writes_.at(token.GetId());
+  std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
 
     if (!async_request.future.IsReady()) {
       return std::nullopt;
     }
 
-    return AwaitAsyncWriteRequest();
+    return AwaitAsyncWriteRequest(readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const AsyncRequestToken &token) {
-    auto &async_request = async_writes_.at(token.GetId());
+  std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
     ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
 
     const Duration overall_timeout = io_.GetDefaultTimeout();
@@ -236,7 +229,7 @@ class RsmClient {
     if (result_has_error && past_time_out) {
       // TODO static assert the exact type of error.
       spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-      async_writes_.erase(token.GetId());
+      async_writes_.erase(readiness_token.GetId());
       return TimedOut{};
     }
 
@@ -248,14 +241,14 @@ class RsmClient {
       PossiblyRedirectLeader(write_get_response);
 
       if (write_get_response.success) {
-        async_writes_.erase(token.GetId());
+        async_writes_.erase(readiness_token.GetId());
         return std::move(write_get_response.write_return);
       }
     } else {
       SelectRandomLeader();
     }
 
-    ResendAsyncWriteRequest(token);
+    ResendAsyncWriteRequest(readiness_token);
 
     return std::nullopt;
   }

src/io/simulator/simulator.hpp

@@ -49,5 +49,10 @@ class Simulator {
   }
 
   SimulatorStats Stats() { return simulator_handle_->Stats(); }
+
+  std::function<bool()> GetSimulatorTickClosure() {
+    std::function<bool()> tick_closure = [handle_copy = simulator_handle_] { return handle_copy->MaybeTickSimulator(); };
+    return tick_closure;
+  }
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_handle.hpp

@@ -22,6 +22,8 @@
 #include <variant>
 #include <vector>
 
+#include <boost/core/demangle.hpp>
+
 #include "io/address.hpp"
 #include "io/errors.hpp"
 #include "io/message_conversion.hpp"
@@ -105,41 +107,48 @@ class SimulatorHandle {
 
   template <Message Request, Message Response>
   ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
-                                         std::function<bool()> &&maybe_tick_simulator) {
-    spdlog::trace("submitting request to {}", to_address.last_known_port);
+                                         std::function<bool()> &&maybe_tick_simulator,
+                                         std::function<void()> &&fill_notifier) {
     auto type_info = TypeInfoFor(request);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
 
-    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(
-        std::forward<std::function<bool()>>(maybe_tick_simulator));
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<Response>>(
+        // set notifier for when the Future::Wait is called
+        std::forward<std::function<bool()>>(maybe_tick_simulator),
+        // set notifier for when Promise::Fill is called
+        std::forward<std::function<void()>>(fill_notifier));
 
-    std::unique_lock<std::mutex> lock(mu_);
+    {
+      std::unique_lock<std::mutex> lock(mu_);
 
-    RequestId request_id = ++request_id_counter_;
+      RequestId request_id = ++request_id_counter_;
 
-    const Time deadline = cluster_wide_time_microseconds_ + timeout;
+      const Time deadline = cluster_wide_time_microseconds_ + timeout;
 
-    std::any message(request);
-    OpaqueMessage om{.to_address = to_address,
-                     .from_address = from_address,
-                     .request_id = request_id,
-                     .message = std::move(message),
-                     .type_info = type_info};
-    in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
+      std::any message(request);
+      OpaqueMessage om{.to_address = to_address,
+                       .from_address = from_address,
+                       .request_id = request_id,
+                       .message = std::move(message),
+                       .type_info = type_info};
+      in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
 
-    PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
-    OpaquePromise opaque_promise(std::move(promise).ToUnique());
-    DeadlineAndOpaquePromise dop{
-        .requested_at = cluster_wide_time_microseconds_,
-        .deadline = deadline,
-        .promise = std::move(opaque_promise),
-    };
+      PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
+      OpaquePromise opaque_promise(std::move(promise).ToUnique());
+      DeadlineAndOpaquePromise dop{
+          .requested_at = cluster_wide_time_microseconds_,
+          .deadline = deadline,
+          .promise = std::move(opaque_promise),
+      };
 
-    MG_ASSERT(!promises_.contains(promise_key));
+      MG_ASSERT(!promises_.contains(promise_key));
 
-    promises_.emplace(std::move(promise_key), std::move(dop));
+      promises_.emplace(std::move(promise_key), std::move(dop));
 
-    stats_.total_messages++;
-    stats_.total_requests++;
+      stats_.total_messages++;
+      stats_.total_requests++;
+    }  // lock dropped here
 
     cv_.notify_all();
 

src/io/simulator/simulator_transport.hpp

@@ -15,6 +15,7 @@
 #include <utility>
 
 #include "io/address.hpp"
+#include "io/notifier.hpp"
 #include "io/simulator/simulator_handle.hpp"
 #include "io/time.hpp"
 
@@ -33,11 +34,12 @@ class SimulatorTransport {
       : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
 
   template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
-    std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
+                                    std::function<void()> notification, Duration timeout) {
+    std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] { return handle_copy->MaybeTickSimulator(); };
 
-    return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address, std::move(request),
-                                                                          timeout, std::move(maybe_tick_simulator));
+    return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(
+        to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
   }
 
   template <Message... Ms>

src/io/transport.hpp

@@ -20,6 +20,7 @@
 #include "io/errors.hpp"
 #include "io/future.hpp"
 #include "io/message_histogram_collector.hpp"
+#include "io/notifier.hpp"
 #include "io/time.hpp"
 #include "utils/result.hpp"
 
@@ -84,7 +85,9 @@ class Io {
   template <Message RequestT, Message ResponseT>
   ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
     const Address from_address = address_;
-    return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, timeout);
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, fill_notifier,
+                                                                 timeout);
   }
 
   /// Issue a request that times out after the default timeout. This tends
@@ -93,7 +96,30 @@ class Io {
   ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
     const Duration timeout = default_timeout_;
     const Address from_address = address_;
-    return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request), timeout);
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message RequestT, Message ResponseT>
+  ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
+                                                    ReadinessToken readiness_token) {
+    const Duration timeout = default_timeout_;
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message RequestT, Message ResponseT>
+  ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
+                                                              ReadinessToken readiness_token, Duration timeout) {
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
   }
 
   /// Wait for an explicit number of microseconds for a request of one of the

src/query/v2/CMakeLists.txt

@@ -23,7 +23,8 @@ set(mg_query_v2_sources
     plan/variable_start_planner.cpp
     serialization/property_value.cpp
     bindings/typed_value.cpp
-    accessors.cpp)
+    accessors.cpp
+    multiframe.cpp)
 
 find_package(Boost REQUIRED)
 

src/query/v2/bindings/frame.hpp

@@ -13,9 +13,10 @@
 
 #include "query/v2/bindings/bindings.hpp"
 
-#include "query/v2/bindings/typed_value.hpp"
 #include "expr/interpret/frame.hpp"
+#include "query/v2/bindings/typed_value.hpp"
 
 namespace memgraph::query::v2 {
-using Frame = memgraph::expr::Frame<TypedValue>;
-} // namespace memgraph::query::v2
+using Frame = memgraph::expr::Frame;
+using FrameWithValidity = memgraph::expr::FrameWithValidity;
+}  // namespace memgraph::query::v2

src/query/v2/interpreter.cpp

@@ -41,6 +41,7 @@
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/frontend/semantic/required_privileges.hpp"
 #include "query/v2/metadata.hpp"
+#include "query/v2/multiframe.hpp"
 #include "query/v2/plan/planner.hpp"
 #include "query/v2/plan/profile.hpp"
 #include "query/v2/plan/vertex_count_cache.hpp"
@@ -147,7 +148,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
   // Empty frame for evaluation of password expression. This is OK since
   // password should be either null or string literal and it's evaluation
   // should not depend on frame.
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -314,7 +315,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
 Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters,
                                 InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
                                 std::vector<Notification> *notifications) {
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -449,7 +450,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
 
 Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &parameters,
                             RequestRouterInterface *request_router) {
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -655,11 +656,15 @@ struct PullPlan {
   std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
                                                         const std::vector<Symbol> &output_symbols,
                                                         std::map<std::string, TypedValue> *summary);
+  std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
+                                                                const std::vector<Symbol> &output_symbols,
+                                                                std::map<std::string, TypedValue> *summary);
 
  private:
   std::shared_ptr<CachedPlan> plan_ = nullptr;
   plan::UniqueCursorPtr cursor_ = nullptr;
-  expr::Frame<TypedValue> frame_;
+  expr::FrameWithValidity frame_;
+  MultiFrame multi_frame_;
   ExecutionContext ctx_;
   std::optional<size_t> memory_limit_;
 
@@ -683,6 +688,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
     : plan_(plan),
       cursor_(plan->plan().MakeCursor(execution_memory)),
       frame_(plan->symbol_table().max_position(), execution_memory),
+      multi_frame_(plan->symbol_table().max_position(), kNumberOfFramesInMultiframe, execution_memory),
       memory_limit_(memory_limit) {
   ctx_.db_accessor = dba;
   ctx_.symbol_table = plan->symbol_table();
@@ -699,9 +705,116 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
   ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
 }
 
+std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
+                                                                        const std::vector<Symbol> &output_symbols,
+                                                                        std::map<std::string, TypedValue> *summary) {
+  // Set up temporary memory for a single Pull. Initial memory comes from the
+  // stack. 256 KiB should fit on the stack and should be more than enough for a
+  // single `Pull`.
+  MG_ASSERT(!n.has_value(), "should pull all!");
+  static constexpr size_t stack_size = 256UL * 1024UL;
+  char stack_data[stack_size];
+  utils::ResourceWithOutOfMemoryException resource_with_exception;
+  utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
+  // We can throw on every query because a simple queries for deleting will use only
+  // the stack allocated buffer.
+  // Also, we want to throw only when the query engine requests more memory and not the storage
+  // so we add the exception to the allocator.
+  // TODO (mferencevic): Tune the parameters accordingly.
+  utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
+  std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
+
+  if (memory_limit_) {
+    maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
+    ctx_.evaluation_context.memory = &*maybe_limited_resource;
+  } else {
+    ctx_.evaluation_context.memory = &pool_memory;
+  }
+
+  // Returns true if a result was pulled.
+  const auto pull_result = [&]() -> bool {
+    cursor_->PullMultiple(multi_frame_, ctx_);
+    return multi_frame_.HasValidFrame();
+  };
+
+  const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
+    // TODO: The streamed values should also probably use the above memory.
+    std::vector<TypedValue> values;
+    values.reserve(output_symbols.size());
+
+    for (const auto &symbol : output_symbols) {
+      values.emplace_back(frame[symbol]);
+    }
+
+    stream->Result(values);
+  };
+
+  // Get the execution time of all possible result pulls and streams.
+  utils::Timer timer;
+
+  int i = 0;
+  if (has_unsent_results_ && !output_symbols.empty()) {
+    // stream unsent results from previous pull
+
+    auto iterator_for_valid_frame_only = multi_frame_.GetValidFramesReader();
+    for (const auto &frame : iterator_for_valid_frame_only) {
+      stream_values(frame);
+      ++i;
+    }
+    multi_frame_.MakeAllFramesInvalid();
+  }
+
+  for (; !n || i < n;) {
+    if (!pull_result()) {
+      break;
+    }
+
+    if (!output_symbols.empty()) {
+      auto iterator_for_valid_frame_only = multi_frame_.GetValidFramesReader();
+      for (const auto &frame : iterator_for_valid_frame_only) {
+        stream_values(frame);
+        ++i;
+      }
+    }
+    multi_frame_.MakeAllFramesInvalid();
+  }
+
+  // If we finished because we streamed the requested n results,
+  // we try to pull the next result to see if there is more.
+  // If there is additional result, we leave the pulled result in the frame
+  // and set the flag to true.
+  has_unsent_results_ = i == n && pull_result();
+
+  execution_time_ += timer.Elapsed();
+
+  if (has_unsent_results_) {
+    return std::nullopt;
+  }
+  summary->insert_or_assign("plan_execution_time", execution_time_.count());
+  // We are finished with pulling all the data, therefore we can send any
+  // metadata about the results i.e. notifications and statistics
+  const bool is_any_counter_set =
+      std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
+                  [](const auto &counter) { return counter > 0; });
+  if (is_any_counter_set) {
+    std::map<std::string, TypedValue> stats;
+    for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
+      stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
+    }
+    summary->insert_or_assign("stats", std::move(stats));
+  }
+  cursor_->Shutdown();
+  ctx_.profile_execution_time = execution_time_;
+  return GetStatsWithTotalTime(ctx_);
+}
+
 std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
                                                                 const std::vector<Symbol> &output_symbols,
                                                                 std::map<std::string, TypedValue> *summary) {
+  auto should_pull_multiple = false;  // TODO on the long term, we will only use PullMultiple
+  if (should_pull_multiple) {
+    return PullMultiple(stream, n, output_symbols, summary);
+  }
   // Set up temporary memory for a single Pull. Initial memory comes from the
   // stack. 256 KiB should fit on the stack and should be more than enough for a
   // single `Pull`.
@@ -885,7 +998,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
   //                                 TriggerContextCollector *trigger_context_collector = nullptr) {
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
 
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   evaluation_context.timestamp = QueryTimestamp();
@@ -1027,7 +1140,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
 
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
   MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   evaluation_context.timestamp = QueryTimestamp();

src/query/v2/multiframe.cpp

@@ -0,0 +1,144 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/v2/multiframe.hpp"
+
+#include <algorithm>
+#include <iterator>
+
+#include "query/v2/bindings/frame.hpp"
+#include "utils/pmr/vector.hpp"
+
+namespace memgraph::query::v2 {
+
+static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
+static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
+static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
+static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
+
+MultiFrame::MultiFrame(int64_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
+    : frames_(utils::pmr::vector<FrameWithValidity>(
+          number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
+  MG_ASSERT(number_of_frames > 0);
+}
+
+MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
+
+FrameWithValidity &MultiFrame::GetFirstFrame() {
+  MG_ASSERT(!frames_.empty());
+  return frames_.front();
+}
+
+void MultiFrame::MakeAllFramesInvalid() noexcept {
+  std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
+}
+
+bool MultiFrame::HasValidFrame() const noexcept {
+  return std::any_of(frames_.begin(), frames_.end(), [](auto &frame) { return frame.IsValid(); });
+}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+void MultiFrame::DefragmentValidFrames() noexcept {
+  /*
+  from: https://en.cppreference.com/w/cpp/algorithm/remove
+  "Removing is done by shifting (by means of copy assignment (until C++11)move assignment (since C++11)) the elements
+  in the range in such a way that the elements that are not to be removed appear in the beginning of the range.
+  Relative order of the elements that remain is preserved and the physical size of the container is unchanged."
+  */
+
+  // NOLINTNEXTLINE (bugprone-unused-return-value)
+  std::remove_if(frames_.begin(), frames_.end(), [](auto &frame) { return !frame.IsValid(); });
+}
+
+ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
+
+ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
+
+ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
+
+InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
+
+ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
+  /*
+  From: https://en.cppreference.com/w/cpp/algorithm/find
+  Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
+  find_if searches for an element for which predicate p returns true
+  Return value
+    Iterator to the first element satisfying the condition or last if no such element is found.
+
+  -> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
+  */
+  auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
+                         [](const auto &frame) { return !frame.IsValid(); });
+  after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
+}
+
+ValidFramesReader::Iterator ValidFramesReader::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0]};
+  }
+  return end();
+}
+
+ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
+
+ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+ValidFramesModifier::Iterator ValidFramesModifier::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesModifier::Iterator ValidFramesModifier::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+ValidFramesConsumer::~ValidFramesConsumer() noexcept {
+  // TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
+  // cost too much to store it
+  multiframe_->DefragmentValidFrames();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
+  for (auto &frame : multiframe_->frames_) {
+    if (!frame.IsValid()) {
+      return Iterator{&frame};
+    }
+  }
+  return end();
+}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
+}
+
+}  // namespace memgraph::query::v2

src/query/v2/multiframe.hpp

@@ -0,0 +1,302 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <iterator>
+
+#include "query/v2/bindings/frame.hpp"
+
+namespace memgraph::query::v2 {
+constexpr uint64_t kNumberOfFramesInMultiframe = 1000;  // TODO have it configurable
+
+class ValidFramesConsumer;
+class ValidFramesModifier;
+class ValidFramesReader;
+class InvalidFramesPopulator;
+
+class MultiFrame {
+ public:
+  friend class ValidFramesConsumer;
+  friend class ValidFramesModifier;
+  friend class ValidFramesReader;
+  friend class InvalidFramesPopulator;
+
+  MultiFrame(int64_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
+  ~MultiFrame() = default;
+
+  MultiFrame(const MultiFrame &other);
+  MultiFrame(MultiFrame &&other) noexcept;
+  MultiFrame &operator=(const MultiFrame &other) = delete;
+  MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame nor its content with this implementation.
+   */
+  ValidFramesReader GetValidFramesReader();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame with this implementation. One can modify its content.
+   */
+  ValidFramesModifier GetValidFramesModifier();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can modify the validity of the Frame with this implementation.
+   * If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
+   * this is faster.
+   */
+  ValidFramesConsumer GetValidFramesConsumer();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
+   * invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
+   * the Frame with this implementation.
+   */
+  InvalidFramesPopulator GetInvalidFramesPopulator();
+
+  /**
+   * Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
+   * using the iterators instead.
+   * The Frame can be valid or invalid.
+   */
+  FrameWithValidity &GetFirstFrame();
+
+  void MakeAllFramesInvalid() noexcept;
+
+  bool HasValidFrame() const noexcept;
+
+  inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
+
+ private:
+  void DefragmentValidFrames() noexcept;
+
+  utils::pmr::vector<FrameWithValidity> frames_;
+};
+
+class ValidFramesReader {
+ public:
+  explicit ValidFramesReader(MultiFrame &multiframe);
+
+  ~ValidFramesReader() = default;
+  ValidFramesReader(const ValidFramesReader &other) = delete;
+  ValidFramesReader(ValidFramesReader &&other) noexcept = delete;
+  ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
+  ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = delete;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = const Frame;
+    using pointer = value_type *;
+    using reference = const Frame &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ptr_++;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  FrameWithValidity *after_last_valid_frame_;
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesModifier {
+ public:
+  explicit ValidFramesModifier(MultiFrame &multiframe);
+
+  ~ValidFramesModifier() = default;
+  ValidFramesModifier(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier(ValidFramesModifier &&other) noexcept = delete;
+  ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = delete;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = Frame;
+    using pointer = value_type *;
+    using reference = Frame &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    // Prefix increment
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesModifier *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesConsumer {
+ public:
+  explicit ValidFramesConsumer(MultiFrame &multiframe);
+
+  ~ValidFramesConsumer() noexcept;
+  ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = delete;
+  ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = delete;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesConsumer *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class InvalidFramesPopulator {
+ public:
+  explicit InvalidFramesPopulator(MultiFrame &multiframe);
+  ~InvalidFramesPopulator() = default;
+
+  InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = delete;
+  InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = delete;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_->MakeValid();
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++ptr_;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+}  // namespace memgraph::query::v2

src/query/v2/plan/operator.cpp

@@ -264,6 +264,16 @@ bool Once::OnceCursor::Pull(Frame &, ExecutionContext &context) {
   return false;
 }
 
+void Once::OnceCursor::PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) {
+  SCOPED_PROFILE_OP("OnceMF");
+
+  if (!did_pull_) {
+    auto &first_frame = multi_frame.GetFirstFrame();
+    first_frame.MakeValid();
+    did_pull_ = true;
+  }
+}
+
 UniqueCursorPtr Once::MakeCursor(utils::MemoryResource *mem) const {
   EventCounter::IncrementCounter(EventCounter::OnceOperator);
 
@@ -748,6 +758,23 @@ bool Produce::ProduceCursor::Pull(Frame &frame, ExecutionContext &context) {
   return false;
 }
 
+void Produce::ProduceCursor::PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) {
+  SCOPED_PROFILE_OP("ProduceMF");
+
+  input_cursor_->PullMultiple(multi_frame, context);
+
+  auto iterator_for_valid_frame_only = multi_frame.GetValidFramesModifier();
+  for (auto &frame : iterator_for_valid_frame_only) {
+    // Produce should always yield the latest results.
+    ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.request_router,
+                                  storage::v3::View::NEW);
+
+    for (auto *named_expr : self_.named_expressions_) {
+      named_expr->Accept(evaluator);
+    }
+  }
+};
+
 void Produce::ProduceCursor::Shutdown() { input_cursor_->Shutdown(); }
 
 void Produce::ProduceCursor::Reset() { input_cursor_->Reset(); }

src/query/v2/plan/operator.lcp

@@ -28,6 +28,7 @@
 #include "query/v2/bindings/typed_value.hpp"
 #include "query/v2/bindings/frame.hpp"
 #include "query/v2/bindings/symbol_table.hpp"
+#include "query/v2/multiframe.hpp"
 #include "storage/v3/id_types.hpp"
 #include "utils/bound.hpp"
 #include "utils/fnv.hpp"
@@ -71,6 +72,8 @@ class Cursor {
   /// @throws QueryRuntimeException if something went wrong with execution
   virtual bool Pull(Frame &, ExecutionContext &) = 0;
 
+  virtual void PullMultiple(MultiFrame &, ExecutionContext &) { LOG_FATAL("PullMultipleIsNotImplemented"); }
+
   /// Resets the Cursor to its initial state.
   virtual void Reset() = 0;
 
@@ -332,6 +335,7 @@ and false on every following Pull.")
    class OnceCursor : public Cursor {
     public:
      OnceCursor() {}
+     void PullMultiple(MultiFrame &, ExecutionContext &) override;
      bool Pull(Frame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
@@ -1207,6 +1211,7 @@ RETURN clause) the Produce's pull succeeds exactly once.")
     public:
      ProduceCursor(const Produce &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     void PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 

src/query/v2/request_router.hpp

@@ -11,6 +11,7 @@
 
 #pragma once
 
+#include <algorithm>
 #include <chrono>
 #include <deque>
 #include <iostream>
@@ -31,6 +32,7 @@
 #include "coordinator/shard_map.hpp"
 #include "io/address.hpp"
 #include "io/errors.hpp"
+#include "io/notifier.hpp"
 #include "io/rsm/raft.hpp"
 #include "io/rsm/rsm_client.hpp"
 #include "io/rsm/shard_rsm.hpp"
@@ -43,11 +45,12 @@
 #include "utils/result.hpp"
 
 namespace memgraph::query::v2 {
+
 template <typename TStorageClient>
 class RsmStorageClientManager {
  public:
   using CompoundKey = io::rsm::ShardRsmKey;
-  using Shard = coordinator::Shard;
+  using ShardMetadata = coordinator::ShardMetadata;
   RsmStorageClientManager() = default;
   RsmStorageClientManager(const RsmStorageClientManager &) = delete;
   RsmStorageClientManager(RsmStorageClientManager &&) = delete;
@@ -55,45 +58,31 @@ class RsmStorageClientManager {
   RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
   ~RsmStorageClientManager() = default;
 
-  void AddClient(Shard key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
+  void AddClient(ShardMetadata key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
 
-  bool Exists(const Shard &key) { return cli_cache_.contains(key); }
+  bool Exists(const ShardMetadata &key) { return cli_cache_.contains(key); }
 
   void PurgeCache() { cli_cache_.clear(); }
 
-  TStorageClient &GetClient(const Shard &key) {
+  TStorageClient &GetClient(const ShardMetadata &key) {
     auto it = cli_cache_.find(key);
     MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
     return it->second;
   }
 
  private:
-  std::map<Shard, TStorageClient> cli_cache_;
+  std::map<ShardMetadata, TStorageClient> cli_cache_;
 };
 
 template <typename TRequest>
 struct ShardRequestState {
-  memgraph::coordinator::Shard shard;
+  memgraph::coordinator::ShardMetadata shard;
   TRequest request;
-  std::optional<io::rsm::AsyncRequestToken> async_request_token;
 };
 
+// maps from ReadinessToken's internal size_t to the associated state
 template <typename TRequest>
-struct ExecutionState {
-  using CompoundKey = io::rsm::ShardRsmKey;
-  using Shard = coordinator::Shard;
-
-  // label is optional because some operators can create/remove etc, vertices. These kind of requests contain the label
-  // on the request itself.
-  std::optional<std::string> label;
-  // Transaction id to be filled by the RequestRouter implementation
-  coordinator::Hlc transaction_id;
-  // Initialized by RequestRouter implementation. This vector is filled with the shards that
-  // the RequestRouter impl will send requests to. When a request to a shard exhausts it, meaning that
-  // it pulled all the requested data from the given Shard, it will be removed from the Vector. When the Vector becomes
-  // empty, it means that all of the requests have completed succefully.
-  std::vector<ShardRequestState<TRequest>> requests;
-};
+using RunningRequests = std::unordered_map<size_t, ShardRequestState<TRequest>>;
 
 class RequestRouterInterface {
  public:
@@ -112,6 +101,7 @@ class RequestRouterInterface {
   virtual std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) = 0;
   virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
   virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
+  virtual std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest request) = 0;
 
   virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
   virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
@@ -135,7 +125,7 @@ class RequestRouter : public RequestRouterInterface {
   using CoordinatorWriteRequests = coordinator::CoordinatorWriteRequests;
   using CoordinatorClient = coordinator::CoordinatorClient<TTransport>;
   using Address = io::Address;
-  using Shard = coordinator::Shard;
+  using ShardMetadata = coordinator::ShardMetadata;
   using ShardMap = coordinator::ShardMap;
   using CompoundKey = coordinator::PrimaryKey;
   using VertexAccessor = query::v2::accessors::VertexAccessor;
@@ -148,10 +138,16 @@ class RequestRouter : public RequestRouterInterface {
 
   ~RequestRouter() override {}
 
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
+    notifier_.InstallSimulatorTicker(tick_simulator);
+  }
+
   void StartTransaction() override {
     coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
     CoordinatorWriteRequests write_req = req;
+    spdlog::trace("sending hlc request to start transaction");
     auto write_res = coord_cli_.SendWriteRequest(write_req);
+    spdlog::trace("received hlc response to start transaction");
     if (write_res.HasError()) {
       throw std::runtime_error("HLC request failed");
     }
@@ -170,7 +166,9 @@ class RequestRouter : public RequestRouterInterface {
   void Commit() override {
     coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
     CoordinatorWriteRequests write_req = req;
+    spdlog::trace("sending hlc request before committing transaction");
     auto write_res = coord_cli_.SendWriteRequest(write_req);
+    spdlog::trace("received hlc response before committing transaction");
     if (write_res.HasError()) {
       throw std::runtime_error("HLC request for commit failed");
     }
@@ -238,26 +236,25 @@ class RequestRouter : public RequestRouterInterface {
 
   // TODO(kostasrim) Simplify return result
   std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
-    ExecutionState<msgs::ScanVerticesRequest> state = {};
-    state.label = label;
-
     // create requests
-    InitializeExecutionState(state);
+    std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests_to_be_sent = RequestsForScanVertices(label);
+    spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
 
     // begin all requests in parallel
-    for (auto &request : state.requests) {
+    RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
       auto &storage_client = GetStorageClientForShard(request.shard);
-      msgs::ReadRequests req = request.request;
-
-      request.async_request_token = storage_client.SendAsyncReadRequest(request.request);
+      storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
     }
+    spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
 
     // drive requests to completion
-    std::vector<msgs::ScanVerticesResponse> responses;
-    responses.reserve(state.requests.size());
-    do {
-      DriveReadResponses(state, responses);
-    } while (!state.requests.empty());
+    auto responses = DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
+    spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
 
     // convert responses into VertexAccessor objects to return
     std::vector<VertexAccessor> accessors;
@@ -272,62 +269,55 @@ class RequestRouter : public RequestRouterInterface {
   }
 
   std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) override {
-    ExecutionState<msgs::CreateVerticesRequest> state = {};
     MG_ASSERT(!new_vertices.empty());
 
     // create requests
-    InitializeExecutionState(state, new_vertices);
+    std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
+        RequestsForCreateVertices(new_vertices);
+    spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
 
     // begin all requests in parallel
-    for (auto &request : state.requests) {
-      auto req_deep_copy = request.request;
-
-      for (auto &new_vertex : req_deep_copy.new_vertices) {
+    RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      for (auto &new_vertex : request.request.new_vertices) {
         new_vertex.label_ids.erase(new_vertex.label_ids.begin());
       }
-
       auto &storage_client = GetStorageClientForShard(request.shard);
-
-      msgs::WriteRequests req = req_deep_copy;
-      request.async_request_token = storage_client.SendAsyncWriteRequest(req);
+      storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
     }
+    spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
 
     // drive requests to completion
-    std::vector<msgs::CreateVerticesResponse> responses;
-    responses.reserve(state.requests.size());
-    do {
-      DriveWriteResponses(state, responses);
-    } while (!state.requests.empty());
-
-    return responses;
+    return DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
   }
 
   std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) override {
-    ExecutionState<msgs::CreateExpandRequest> state = {};
     MG_ASSERT(!new_edges.empty());
 
     // create requests
-    InitializeExecutionState(state, new_edges);
+    std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent = RequestsForCreateExpand(new_edges);
 
     // begin all requests in parallel
-    for (auto &request : state.requests) {
+    RunningRequests<msgs::CreateExpandRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
       auto &storage_client = GetStorageClientForShard(request.shard);
       msgs::WriteRequests req = request.request;
-      request.async_request_token = storage_client.SendAsyncWriteRequest(req);
+      storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
     }
 
     // drive requests to completion
-    std::vector<msgs::CreateExpandResponse> responses;
-    responses.reserve(state.requests.size());
-    do {
-      DriveWriteResponses(state, responses);
-    } while (!state.requests.empty());
-
-    return responses;
+    return DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
   }
 
   std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) override {
-    ExecutionState<msgs::ExpandOneRequest> state = {};
     // TODO(kostasrim)Update to limit the batch size here
     // Expansions of the destination must be handled by the caller. For example
     // match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
@@ -335,21 +325,22 @@ class RequestRouter : public RequestRouterInterface {
     // must be fetched again with an ExpandOne(Edges.dst)
 
     // create requests
-    InitializeExecutionState(state, std::move(request));
+    std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
 
     // begin all requests in parallel
-    for (auto &request : state.requests) {
+    RunningRequests<msgs::ExpandOneRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
       auto &storage_client = GetStorageClientForShard(request.shard);
       msgs::ReadRequests req = request.request;
-      request.async_request_token = storage_client.SendAsyncReadRequest(req);
+      storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
     }
 
     // drive requests to completion
-    std::vector<msgs::ExpandOneResponse> responses;
-    responses.reserve(state.requests.size());
-    do {
-      DriveReadResponses(state, responses);
-    } while (!state.requests.empty());
+    auto responses = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
 
     // post-process responses
     std::vector<msgs::ExpandOneResultRow> result_rows;
@@ -367,6 +358,36 @@ class RequestRouter : public RequestRouterInterface {
     return result_rows;
   }
 
+  std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest requests) override {
+    // create requests
+    std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
+        RequestsForGetProperties(std::move(requests));
+
+    // begin all requests in parallel
+    RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      msgs::ReadRequests req = request.request;
+      storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
+    }
+
+    // drive requests to completion
+    auto responses = DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
+
+    // post-process responses
+    std::vector<msgs::GetPropertiesResultRow> result_rows;
+
+    for (auto &&response : responses) {
+      std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
+    }
+
+    return result_rows;
+  }
+
   std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
     return shards_map_.GetPropertyId(name);
   }
@@ -380,11 +401,9 @@ class RequestRouter : public RequestRouterInterface {
   }
 
  private:
-  void InitializeExecutionState(ExecutionState<msgs::CreateVerticesRequest> &state,
-                                std::vector<msgs::NewVertex> new_vertices) {
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, msgs::CreateVerticesRequest> per_shard_request_table;
+  std::vector<ShardRequestState<msgs::CreateVerticesRequest>> RequestsForCreateVertices(
+      const std::vector<msgs::NewVertex> &new_vertices) {
+    std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
 
     for (auto &new_vertex : new_vertices) {
       MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
@@ -397,23 +416,24 @@ class RequestRouter : public RequestRouterInterface {
       per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
     }
 
+    std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests = {};
+
     for (auto &[shard, request] : per_shard_request_table) {
       ShardRequestState<msgs::CreateVerticesRequest> shard_request_state{
           .shard = shard,
           .request = request,
-          .async_request_token = std::nullopt,
       };
-      state.requests.emplace_back(std::move(shard_request_state));
+      requests.emplace_back(std::move(shard_request_state));
     }
+
+    return requests;
   }
 
-  void InitializeExecutionState(ExecutionState<msgs::CreateExpandRequest> &state,
-                                std::vector<msgs::NewExpand> new_expands) {
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, msgs::CreateExpandRequest> per_shard_request_table;
+  std::vector<ShardRequestState<msgs::CreateExpandRequest>> RequestsForCreateExpand(
+      const std::vector<msgs::NewExpand> &new_expands) {
+    std::map<ShardMetadata, msgs::CreateExpandRequest> per_shard_request_table;
     auto ensure_shard_exists_in_table = [&per_shard_request_table,
-                                         transaction_id = transaction_id_](const Shard &shard) {
+                                         transaction_id = transaction_id_](const ShardMetadata &shard) {
       if (!per_shard_request_table.contains(shard)) {
         msgs::CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
         per_shard_request_table.insert({shard, std::move(create_expand_request)});
@@ -435,30 +455,36 @@ class RequestRouter : public RequestRouterInterface {
       per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
     }
 
+    std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests = {};
+
     for (auto &[shard, request] : per_shard_request_table) {
       ShardRequestState<msgs::CreateExpandRequest> shard_request_state{
           .shard = shard,
           .request = request,
-          .async_request_token = std::nullopt,
       };
-      state.requests.emplace_back(std::move(shard_request_state));
+      requests.emplace_back(std::move(shard_request_state));
     }
+
+    return requests;
   }
 
-  void InitializeExecutionState(ExecutionState<msgs::ScanVerticesRequest> &state) {
+  std::vector<ShardRequestState<msgs::ScanVerticesRequest>> RequestsForScanVertices(
+      const std::optional<std::string> &label) {
     std::vector<coordinator::Shards> multi_shards;
-    state.transaction_id = transaction_id_;
-    if (!state.label) {
-      multi_shards = shards_map_.GetAllShards();
-    } else {
-      const auto label_id = shards_map_.GetLabelId(*state.label);
+    if (label) {
+      const auto label_id = shards_map_.GetLabelId(*label);
       MG_ASSERT(label_id);
       MG_ASSERT(IsPrimaryLabel(*label_id));
-      multi_shards = {shards_map_.GetShardsForLabel(*state.label)};
+      multi_shards = {shards_map_.GetShardsForLabel(*label)};
+    } else {
+      multi_shards = shards_map_.GetAllShards();
     }
+
+    std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
+
     for (auto &shards : multi_shards) {
       for (auto &[key, shard] : shards) {
-        MG_ASSERT(!shard.empty());
+        MG_ASSERT(!shard.peers.empty());
 
         msgs::ScanVerticesRequest request;
         request.transaction_id = transaction_id_;
@@ -467,22 +493,21 @@ class RequestRouter : public RequestRouterInterface {
         ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
             .shard = shard,
             .request = std::move(request),
-            .async_request_token = std::nullopt,
         };
 
-        state.requests.emplace_back(std::move(shard_request_state));
+        requests.emplace_back(std::move(shard_request_state));
       }
     }
+
+    return requests;
   }
 
-  void InitializeExecutionState(ExecutionState<msgs::ExpandOneRequest> &state, msgs::ExpandOneRequest request) {
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, msgs::ExpandOneRequest> per_shard_request_table;
-    auto top_level_rqst_template = request;
+  std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(const msgs::ExpandOneRequest &request) {
+    std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
+    msgs::ExpandOneRequest top_level_rqst_template = request;
     top_level_rqst_template.transaction_id = transaction_id_;
     top_level_rqst_template.src_vertices.clear();
-    state.requests.clear();
+
     for (auto &vertex : request.src_vertices) {
       auto shard =
           shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
@@ -492,18 +517,61 @@ class RequestRouter : public RequestRouterInterface {
       per_shard_request_table[shard].src_vertices.push_back(vertex);
     }
 
+    std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests = {};
+
     for (auto &[shard, request] : per_shard_request_table) {
       ShardRequestState<msgs::ExpandOneRequest> shard_request_state{
           .shard = shard,
           .request = request,
-          .async_request_token = std::nullopt,
       };
 
-      state.requests.emplace_back(std::move(shard_request_state));
+      requests.emplace_back(std::move(shard_request_state));
     }
+
+    return requests;
   }
 
-  StorageClient &GetStorageClientForShard(Shard shard) {
+  std::vector<ShardRequestState<msgs::GetPropertiesRequest>> RequestsForGetProperties(
+      msgs::GetPropertiesRequest &&request) {
+    std::map<ShardMetadata, msgs::GetPropertiesRequest> per_shard_request_table;
+    auto top_level_rqst_template = request;
+    top_level_rqst_template.transaction_id = transaction_id_;
+    top_level_rqst_template.vertex_ids.clear();
+    top_level_rqst_template.vertices_and_edges.clear();
+
+    for (auto &&vertex : request.vertex_ids) {
+      auto shard =
+          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
+      if (!per_shard_request_table.contains(shard)) {
+        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
+      }
+      per_shard_request_table[shard].vertex_ids.emplace_back(std::move(vertex));
+    }
+
+    for (auto &[vertex, maybe_edge] : request.vertices_and_edges) {
+      auto shard =
+          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
+      if (!per_shard_request_table.contains(shard)) {
+        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
+      }
+      per_shard_request_table[shard].vertices_and_edges.emplace_back(std::move(vertex), maybe_edge);
+    }
+
+    std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests;
+
+    for (auto &[shard, rqst] : per_shard_request_table) {
+      ShardRequestState<msgs::GetPropertiesRequest> shard_request_state{
+          .shard = shard,
+          .request = std::move(rqst),
+      };
+
+      requests.emplace_back(std::move(shard_request_state));
+    }
+
+    return requests;
+  }
+
+  StorageClient &GetStorageClientForShard(ShardMetadata shard) {
     if (!storage_cli_manager_.Exists(shard)) {
       AddStorageClientToManager(shard);
     }
@@ -515,12 +583,12 @@ class RequestRouter : public RequestRouterInterface {
     return GetStorageClientForShard(std::move(shard));
   }
 
-  void AddStorageClientToManager(Shard target_shard) {
-    MG_ASSERT(!target_shard.empty());
-    auto leader_addr = target_shard.front();
+  void AddStorageClientToManager(ShardMetadata target_shard) {
+    MG_ASSERT(!target_shard.peers.empty());
+    auto leader_addr = target_shard.peers.front();
     std::vector<Address> addresses;
-    addresses.reserve(target_shard.size());
-    for (auto &address : target_shard) {
+    addresses.reserve(target_shard.peers.size());
+    for (auto &address : target_shard.peers) {
       addresses.push_back(std::move(address.address));
     }
     auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
@@ -528,13 +596,24 @@ class RequestRouter : public RequestRouterInterface {
   }
 
   template <typename RequestT, typename ResponseT>
-  void DriveReadResponses(ExecutionState<RequestT> &state, std::vector<ResponseT> &responses) {
-    for (auto &request : state.requests) {
+  std::vector<ResponseT> DriveReadResponses(RunningRequests<RequestT> &running_requests) {
+    // Store responses in a map based on the corresponding request
+    // offset, so that they can be reassembled in the correct order
+    // even if they came back in randomized orders.
+    std::map<size_t, ResponseT> response_map;
+
+    spdlog::trace("waiting on readiness for token");
+    while (response_map.size() < running_requests.size()) {
+      auto ready = notifier_.Await();
+      spdlog::trace("got readiness for token {}", ready.GetId());
+      auto &request = running_requests.at(ready.GetId());
       auto &storage_client = GetStorageClientForShard(request.shard);
 
-      auto poll_result = storage_client.AwaitAsyncReadRequest(request.async_request_token.value());
-      while (!poll_result) {
-        poll_result = storage_client.AwaitAsyncReadRequest(request.async_request_token.value());
+      std::optional<utils::BasicResult<io::TimedOut, msgs::ReadResponses>> poll_result =
+          storage_client.PollAsyncReadRequest(ready);
+
+      if (!poll_result.has_value()) {
+        continue;
       }
 
       if (poll_result->HasError()) {
@@ -547,19 +626,40 @@ class RequestRouter : public RequestRouterInterface {
         throw std::runtime_error("RequestRouter Read request did not succeed");
       }
 
-      responses.push_back(std::move(response));
+      // the readiness token has an ID based on the request vector offset
+      response_map.emplace(ready.GetId(), std::move(response));
     }
-    state.requests.clear();
+
+    std::vector<ResponseT> responses;
+    responses.reserve(running_requests.size());
+
+    int last = -1;
+    for (auto &&[offset, response] : response_map) {
+      MG_ASSERT(last + 1 == offset);
+      responses.emplace_back(std::forward<ResponseT>(response));
+      last = offset;
+    }
+
+    return responses;
   }
 
   template <typename RequestT, typename ResponseT>
-  void DriveWriteResponses(ExecutionState<RequestT> &state, std::vector<ResponseT> &responses) {
-    for (auto &request : state.requests) {
+  std::vector<ResponseT> DriveWriteResponses(RunningRequests<RequestT> &running_requests) {
+    // Store responses in a map based on the corresponding request
+    // offset, so that they can be reassembled in the correct order
+    // even if they came back in randomized orders.
+    std::map<size_t, ResponseT> response_map;
+
+    while (response_map.size() < running_requests.size()) {
+      auto ready = notifier_.Await();
+      auto &request = running_requests.at(ready.GetId());
       auto &storage_client = GetStorageClientForShard(request.shard);
 
-      auto poll_result = storage_client.AwaitAsyncWriteRequest(request.async_request_token.value());
-      while (!poll_result) {
-        poll_result = storage_client.AwaitAsyncWriteRequest(request.async_request_token.value());
+      std::optional<utils::BasicResult<io::TimedOut, msgs::WriteResponses>> poll_result =
+          storage_client.PollAsyncWriteRequest(ready);
+
+      if (!poll_result.has_value()) {
+        continue;
       }
 
       if (poll_result->HasError()) {
@@ -572,9 +672,21 @@ class RequestRouter : public RequestRouterInterface {
         throw std::runtime_error("RequestRouter Write request did not succeed");
       }
 
-      responses.push_back(std::move(response));
+      // the readiness token has an ID based on the request vector offset
+      response_map.emplace(ready.GetId(), std::move(response));
     }
-    state.requests.clear();
+
+    std::vector<ResponseT> responses;
+    responses.reserve(running_requests.size());
+
+    int last = -1;
+    for (auto &&[offset, response] : response_map) {
+      MG_ASSERT(last + 1 == offset);
+      responses.emplace_back(std::forward<ResponseT>(response));
+      last = offset;
+    }
+
+    return responses;
   }
 
   void SetUpNameIdMappers() {
@@ -603,6 +715,7 @@ class RequestRouter : public RequestRouterInterface {
   RsmStorageClientManager<StorageClient> storage_cli_manager_;
   io::Io<TTransport> io_;
   coordinator::Hlc transaction_id_;
+  io::Notifier notifier_ = {};
   // TODO(kostasrim) Add batch prefetching
 };
 }  // namespace memgraph::query::v2

src/query/v2/requests.hpp

@@ -327,10 +327,6 @@ struct Expression {
   std::string expression;
 };
 
-struct Filter {
-  std::string filter_expression;
-};
-
 enum class OrderingDirection { ASCENDING = 1, DESCENDING = 2 };
 
 struct OrderBy {
@@ -372,21 +368,32 @@ struct ScanVerticesResponse {
   std::vector<ScanResultRow> results;
 };
 
-using VertexOrEdgeIds = std::variant<VertexId, EdgeId>;
-
 struct GetPropertiesRequest {
   Hlc transaction_id;
-  // Shouldn't contain mixed vertex and edge ids
-  VertexOrEdgeIds vertex_or_edge_ids;
-  std::vector<PropertyId> property_ids;
-  std::vector<Expression> expressions;
-  bool only_unique = false;
-  std::optional<std::vector<OrderBy>> order_by;
+  std::vector<VertexId> vertex_ids;
+  std::vector<std::pair<VertexId, EdgeId>> vertices_and_edges;
+
+  std::optional<std::vector<PropertyId>> property_ids;
+  std::vector<std::string> expressions;
+
+  std::vector<OrderBy> order_by;
   std::optional<size_t> limit;
-  std::optional<Filter> filter;
+
+  // Return only the properties of the vertices or edges that the filter predicate
+  // evaluates to true
+  std::optional<std::string> filter;
+};
+
+struct GetPropertiesResultRow {
+  VertexId vertex;
+  std::optional<EdgeId> edge;
+
+  std::vector<std::pair<PropertyId, Value>> props;
+  std::vector<Value> evaluated_expressions;
 };
 
 struct GetPropertiesResponse {
+  std::vector<GetPropertiesResultRow> result_row;
   std::optional<ShardError> error;
 };
 

src/storage/v3/CMakeLists.txt

@@ -9,8 +9,6 @@ set(storage_v3_src_files
     temporal.cpp
     edge_accessor.cpp
     indices.cpp
-    key_store.cpp
-    lexicographically_ordered_vertex.cpp
     property_store.cpp
     vertex_accessor.cpp
     schemas.cpp
@@ -19,6 +17,7 @@ set(storage_v3_src_files
     shard_rsm.cpp
     bindings/typed_value.cpp
     expr.cpp
+    vertex.cpp
     request_helper.cpp)
 
 # ######################

src/storage/v3/bindings/frame.hpp

@@ -17,5 +17,5 @@
 #include "storage/v3/bindings/typed_value.hpp"
 
 namespace memgraph::storage::v3 {
-using Frame = memgraph::expr::Frame<TypedValue>;
+using Frame = memgraph::expr::Frame;
 }  // namespace memgraph::storage::v3

src/storage/v3/delta.hpp

@@ -11,17 +11,18 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
 #include "storage/v3/edge_ref.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_id.hpp"
 #include "utils/logging.hpp"
 
 namespace memgraph::storage::v3 {
 
 // Forward declarations because we only store pointers here.
-struct Vertex;
 struct Edge;
 struct Delta;
 struct CommitInfo;
@@ -129,7 +130,7 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer
 inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { return !(a == b); }
 
 struct Delta {
-  enum class Action {
+  enum class Action : uint8_t {
     // Used for both Vertex and Edge
     DELETE_OBJECT,
     RECREATE_OBJECT,

src/storage/v3/edge.hpp

@@ -21,7 +21,7 @@
 
 namespace memgraph::storage::v3 {
 
-struct Vertex;
+using EdgeContainer = std::map<Gid, Edge>;
 
 struct Edge {
   Edge(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {

src/storage/v3/edge_accessor.hpp

@@ -25,7 +25,6 @@
 
 namespace memgraph::storage::v3 {
 
-struct Vertex;
 class VertexAccessor;
 struct Indices;
 

src/storage/v3/indices.cpp

@@ -11,12 +11,16 @@
 
 #include "indices.hpp"
 
+#include <algorithm>
+#include <functional>
 #include <limits>
 
+#include "storage/v3/delta.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/mvcc.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/schemas.hpp"
+#include "storage/v3/vertex.hpp"
 #include "utils/bound.hpp"
 #include "utils/logging.hpp"
 #include "utils/memory_tracker.hpp"
@@ -53,9 +57,9 @@ bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t timestamp)
   bool deleted{false};
   const Delta *delta{nullptr};
   {
-    has_label = utils::Contains(vertex.labels, label);
-    deleted = vertex.deleted;
-    delta = vertex.delta;
+    has_label = utils::Contains(vertex.second.labels, label);
+    deleted = vertex.second.deleted;
+    delta = vertex.second.delta;
   }
   if (!deleted && has_label) {
     return true;
@@ -105,10 +109,10 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, PropertyId
   bool deleted{false};
   const Delta *delta{nullptr};
   {
-    has_label = utils::Contains(vertex.labels, label);
-    current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
-    deleted = vertex.deleted;
-    delta = vertex.delta;
+    has_label = utils::Contains(vertex.second.labels, label);
+    current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
+    deleted = vertex.second.deleted;
+    delta = vertex.second.delta;
   }
 
   if (!deleted && has_label && current_value_equal_to_value) {
@@ -163,9 +167,9 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
   bool has_label{false};
   const Delta *delta{nullptr};
   {
-    deleted = vertex.deleted;
-    has_label = utils::Contains(vertex.labels, label);
-    delta = vertex.delta;
+    deleted = vertex.second.deleted;
+    has_label = utils::Contains(vertex.second.labels, label);
+    delta = vertex.second.delta;
   }
   ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
     switch (delta.action) {
@@ -214,10 +218,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
   bool current_value_equal_to_value = value.IsNull();
   const Delta *delta{nullptr};
   {
-    deleted = vertex.deleted;
-    has_label = utils::Contains(vertex.labels, label);
-    current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
-    delta = vertex.delta;
+    deleted = vertex.second.deleted;
+    has_label = utils::Contains(vertex.second.labels, label);
+    current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
+    delta = vertex.second.delta;
   }
   ApplyDeltasForRead(transaction, delta, view,
                      [&deleted, &has_label, &current_value_equal_to_value, key, label, &value](const Delta &delta) {
@@ -265,11 +269,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
 void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
   auto it = index_.find(label);
   if (it == index_.end()) return;
-  auto acc = it->second.access();
-  acc.insert(Entry{vertex, tx.start_timestamp.logical_id});
+  it->second.insert(Entry{vertex, tx.start_timestamp.logical_id});
 }
 
-bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
+bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
   auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
   if (!emplaced) {
@@ -277,13 +280,11 @@ bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices)
     return false;
   }
   try {
-    auto acc = it->second.access();
-    for (auto &lgo_vertex : vertices) {
-      auto &vertex = lgo_vertex.vertex;
-      if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
+    for (auto &vertex : vertices) {
+      if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
         continue;
       }
-      acc.insert(Entry{&vertex, 0});
+      it->second.insert(Entry{&vertex, 0});
     }
   } catch (const utils::OutOfMemoryException &) {
     utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -304,7 +305,7 @@ std::vector<LabelId> LabelIndex::ListIndices() const {
 
 void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
   for (auto &label_storage : index_) {
-    auto vertices_acc = label_storage.second.access();
+    auto &vertices_acc = label_storage.second;
     for (auto it = vertices_acc.begin(); it != vertices_acc.end();) {
       auto next_it = it;
       ++next_it;
@@ -316,7 +317,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
 
       if ((next_it != vertices_acc.end() && it->vertex == next_it->vertex) ||
           !AnyVersionHasLabel(*it->vertex, label_storage.first, clean_up_before_timestamp)) {
-        vertices_acc.remove(*it);
+        vertices_acc.erase(*it);
       }
 
       it = next_it;
@@ -324,7 +325,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
   }
 }
 
-LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
+LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -339,7 +340,7 @@ LabelIndex::Iterable::Iterator &LabelIndex::Iterable::Iterator::operator++() {
 }
 
 void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
-  for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
+  for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
     if (index_iterator_->vertex == current_vertex_) {
       continue;
     }
@@ -352,10 +353,9 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
   }
 }
 
-LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view,
-                               Transaction *transaction, Indices *indices, Config::Items config,
-                               const VertexValidator &vertex_validator)
-    : index_accessor_(std::move(index_accessor)),
+LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
+                               Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
+    : index_container_(&index_container),
       label_(label),
       view_(view),
       transaction_(transaction),
@@ -363,12 +363,6 @@ LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor,
       config_(config),
       vertex_validator_(&vertex_validator) {}
 
-void LabelIndex::RunGC() {
-  for (auto &index_entry : index_) {
-    index_entry.second.run_gc();
-  }
-}
-
 bool LabelPropertyIndex::Entry::operator<(const Entry &rhs) const {
   if (value < rhs.value) {
     return true;
@@ -388,14 +382,13 @@ bool LabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) const { retu
 bool LabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) const { return value == rhs; }
 
 void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
-  for (auto &[label_prop, storage] : index_) {
+  for (auto &[label_prop, index] : index_) {
     if (label_prop.first != label) {
       continue;
     }
-    auto prop_value = vertex->properties.GetProperty(label_prop.second);
+    auto prop_value = vertex->second.properties.GetProperty(label_prop.second);
     if (!prop_value.IsNull()) {
-      auto acc = storage.access();
-      acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp.logical_id});
+      index.emplace(Entry{prop_value, vertex, tx.start_timestamp.logical_id});
     }
   }
 }
@@ -405,18 +398,17 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
   if (value.IsNull()) {
     return;
   }
-  for (auto &[label_prop, storage] : index_) {
+  for (auto &[label_prop, index] : index_) {
     if (label_prop.second != property) {
       continue;
     }
-    if (utils::Contains(vertex->labels, label_prop.first)) {
-      auto acc = storage.access();
-      acc.insert(Entry{value, vertex, tx.start_timestamp.logical_id});
+    if (VertexHasLabel(*vertex, label_prop.first)) {
+      index.emplace(Entry{value, vertex, tx.start_timestamp.logical_id});
     }
   }
 }
 
-bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
+bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
   auto [it, emplaced] =
       index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
@@ -425,17 +417,15 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, Vertice
     return false;
   }
   try {
-    auto acc = it->second.access();
-    for (auto &lgo_vertex : vertices) {
-      auto &vertex = lgo_vertex.vertex;
-      if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
+    for (auto &vertex : vertices) {
+      if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
         continue;
       }
-      auto value = vertex.properties.GetProperty(property);
+      auto value = vertex.second.properties.GetProperty(property);
       if (value.IsNull()) {
         continue;
       }
-      acc.insert(Entry{std::move(value), &vertex, 0});
+      it->second.emplace(Entry{value, &vertex, 0});
     }
   } catch (const utils::OutOfMemoryException &) {
     utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -456,8 +446,7 @@ std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ListIndices() co
 
 void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
   for (auto &[label_property, index] : index_) {
-    auto index_acc = index.access();
-    for (auto it = index_acc.begin(); it != index_acc.end();) {
+    for (auto it = index.begin(); it != index.end();) {
       auto next_it = it;
       ++next_it;
 
@@ -466,17 +455,17 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
         continue;
       }
 
-      if ((next_it != index_acc.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
+      if ((next_it != index.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
           !AnyVersionHasLabelProperty(*it->vertex, label_property.first, label_property.second, it->value,
                                       clean_up_before_timestamp)) {
-        index_acc.remove(*it);
+        index.erase(it);
       }
       it = next_it;
     }
   }
 }
 
-LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
+LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -491,7 +480,7 @@ LabelPropertyIndex::Iterable::Iterator &LabelPropertyIndex::Iterable::Iterator::
 }
 
 void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
-  for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
+  for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
     if (index_iterator_->vertex == current_vertex_) {
       continue;
     }
@@ -506,11 +495,11 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
     }
     if (self_->upper_bound_) {
       if (self_->upper_bound_->value() < index_iterator_->value) {
-        index_iterator_ = self_->index_accessor_.end();
+        index_iterator_ = self_->index_container_->end();
         break;
       }
       if (!self_->upper_bound_->IsInclusive() && index_iterator_->value == self_->upper_bound_->value()) {
-        index_iterator_ = self_->index_accessor_.end();
+        index_iterator_ = self_->index_container_->end();
         break;
       }
     }
@@ -537,13 +526,12 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
 const PropertyValue kSmallestTemporalData =
     PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
 
-LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label,
-                                       PropertyId property,
+LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
                                        const std::optional<utils::Bound<PropertyValue>> &lower_bound,
                                        const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
                                        Transaction *transaction, Indices *indices, Config::Items config,
                                        const VertexValidator &vertex_validator)
-    : index_accessor_(std::move(index_accessor)),
+    : index_container_(&index_container),
       label_(label),
       property_(property),
       lower_bound_(lower_bound),
@@ -651,49 +639,55 @@ LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::begin() {
   // If the bounds are set and don't have comparable types we don't yield any
   // items from the index.
   if (!bounds_valid_) {
-    return {this, index_accessor_.end()};
+    return {this, index_container_->end()};
   }
-  auto index_iterator = index_accessor_.begin();
   if (lower_bound_) {
-    index_iterator = index_accessor_.find_equal_or_greater(lower_bound_->value());
+    return {this, std::ranges::lower_bound(*index_container_, lower_bound_->value(), std::less{}, &Entry::value)};
   }
-  return {this, index_iterator};
+  return {this, index_container_->begin()};
 }
 
-LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_accessor_.end()}; }
+LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_container_->end()}; }
 
-int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
-                                                   const PropertyValue &value) const {
+int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
   auto it = index_.find({label, property});
   MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
-  auto acc = it->second.access();
-  if (!value.IsNull()) {
-    return static_cast<int64_t>(
-        acc.estimate_count(value, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
-  }
-  // The value `Null` won't ever appear in the index because it indicates that
-  // the property shouldn't exist. Instead, this value is used as an indicator
-  // to estimate the average number of equal elements in the list (for any
-  // given value).
-  return static_cast<int64_t>(acc.estimate_average_number_of_equals(
-      [](const auto &first, const auto &second) { return first.value == second.value; },
-      static_cast<int>(utils::SkipListLayerForAverageEqualsEstimation(acc.size()))));
-}
+  MG_ASSERT(!value.IsNull(), "Null is not supported!");
 
-int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
-                                                   const std::optional<utils::Bound<PropertyValue>> &lower,
-                                                   const std::optional<utils::Bound<PropertyValue>> &upper) const {
-  auto it = index_.find({label, property});
-  MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
-  auto acc = it->second.access();
+  // TODO(jbajic) This can be improved by exiting early
+  auto start_it = std::ranges::lower_bound(it->second, value, std::less{}, &Entry::value);
   return static_cast<int64_t>(
-      acc.estimate_range_count(lower, upper, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
+      std::ranges::count_if(start_it, it->second.end(), [&value](const auto &elem) { return elem.value == value; }));
 }
 
-void LabelPropertyIndex::RunGC() {
-  for (auto &index_entry : index_) {
-    index_entry.second.run_gc();
-  }
+int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property,
+                                        const std::optional<utils::Bound<PropertyValue>> &lower,
+                                        const std::optional<utils::Bound<PropertyValue>> &upper) const {
+  auto it = index_.find({label, property});
+  MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
+  const auto lower_it = std::invoke(
+      [&index = it->second](const auto value, const auto def) {
+        if (value) {
+          if (value->IsInclusive()) {
+            return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
+          }
+          return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
+        }
+        return def;
+      },
+      lower, it->second.begin());
+  const auto upper_it = std::invoke(
+      [&index = it->second](const auto value, const auto def) {
+        if (value) {
+          if (value->IsInclusive()) {
+            return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
+          }
+          return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
+        }
+        return def;
+      },
+      upper, it->second.end());
+  return static_cast<int64_t>(std::distance(lower_it, upper_it));
 }
 
 void RemoveObsoleteEntries(Indices *indices, const uint64_t clean_up_before_timestamp) {

src/storage/v3/indices.hpp

@@ -13,6 +13,7 @@
 
 #include <cstdint>
 #include <optional>
+#include <set>
 #include <tuple>
 #include <utility>
 
@@ -20,7 +21,6 @@
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/transaction.hpp"
 #include "storage/v3/vertex_accessor.hpp"
-#include "storage/v3/vertices_skip_list.hpp"
 #include "utils/bound.hpp"
 #include "utils/logging.hpp"
 #include "utils/skip_list.hpp"
@@ -30,7 +30,6 @@ namespace memgraph::storage::v3 {
 struct Indices;
 
 class LabelIndex {
- private:
   struct Entry {
     Vertex *vertex;
     uint64_t timestamp;
@@ -41,17 +40,9 @@ class LabelIndex {
     bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
   };
 
-  struct LabelStorage {
-    LabelId label;
-    utils::SkipList<Entry> vertices;
-
-    bool operator<(const LabelStorage &rhs) const { return label < rhs.label; }
-    bool operator<(LabelId rhs) const { return label < rhs; }
-    bool operator==(const LabelStorage &rhs) const { return label == rhs.label; }
-    bool operator==(LabelId rhs) const { return label == rhs; }
-  };
-
  public:
+  using LabelIndexContainer = std::set<Entry>;
+
   LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
 
@@ -59,7 +50,7 @@ class LabelIndex {
   void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
+  bool CreateIndex(LabelId label, VertexContainer &vertices);
 
   /// Returns false if there was no index to drop
   bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
@@ -72,12 +63,12 @@ class LabelIndex {
 
   class Iterable {
    public:
-    Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view, Transaction *transaction,
-             Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
+    Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
+             Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
+      Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -90,16 +81,16 @@ class LabelIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      utils::SkipList<Entry>::Iterator index_iterator_;
+      LabelIndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
 
-    Iterator begin() { return {this, index_accessor_.begin()}; }
-    Iterator end() { return {this, index_accessor_.end()}; }
+    Iterator begin() { return {this, index_container_->begin()}; }
+    Iterator end() { return {this, index_container_->end()}; }
 
    private:
-    utils::SkipList<Entry>::Accessor index_accessor_;
+    LabelIndexContainer *index_container_;
     LabelId label_;
     View view_;
     Transaction *transaction_;
@@ -112,7 +103,7 @@ class LabelIndex {
   Iterable Vertices(LabelId label, View view, Transaction *transaction) {
     auto it = index_.find(label);
     MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint());
-    return {it->second.access(), label, view, transaction, indices_, config_, *vertex_validator_};
+    return {it->second, label, view, transaction, indices_, config_, *vertex_validator_};
   }
 
   int64_t ApproximateVertexCount(LabelId label) {
@@ -123,17 +114,14 @@ class LabelIndex {
 
   void Clear() { index_.clear(); }
 
-  void RunGC();
-
  private:
-  std::map<LabelId, utils::SkipList<Entry>> index_;
+  std::map<LabelId, LabelIndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;
 };
 
 class LabelPropertyIndex {
- private:
   struct Entry {
     PropertyValue value;
     Vertex *vertex;
@@ -147,6 +135,8 @@ class LabelPropertyIndex {
   };
 
  public:
+  using LabelPropertyIndexContainer = std::set<Entry>;
+
   LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
 
@@ -157,7 +147,7 @@ class LabelPropertyIndex {
   void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
+  bool CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices);
 
   bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
 
@@ -169,14 +159,14 @@ class LabelPropertyIndex {
 
   class Iterable {
    public:
-    Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, PropertyId property,
+    Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
              const std::optional<utils::Bound<PropertyValue>> &lower_bound,
              const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
              Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
+      Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -189,7 +179,7 @@ class LabelPropertyIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      utils::SkipList<Entry>::Iterator index_iterator_;
+      LabelPropertyIndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
@@ -198,7 +188,7 @@ class LabelPropertyIndex {
     Iterator end();
 
    private:
-    utils::SkipList<Entry>::Accessor index_accessor_;
+    LabelPropertyIndexContainer *index_container_;
     LabelId label_;
     PropertyId property_;
     std::optional<utils::Bound<PropertyValue>> lower_bound_;
@@ -217,11 +207,11 @@ class LabelPropertyIndex {
     auto it = index_.find({label, property});
     MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
               property.AsUint());
-    return {it->second.access(), label,    property, lower_bound,       upper_bound, view,
-            transaction,         indices_, config_,  *vertex_validator_};
+    return {it->second, label,       property, lower_bound, upper_bound,
+            view,       transaction, indices_, config_,     *vertex_validator_};
   }
 
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
+  int64_t VertexCount(LabelId label, PropertyId property) const {
     auto it = index_.find({label, property});
     MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
               property.AsUint());
@@ -232,18 +222,15 @@ class LabelPropertyIndex {
   /// an estimated count of nodes which have their property's value set to
   /// `value`. If the `value` specified is `Null`, then an average number of
   /// equal elements is returned.
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
+  int64_t VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
 
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property,
-                                 const std::optional<utils::Bound<PropertyValue>> &lower,
-                                 const std::optional<utils::Bound<PropertyValue>> &upper) const;
+  int64_t VertexCount(LabelId label, PropertyId property, const std::optional<utils::Bound<PropertyValue>> &lower,
+                      const std::optional<utils::Bound<PropertyValue>> &upper) const;
 
   void Clear() { index_.clear(); }
 
-  void RunGC();
-
  private:
-  std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>> index_;
+  std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;

src/storage/v3/key_store.cpp

@@ -1,43 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <algorithm>
-#include <iterator>
-#include <ranges>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/property_value.hpp"
-
-namespace memgraph::storage::v3 {
-
-KeyStore::KeyStore(const PrimaryKey &key_values) {
-  for (auto i = 0; i < key_values.size(); ++i) {
-    MG_ASSERT(!key_values[i].IsNull());
-    store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
-  }
-}
-
-PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
-
-PropertyValue KeyStore::GetKey(const PropertyId property_id) const { return store_.GetProperty(property_id); }
-
-PrimaryKey KeyStore::Keys() const {
-  auto keys_map = store_.Properties();
-  PrimaryKey keys;
-  keys.reserve(keys_map.size());
-  std::ranges::transform(
-      keys_map, std::back_inserter(keys),
-      [](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
-  return keys;
-}
-
-}  // namespace memgraph::storage::v3

src/storage/v3/key_store.hpp

@@ -11,12 +11,6 @@
 
 #pragma once
 
-#include <algorithm>
-#include <compare>
-#include <functional>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/property_store.hpp"
 #include "storage/v3/property_value.hpp"
 
 namespace memgraph::storage::v3 {
@@ -24,39 +18,4 @@ namespace memgraph::storage::v3 {
 // Primary key is a collection of primary properties.
 using PrimaryKey = std::vector<PropertyValue>;
 
-class KeyStore {
- public:
-  explicit KeyStore(const PrimaryKey &key_values);
-
-  KeyStore(const KeyStore &) = delete;
-  KeyStore(KeyStore &&other) noexcept = default;
-  KeyStore &operator=(const KeyStore &) = delete;
-  KeyStore &operator=(KeyStore &&other) noexcept = default;
-
-  ~KeyStore() = default;
-
-  PropertyValue GetKey(size_t index) const;
-
-  PropertyValue GetKey(PropertyId property) const;
-
-  PrimaryKey Keys() const;
-
-  friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
-    return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
-  }
-
-  friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
-    return std::ranges::equal(lhs.Keys(), rhs.Keys());
-  }
-
-  friend bool operator<(const KeyStore &lhs, const PrimaryKey &rhs) {
-    return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
-  }
-
-  friend bool operator==(const KeyStore &lhs, const PrimaryKey &rhs) { return std::ranges::equal(lhs.Keys(), rhs); }
-
- private:
-  PropertyStore store_;
-};
-
 }  // namespace memgraph::storage::v3

src/storage/v3/lexicographically_ordered_vertex.cpp

@@ -1,14 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "storage/v3/lexicographically_ordered_vertex.hpp"
-
-namespace memgraph::storage::v3 {}  // namespace memgraph::storage::v3

src/storage/v3/lexicographically_ordered_vertex.hpp

@@ -1,42 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <concepts>
-#include <type_traits>
-
-#include "storage/v3/vertex.hpp"
-#include "utils/concepts.hpp"
-
-namespace memgraph::storage::v3 {
-
-struct LexicographicallyOrderedVertex {
-  Vertex vertex;
-
-  friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
-    return lhs.vertex.keys == rhs.vertex.keys;
-  }
-
-  friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
-    return lhs.vertex.keys < rhs.vertex.keys;
-  }
-
-  // TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
-  friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
-    return lhs.vertex.keys == rhs;
-  }
-
-  friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
-    return lhs.vertex.keys < rhs;
-  }
-};
-}  // namespace memgraph::storage::v3

src/storage/v3/mvcc.hpp

@@ -11,12 +11,21 @@
 
 #pragma once
 
+#include <type_traits>
+
+#include "storage/v3/edge.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
 #include "storage/v3/view.hpp"
+#include "utils/concepts.hpp"
 
 namespace memgraph::storage::v3 {
 
+inline VertexData *GetDeltaHolder(Vertex *vertex) { return &vertex->second; }
+
+inline Edge *GetDeltaHolder(Edge *edge) { return edge; }
+
 /// This function iterates through the undo buffers from an object (starting
 /// from the supplied delta) and determines what deltas should be applied to get
 /// the currently visible version of the object. When the function finds a delta
@@ -77,10 +86,12 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie
 /// transaction) and returns a `bool` value indicating whether the caller can
 /// proceed with a write operation.
 template <typename TObj>
+requires utils::SameAsAnyOf<TObj, Edge, Vertex>
 inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
-  if (object->delta == nullptr) return true;
+  auto *delta_holder = GetDeltaHolder(object);
+  if (delta_holder->delta == nullptr) return true;
 
-  const auto &delta_commit_info = *object->delta->commit_info;
+  const auto &delta_commit_info = *delta_holder->delta->commit_info;
   if (delta_commit_info.start_or_commit_timestamp == transaction->commit_info->start_or_commit_timestamp ||
       (delta_commit_info.is_locally_committed &&
        delta_commit_info.start_or_commit_timestamp < transaction->start_timestamp)) {
@@ -105,9 +116,11 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
 /// the delta into the object's delta list.
 /// @throw std::bad_alloc
 template <typename TObj, class... Args>
+requires utils::SameAsAnyOf<TObj, Edge, Vertex>
 inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
   auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
                                                  transaction->command_id);
+  auto *delta_holder = GetDeltaHolder(object);
 
   // The operations are written in such order so that both `next` and `prev`
   // chains are valid at all times. The chains must be valid at all times
@@ -118,21 +131,21 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
   // TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in
   // `Shard::Accessor::CreateEdge`.
   // NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
-  delta->next = object->delta;
+  delta->next = delta_holder->delta;
   // 2. We need to set the previous delta of the new delta to the object.
   delta->prev.Set(object);
   // 3. We need to set the previous delta of the existing delta to the new
   // delta. After this point the garbage collector will be able to see the new
   // delta but won't modify it until we are done with all of our modifications.
-  if (object->delta) {
-    object->delta->prev.Set(delta);
+  if (delta_holder->delta) {
+    delta_holder->delta->prev.Set(delta);
   }
   // 4. Finally, we need to set the object's delta to the new delta. The garbage
   // collector and other transactions will acquire the object lock to read the
   // delta from the object. Because the lock is held during the whole time this
   // modification is being done, everybody else will wait until we are fully
   // done with our modification before they read the object's delta value.
-  object->delta = delta;
+  delta_holder->delta = delta;
 }
 
 }  // namespace memgraph::storage::v3

src/storage/v3/request_helper.cpp

@@ -11,6 +11,7 @@
 
 #include "storage/v3/request_helper.hpp"
 
+#include <iterator>
 #include <vector>
 
 #include "storage/v3/bindings/db_accessor.hpp"
@@ -220,30 +221,39 @@ std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexA
   return evaluated_expressions;
 }
 
+std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
+                                                const std::vector<std::string> &expressions) {
+  std::vector<TypedValue> evaluated_expressions;
+  evaluated_expressions.reserve(expressions.size());
+
+  std::transform(expressions.begin(), expressions.end(), std::back_inserter(evaluated_expressions),
+                 [&dba, &v_acc, &e_acc](const auto &expression) {
+                   return ComputeExpression(dba, v_acc, e_acc, expression, expr::identifier_node_symbol,
+                                            expr::identifier_edge_symbol);
+                 });
+
+  return evaluated_expressions;
+}
+
 ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
                                                                           const Schemas::Schema &schema) {
-  std::map<PropertyId, Value> ret;
-  auto props = acc.Properties(view);
-  if (props.HasError()) {
-    spdlog::debug("Encountered an error while trying to get vertex properties.");
-    return props.GetError();
+  auto ret = impl::CollectAllPropertiesImpl<VertexAccessor>(acc, view);
+  if (ret.HasError()) {
+    return ret.GetError();
   }
 
-  auto &properties = props.GetValue();
-  std::transform(properties.begin(), properties.end(), std::inserter(ret, ret.begin()),
-                 [](std::pair<const PropertyId, PropertyValue> &pair) {
-                   return std::make_pair(pair.first, FromPropertyValueToValue(std::move(pair.second)));
-                 });
-  properties.clear();
-
   auto pks = PrimaryKeysFromAccessor(acc, view, schema);
   if (pks) {
-    ret.merge(*pks);
+    ret.GetValue().merge(std::move(*pks));
   }
 
   return ret;
 }
 
+ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view) {
+  return impl::CollectAllPropertiesImpl(acc, view);
+}
+
 EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows) {
   // Functions to select connecting edges based on uniquness
   EdgeUniquenessFunction maybe_filter_based_on_edge_uniquness;
@@ -350,11 +360,20 @@ EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req) {
   return edge_filler;
 }
 
-bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const std::vector<std::string> &filters,
-                    const std::string_view node_name) {
-  return std::ranges::all_of(filters, [&node_name, &dba, &v_acc](const auto &filter_expr) {
-    auto res = ComputeExpression(dba, v_acc, std::nullopt, filter_expr, node_name, "");
-    return res.IsBool() && res.ValueBool();
+bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc,
+                    const std::vector<std::string> &filters) {
+  return std::ranges::all_of(filters, [&dba, &v_acc](const auto &filter_expr) {
+    const auto result = ComputeExpression(dba, v_acc, std::nullopt, filter_expr, expr::identifier_node_symbol, "");
+    return result.IsBool() && result.ValueBool();
+  });
+}
+
+bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
+                  const std::vector<std::string> &filters) {
+  return std::ranges::all_of(filters, [&dba, &v_acc, &e_acc](const auto &filter_expr) {
+    const auto result =
+        ComputeExpression(dba, v_acc, e_acc, filter_expr, expr::identifier_node_symbol, expr::identifier_edge_symbol);
+    return result.IsBool() && result.ValueBool();
   });
 }
 
@@ -526,4 +545,36 @@ std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<Edg
   return ordered;
 }
 
+std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
+    DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
+    const std::vector<VertexAccessor> &vertex_acc) {
+  MG_ASSERT(vertex_acc.size() == iterable.size());
+  std::vector<Ordering> ordering;
+  ordering.reserve(order_by_edges.size());
+  std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(ordering),
+                 [](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
+
+  std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> ordered;
+  VertexAccessor current = vertex_acc.front();
+  size_t id = 0;
+  for (auto it = iterable.begin(); it != iterable.end(); it++, id++) {
+    current = vertex_acc[id];
+    std::vector<TypedValue> properties_order_by;
+    properties_order_by.reserve(order_by_edges.size());
+    std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(properties_order_by),
+                   [&dba, it, current](const auto &order_by) {
+                     return ComputeExpression(dba, current, *it, order_by.expression.expression,
+                                              expr::identifier_node_symbol, expr::identifier_edge_symbol);
+                   });
+
+    ordered.push_back({std::move(properties_order_by), {current, *it}});
+  }
+
+  auto compare_typed_values = TypedValueVectorCompare(ordering);
+  std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
+    return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
+  });
+  return ordered;
+}
+
 }  // namespace memgraph::storage::v3

src/storage/v3/request_helper.hpp

@@ -20,6 +20,7 @@
 #include "storage/v3/edge_accessor.hpp"
 #include "storage/v3/expr.hpp"
 #include "storage/v3/shard.hpp"
+#include "storage/v3/value_conversions.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "utils/template_utils.hpp"
 
@@ -31,7 +32,7 @@ using EdgeFiller =
 using msgs::Value;
 
 template <typename T>
-concept ObjectAccessor = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor>;
+concept OrderableObject = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor, std::pair<VertexAccessor, EdgeAccessor>>;
 
 inline bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
   // in ordering null comes after everything else
@@ -125,7 +126,7 @@ class TypedValueVectorCompare final {
   std::vector<Ordering> ordering_;
 };
 
-template <ObjectAccessor TObjectAccessor>
+template <OrderableObject TObjectAccessor>
 struct Element {
   std::vector<TypedValue> properties_order_by;
   TObjectAccessor object_acc;
@@ -167,6 +168,10 @@ std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<Edg
                                                 std::vector<msgs::OrderBy> &order_by_edges,
                                                 const VertexAccessor &vertex_acc);
 
+std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
+    DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
+    const std::vector<VertexAccessor> &vertex_acc);
+
 VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_iterable,
                                                   const std::vector<PropertyValue> &primary_key, View view);
 
@@ -177,19 +182,65 @@ std::vector<Element<VertexAccessor>>::const_iterator GetStartOrderedElementsIter
 std::array<std::vector<EdgeAccessor>, 2> GetEdgesFromVertex(const VertexAccessor &vertex_accessor,
                                                             msgs::EdgeDirection direction);
 
-bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const std::vector<std::string> &filters,
-                    std::string_view node_name);
+bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const std::vector<std::string> &filters);
+
+bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
+                  const std::vector<std::string> &filters);
 
 std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexAccessor &v_acc,
                                                   const std::vector<std::string> &expressions,
                                                   std::string_view node_name);
 
-ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const VertexAccessor &acc,
+std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
+                                                const std::vector<std::string> &expressions);
+
+template <typename T>
+concept PropertiesAccessor = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor>;
+
+template <PropertiesAccessor TAccessor>
+ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const TAccessor &acc,
                                                                                const std::vector<PropertyId> &props,
-                                                                               View view);
+                                                                               View view) {
+  std::map<PropertyId, Value> ret;
+
+  for (const auto &prop : props) {
+    auto result = acc.GetProperty(prop, view);
+    if (result.HasError()) {
+      spdlog::debug("Encountered an Error while trying to get a vertex property.");
+      return result.GetError();
+    }
+    auto &value = result.GetValue();
+    ret.emplace(std::make_pair(prop, FromPropertyValueToValue(std::move(value))));
+  }
+
+  return ret;
+}
 
 ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
                                                                           const Schemas::Schema &schema);
+namespace impl {
+template <PropertiesAccessor TAccessor>
+ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesImpl(const TAccessor &acc, View view) {
+  std::map<PropertyId, Value> ret;
+  auto props = acc.Properties(view);
+  if (props.HasError()) {
+    spdlog::debug("Encountered an error while trying to get vertex properties.");
+    return props.GetError();
+  }
+
+  auto &properties = props.GetValue();
+  std::transform(properties.begin(), properties.end(), std::inserter(ret, ret.begin()),
+                 [](std::pair<const PropertyId, PropertyValue> &pair) {
+                   return std::make_pair(pair.first, conversions::FromPropertyValueToValue(std::move(pair.second)));
+                 });
+  return ret;
+}
+}  // namespace impl
+
+template <PropertiesAccessor TAccessor>
+ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const TAccessor &acc, View view) {
+  return impl::CollectAllPropertiesImpl<TAccessor>(acc, view);
+}
 
 EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows);
 

src/storage/v3/shard.cpp

@@ -38,7 +38,7 @@
 #include "storage/v3/transaction.hpp"
 #include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_accessor.hpp"
-#include "storage/v3/vertices_skip_list.hpp"
+#include "storage/v3/view.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/file.hpp"
 #include "utils/logging.hpp"
@@ -75,11 +75,11 @@ uint64_t GetCleanupBeforeTimestamp(const std::map<uint64_t, std::unique_ptr<Tran
 
 }  // namespace
 
-auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Iterator end,
+auto AdvanceToVisibleVertex(VertexContainer::iterator it, VertexContainer::iterator end,
                             std::optional<VertexAccessor> *vertex, Transaction *tx, View view, Indices *indices,
                             Config::Items config, const VertexValidator &vertex_validator) {
   while (it != end) {
-    *vertex = VertexAccessor::Create(&it->vertex, tx, indices, config, vertex_validator, view);
+    *vertex = VertexAccessor::Create(&*it, tx, indices, config, vertex_validator, view);
     if (!*vertex) {
       ++it;
       continue;
@@ -89,17 +89,17 @@ auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Ite
   return it;
 }
 
-AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it)
+AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VertexContainer::iterator it)
     : self_(self),
-      it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), &self->vertex_, self->transaction_, self->view_,
+      it_(AdvanceToVisibleVertex(it, self->vertices_accessor_->end(), &self->vertex_, self->transaction_, self->view_,
                                  self->indices_, self->config_, *self_->vertex_validator_)) {}
 
 VertexAccessor AllVerticesIterable::Iterator::operator*() const { return *self_->vertex_; }
 
 AllVerticesIterable::Iterator &AllVerticesIterable::Iterator::operator++() {
   ++it_;
-  it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_.end(), &self_->vertex_, self_->transaction_, self_->view_,
-                               self_->indices_, self_->config_, *self_->vertex_validator_);
+  it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_->end(), &self_->vertex_, self_->transaction_,
+                               self_->view_, self_->indices_, self_->config_, *self_->vertex_validator_);
   return *this;
 }
 
@@ -346,7 +346,7 @@ Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
     : shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
 
 ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
-    const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
+    const std::vector<LabelId> &labels, const PrimaryKey &primary_properties,
     const std::vector<std::pair<PropertyId, PropertyValue>> &properties) {
   OOMExceptionEnabler oom_exception;
   const auto schema = shard_->GetSchema(shard_->primary_label_)->second;
@@ -357,16 +357,15 @@ ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
     return {std::move(maybe_schema_violation.GetError())};
   }
 
-  auto acc = shard_->vertices_.access();
   auto *delta = CreateDeleteObjectDelta(transaction_);
-  auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}});
-  delta->prev.Set(&it->vertex);
+  auto [it, inserted] = shard_->vertices_.emplace(primary_properties, VertexData{delta});
+  delta->prev.Set(&*it);
 
-  VertexAccessor vertex_acc{&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
+  VertexAccessor vertex_acc{&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
   if (!inserted) {
     return SHARD_ERROR(ErrorCode::VERTEX_ALREADY_INSERTED);
   }
-  MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
+  MG_ASSERT(it != shard_->vertices_.end(), "Invalid Vertex accessor!");
 
   // TODO(jbajic) Improve, maybe delay index update
   for (const auto &[property_id, property_value] : properties) {
@@ -386,13 +385,11 @@ ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
 }
 
 std::optional<VertexAccessor> Shard::Accessor::FindVertex(std::vector<PropertyValue> primary_key, View view) {
-  auto acc = shard_->vertices_.access();
-  // Later on use label space
-  auto it = acc.find(primary_key);
-  if (it == acc.end()) {
+  auto it = shard_->vertices_.find(primary_key);
+  if (it == shard_->vertices_.end()) {
     return std::nullopt;
   }
-  return VertexAccessor::Create(&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
+  return VertexAccessor::Create(&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
 }
 
 ShardResult<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) {
@@ -403,14 +400,15 @@ ShardResult<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexA
 
   if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_ptr->deleted) {
+  if (vertex_ptr->second.deleted) {
     return std::optional<VertexAccessor>{};
   }
 
-  if (!vertex_ptr->in_edges.empty() || !vertex_ptr->out_edges.empty()) return SHARD_ERROR(ErrorCode::VERTEX_HAS_EDGES);
+  if (!vertex_ptr->second.in_edges.empty() || !vertex_ptr->second.out_edges.empty())
+    return SHARD_ERROR(ErrorCode::VERTEX_HAS_EDGES);
 
   CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
-  vertex_ptr->deleted = true;
+  vertex_ptr->second.deleted = true;
 
   return std::make_optional<VertexAccessor>(vertex_ptr, transaction_, &shard_->indices_, config_,
                                             shard_->vertex_validator_, true);
@@ -425,20 +423,20 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
             "accessor when deleting a vertex!");
   auto *vertex_ptr = vertex->vertex_;
 
-  std::vector<Vertex::EdgeLink> in_edges;
-  std::vector<Vertex::EdgeLink> out_edges;
+  std::vector<VertexData::EdgeLink> in_edges;
+  std::vector<VertexData::EdgeLink> out_edges;
 
   {
     if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-    if (vertex_ptr->deleted) return std::optional<ReturnType>{};
+    if (vertex_ptr->second.deleted) return std::optional<ReturnType>{};
 
-    in_edges = vertex_ptr->in_edges;
-    out_edges = vertex_ptr->out_edges;
+    in_edges = vertex_ptr->second.in_edges;
+    out_edges = vertex_ptr->second.out_edges;
   }
 
   std::vector<EdgeAccessor> deleted_edges;
-  const VertexId vertex_id{shard_->primary_label_, vertex_ptr->keys.Keys()};
+  const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)};  // TODO Replace
   for (const auto &item : in_edges) {
     auto [edge_type, from_vertex, edge] = item;
     EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
@@ -472,10 +470,10 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
 
   if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  MG_ASSERT(!vertex_ptr->deleted, "Invalid database state!");
+  MG_ASSERT(!vertex_ptr->second.deleted, "Invalid database state!");
 
   CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
-  vertex_ptr->deleted = true;
+  vertex_ptr->second.deleted = true;
 
   return std::make_optional<ReturnType>(
       VertexAccessor{vertex_ptr, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, true},
@@ -488,51 +486,50 @@ ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, V
   Vertex *from_vertex{nullptr};
   Vertex *to_vertex{nullptr};
 
-  auto acc = shard_->vertices_.access();
+  auto &vertices = shard_->vertices_;
 
   const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
   const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
   MG_ASSERT(from_is_local || to_is_local, "Trying to create an edge without having a local vertex");
 
   if (from_is_local) {
-    auto it = acc.find(from_vertex_id.primary_key);
-    MG_ASSERT(it != acc.end(), "Cannot find local vertex");
-    from_vertex = &it->vertex;
+    auto it = vertices.find(from_vertex_id.primary_key);
+    MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
+    from_vertex = &*it;
   }
 
   if (to_is_local) {
-    auto it = acc.find(to_vertex_id.primary_key);
-    MG_ASSERT(it != acc.end(), "Cannot find local vertex");
-    to_vertex = &it->vertex;
+    auto it = vertices.find(to_vertex_id.primary_key);
+    MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
+    to_vertex = &*it;
   }
 
   if (from_is_local) {
     if (!PrepareForWrite(transaction_, from_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
-    if (from_vertex->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+    if (from_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
   }
   if (to_is_local && to_vertex != from_vertex) {
     if (!PrepareForWrite(transaction_, to_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
-    if (to_vertex->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+    if (to_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
   }
 
   EdgeRef edge(gid);
   if (config_.properties_on_edges) {
-    auto acc = shard_->edges_.access();
     auto *delta = CreateDeleteObjectDelta(transaction_);
-    auto [it, inserted] = acc.insert(Edge(gid, delta));
+    auto [it, inserted] = shard_->edges_.emplace(gid, Edge{gid, delta});
     MG_ASSERT(inserted, "The edge must be inserted here!");
-    MG_ASSERT(it != acc.end(), "Invalid Edge accessor!");
-    edge = EdgeRef(&*it);
-    delta->prev.Set(&*it);
+    MG_ASSERT(it != shard_->edges_.end(), "Invalid Edge accessor!");
+    edge = EdgeRef(&it->second);
+    delta->prev.Set(&it->second);
   }
 
   if (from_is_local) {
     CreateAndLinkDelta(transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex_id, edge);
-    from_vertex->out_edges.emplace_back(edge_type, to_vertex_id, edge);
+    from_vertex->second.out_edges.emplace_back(edge_type, to_vertex_id, edge);
   }
   if (to_is_local) {
     CreateAndLinkDelta(transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex_id, edge);
-    to_vertex->in_edges.emplace_back(edge_type, from_vertex_id, edge);
+    to_vertex->second.in_edges.emplace_back(edge_type, from_vertex_id, edge);
   }
   // Increment edge count.
   ++shard_->edge_count_;
@@ -543,24 +540,24 @@ ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, V
 
 ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id,
                                                                      const Gid edge_id) {
-  Vertex *from_vertex{nullptr};
-  Vertex *to_vertex{nullptr};
+  VertexContainer::value_type *from_vertex{nullptr};
+  VertexContainer::value_type *to_vertex{nullptr};
 
-  auto acc = shard_->vertices_.access();
+  auto &vertices = shard_->vertices_;
 
   const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
   const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
 
   if (from_is_local) {
-    auto it = acc.find(from_vertex_id.primary_key);
-    MG_ASSERT(it != acc.end(), "Cannot find local vertex");
-    from_vertex = &it->vertex;
+    auto it = vertices.find(from_vertex_id.primary_key);
+    MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
+    from_vertex = &*it;
   }
 
   if (to_is_local) {
-    auto it = acc.find(to_vertex_id.primary_key);
-    MG_ASSERT(it != acc.end(), "Cannot find local vertex");
-    to_vertex = &it->vertex;
+    auto it = vertices.find(to_vertex_id.primary_key);
+    MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
+    to_vertex = &*it;
   }
 
   MG_ASSERT(from_is_local || to_is_local, "Trying to delete an edge without having a local vertex");
@@ -569,29 +566,28 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
     if (!PrepareForWrite(transaction_, from_vertex)) {
       return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
     }
-    MG_ASSERT(!from_vertex->deleted, "Invalid database state!");
+    MG_ASSERT(!from_vertex->second.deleted, "Invalid database state!");
   }
   if (to_is_local && to_vertex != from_vertex) {
     if (!PrepareForWrite(transaction_, to_vertex)) {
       return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
     }
-    MG_ASSERT(!to_vertex->deleted, "Invalid database state!");
+    MG_ASSERT(!to_vertex->second.deleted, "Invalid database state!");
   }
 
   const auto edge_ref = std::invoke([edge_id, this]() -> EdgeRef {
     if (!config_.properties_on_edges) {
       return EdgeRef(edge_id);
     }
-    auto edge_acc = shard_->edges_.access();
-    auto res = edge_acc.find(edge_id);
-    MG_ASSERT(res != edge_acc.end(), "Cannot find edge");
-    return EdgeRef(&*res);
+    auto res = shard_->edges_.find(edge_id);
+    MG_ASSERT(res != shard_->edges_.end(), "Cannot find edge");
+    return EdgeRef(&res->second);
   });
 
   std::optional<EdgeTypeId> edge_type{};
-  auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<Vertex::EdgeLink> &edges) mutable {
+  auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<VertexData::EdgeLink> &edges) mutable {
     auto it = std::find_if(edges.begin(), edges.end(),
-                           [&edge_ref](const Vertex::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
+                           [&edge_ref](const VertexData::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
     if (config_.properties_on_edges) {
       MG_ASSERT(it != edges.end(), "Invalid database state!");
     } else if (it == edges.end()) {
@@ -603,8 +599,8 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
     return true;
   };
   // NOLINTNEXTLINE(clang-analyzer-core.NonNullParamChecker)
-  auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->in_edges) : false;
-  auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->out_edges) : false;
+  auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->second.in_edges) : false;
+  auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->second.out_edges) : false;
 
   if (config_.properties_on_edges) {
     // Because of the check above, we are sure that the vertex exists.
@@ -630,10 +626,10 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
   MG_ASSERT(edge_type.has_value(), "Edge type is not determined");
 
   if (from_is_local) {
-    CreateAndLinkDelta(transaction_, from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
+    CreateAndLinkDelta(transaction_, &*from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
   }
   if (to_is_local) {
-    CreateAndLinkDelta(transaction_, to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
+    CreateAndLinkDelta(transaction_, &*to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
   }
 
   // Decrement edge count.
@@ -695,41 +691,41 @@ void Shard::Accessor::Abort() {
     auto prev = delta.prev.Get();
     switch (prev.type) {
       case PreviousPtr::Type::VERTEX: {
-        auto *vertex = prev.vertex;
-        Delta *current = vertex->delta;
+        auto &[pk, vertex] = *prev.vertex;
+        Delta *current = vertex.delta;
         while (current != nullptr && current->commit_info->start_or_commit_timestamp == transaction_->start_timestamp) {
           switch (current->action) {
             case Delta::Action::REMOVE_LABEL: {
-              auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
-              MG_ASSERT(it != vertex->labels.end(), "Invalid database state!");
-              std::swap(*it, *vertex->labels.rbegin());
-              vertex->labels.pop_back();
+              auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
+              MG_ASSERT(it != vertex.labels.end(), "Invalid database state!");
+              std::swap(*it, *vertex.labels.rbegin());
+              vertex.labels.pop_back();
               break;
             }
             case Delta::Action::ADD_LABEL: {
-              auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
-              MG_ASSERT(it == vertex->labels.end(), "Invalid database state!");
-              vertex->labels.push_back(current->label);
+              auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
+              MG_ASSERT(it == vertex.labels.end(), "Invalid database state!");
+              vertex.labels.push_back(current->label);
               break;
             }
             case Delta::Action::SET_PROPERTY: {
-              vertex->properties.SetProperty(current->property.key, current->property.value);
+              vertex.properties.SetProperty(current->property.key, current->property.value);
               break;
             }
             case Delta::Action::ADD_IN_EDGE: {
-              Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
-                                    current->vertex_edge.edge};
-              auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
-              MG_ASSERT(it == vertex->in_edges.end(), "Invalid database state!");
-              vertex->in_edges.push_back(link);
+              VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
+                                        current->vertex_edge.edge};
+              auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
+              MG_ASSERT(it == vertex.in_edges.end(), "Invalid database state!");
+              vertex.in_edges.push_back(link);
               break;
             }
             case Delta::Action::ADD_OUT_EDGE: {
-              Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
-                                    current->vertex_edge.edge};
-              auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
-              MG_ASSERT(it == vertex->out_edges.end(), "Invalid database state!");
-              vertex->out_edges.push_back(link);
+              VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
+                                        current->vertex_edge.edge};
+              auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
+              MG_ASSERT(it == vertex.out_edges.end(), "Invalid database state!");
+              vertex.out_edges.push_back(link);
               // Increment edge count. We only increment the count here because
               // the information in `ADD_IN_EDGE` and `Edge/RECREATE_OBJECT` is
               // redundant. Also, `Edge/RECREATE_OBJECT` isn't available when
@@ -738,21 +734,21 @@ void Shard::Accessor::Abort() {
               break;
             }
             case Delta::Action::REMOVE_IN_EDGE: {
-              Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
-                                    current->vertex_edge.edge};
-              auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
-              MG_ASSERT(it != vertex->in_edges.end(), "Invalid database state!");
-              std::swap(*it, *vertex->in_edges.rbegin());
-              vertex->in_edges.pop_back();
+              VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
+                                        current->vertex_edge.edge};
+              auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
+              MG_ASSERT(it != vertex.in_edges.end(), "Invalid database state!");
+              std::swap(*it, *vertex.in_edges.rbegin());
+              vertex.in_edges.pop_back();
               break;
             }
             case Delta::Action::REMOVE_OUT_EDGE: {
-              Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
-                                    current->vertex_edge.edge};
-              auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
-              MG_ASSERT(it != vertex->out_edges.end(), "Invalid database state!");
-              std::swap(*it, *vertex->out_edges.rbegin());
-              vertex->out_edges.pop_back();
+              VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
+                                        current->vertex_edge.edge};
+              auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
+              MG_ASSERT(it != vertex.out_edges.end(), "Invalid database state!");
+              std::swap(*it, *vertex.out_edges.rbegin());
+              vertex.out_edges.pop_back();
               // Decrement edge count. We only decrement the count here because
               // the information in `REMOVE_IN_EDGE` and `Edge/DELETE_OBJECT` is
               // redundant. Also, `Edge/DELETE_OBJECT` isn't available when edge
@@ -761,20 +757,20 @@ void Shard::Accessor::Abort() {
               break;
             }
             case Delta::Action::DELETE_OBJECT: {
-              vertex->deleted = true;
-              shard_->deleted_vertices_.push_back(vertex->keys.Keys());
+              vertex.deleted = true;
+              shard_->deleted_vertices_.push_back(&pk);
               break;
             }
             case Delta::Action::RECREATE_OBJECT: {
-              vertex->deleted = false;
+              vertex.deleted = false;
               break;
             }
           }
           current = current->next;
         }
-        vertex->delta = current;
+        vertex.delta = current;
         if (current != nullptr) {
-          current->prev.Set(vertex);
+          current->prev.Set(prev.vertex);
         }
 
         break;
@@ -851,10 +847,7 @@ const std::string &Shard::EdgeTypeToName(EdgeTypeId edge_type) const {
 
 bool Shard::CreateIndex(LabelId label, const std::optional<uint64_t> /*desired_commit_timestamp*/) {
   // TODO(jbajic) response should be different when label == primary_label
-  if (label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_.access())) {
-    return false;
-  }
-  return true;
+  return !(label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_));
 }
 
 bool Shard::CreateIndex(LabelId label, PropertyId property,
@@ -865,7 +858,7 @@ bool Shard::CreateIndex(LabelId label, PropertyId property,
     // Index already exists on primary key
     return false;
   }
-  if (!indices_.label_property_index.CreateIndex(label, property, vertices_.access())) {
+  if (!indices_.label_property_index.CreateIndex(label, property, vertices_)) {
     return false;
   }
   return true;
@@ -972,10 +965,12 @@ void Shard::CollectGarbage(const io::Time current_time) {
           auto prev = delta.prev.Get();
           switch (prev.type) {
             case PreviousPtr::Type::VERTEX: {
-              Vertex *vertex = prev.vertex;
-              vertex->delta = nullptr;
-              if (vertex->deleted) {
-                deleted_vertices_.push_back(vertex->keys.Keys());
+              // Here we need to get pk from prev pointer, and that is why change
+              // to the PrevPtr so it points to pair of pk and vertex
+              auto &[pk, vertex] = *prev.vertex;
+              vertex.delta = nullptr;
+              if (vertex.deleted) {
+                deleted_vertices_.push_back(&pk);
               }
               break;
             }
@@ -1022,15 +1017,13 @@ void Shard::CollectGarbage(const io::Time current_time) {
     RemoveObsoleteEntries(&indices_, clean_up_before_timestamp);
   }
 
-  auto vertex_acc = vertices_.access();
-  for (const auto &vertex : deleted_vertices_) {
-    MG_ASSERT(vertex_acc.remove(vertex), "Invalid database state!");
+  for (const auto *vertex : deleted_vertices_) {
+    MG_ASSERT(vertices_.erase(*vertex), "Invalid database state!");
   }
   deleted_vertices_.clear();
   {
-    auto edge_acc = edges_.access();
     for (auto edge : deleted_edges_) {
-      MG_ASSERT(edge_acc.remove(edge), "Invalid database state!");
+      MG_ASSERT(edges_.erase(edge), "Invalid database state!");
     }
   }
   deleted_edges_.clear();

src/storage/v3/shard.hpp

@@ -31,7 +31,6 @@
 #include "storage/v3/indices.hpp"
 #include "storage/v3/isolation_level.hpp"
 #include "storage/v3/key_store.hpp"
-#include "storage/v3/lexicographically_ordered_vertex.hpp"
 #include "storage/v3/mvcc.hpp"
 #include "storage/v3/name_id_mapper.hpp"
 #include "storage/v3/property_value.hpp"
@@ -42,7 +41,6 @@
 #include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "storage/v3/vertex_id.hpp"
-#include "storage/v3/vertices_skip_list.hpp"
 #include "storage/v3/view.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/file_locker.hpp"
@@ -65,7 +63,7 @@ namespace memgraph::storage::v3 {
 /// An instance of this will be usually be wrapped inside VerticesIterable for
 /// generic, public use.
 class AllVerticesIterable final {
-  VerticesSkipList::Accessor vertices_accessor_;
+  VertexContainer *vertices_accessor_;
   Transaction *transaction_;
   View view_;
   Indices *indices_;
@@ -76,10 +74,10 @@ class AllVerticesIterable final {
  public:
   class Iterator final {
     AllVerticesIterable *self_;
-    VerticesSkipList::Iterator it_;
+    VertexContainer::iterator it_;
 
    public:
-    Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it);
+    Iterator(AllVerticesIterable *self, VertexContainer::iterator it);
 
     VertexAccessor operator*() const;
 
@@ -90,17 +88,17 @@ class AllVerticesIterable final {
     bool operator!=(const Iterator &other) const { return !(*this == other); }
   };
 
-  AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view,
-                      Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
-      : vertices_accessor_(std::move(vertices_accessor)),
+  AllVerticesIterable(VertexContainer &vertices_accessor, Transaction *transaction, View view, Indices *indices,
+                      Config::Items config, const VertexValidator &vertex_validator)
+      : vertices_accessor_(&vertices_accessor),
         transaction_(transaction),
         view_(view),
         indices_(indices),
         config_(config),
         vertex_validator_{&vertex_validator} {}
 
-  Iterator begin() { return {this, vertices_accessor_.begin()}; }
-  Iterator end() { return {this, vertices_accessor_.end()}; }
+  Iterator begin() { return {this, vertices_accessor_->begin()}; }
+  Iterator end() { return {this, vertices_accessor_->end()}; }
 };
 
 /// Generic access to different kinds of vertex iterations.
@@ -213,7 +211,7 @@ class Shard final {
     std::optional<VertexAccessor> FindVertex(std::vector<PropertyValue> primary_key, View view);
 
     VerticesIterable Vertices(View view) {
-      return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), transaction_, view, &shard_->indices_,
+      return VerticesIterable(AllVerticesIterable(shard_->vertices_, transaction_, view, &shard_->indices_,
                                                   shard_->config_.items, shard_->vertex_validator_));
     }
 
@@ -236,18 +234,17 @@ class Shard final {
     int64_t ApproximateVertexCount(LabelId label) const {
       return shard_->indices_.label_index.ApproximateVertexCount(label);
     }
-
     /// Return approximate number of vertices with the given label and property.
     /// Note that this is always an over-estimate and never an under-estimate.
     int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
-      return shard_->indices_.label_property_index.ApproximateVertexCount(label, property);
+      return shard_->indices_.label_property_index.VertexCount(label, property);
     }
 
     /// Return approximate number of vertices with the given label and the given
-    /// value for the given property. Note that this is always an over-estimate
-    /// and never an under-estimate.
+    /// value for the given property.
+    /// Note that this is always an over-estimate and never an under-estimate.
     int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
-      return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
+      return shard_->indices_.label_property_index.VertexCount(label, property, value);
     }
 
     /// Return approximate number of vertices with the given label and value for
@@ -256,7 +253,7 @@ class Shard final {
     int64_t ApproximateVertexCount(LabelId label, PropertyId property,
                                    const std::optional<utils::Bound<PropertyValue>> &lower,
                                    const std::optional<utils::Bound<PropertyValue>> &upper) const {
-      return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
+      return shard_->indices_.label_property_index.VertexCount(label, property, lower, upper);
     }
 
     /// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
@@ -309,9 +306,6 @@ class Shard final {
     void Abort();
 
    private:
-    /// @throw std::bad_alloc
-    VertexAccessor CreateVertex(Gid gid, LabelId primary_label);
-
     Shard *shard_;
     Transaction *transaction_;
     Config::Items config_;
@@ -377,8 +371,8 @@ class Shard final {
   // The shard's range is [min, max)
   PrimaryKey min_primary_key_;
   std::optional<PrimaryKey> max_primary_key_;
-  VerticesSkipList vertices_;
-  utils::SkipList<Edge> edges_;
+  VertexContainer vertices_;
+  EdgeContainer edges_;
   // Even though the edge count is already kept in the `edges_` SkipList, the
   // list is used only when properties are enabled for edges. Because of that we
   // keep a separate count of edges that is always updated.
@@ -396,7 +390,7 @@ class Shard final {
 
   // Vertices that are logically deleted but still have to be removed from
   // indices before removing them from the main storage.
-  std::list<PrimaryKey> deleted_vertices_;
+  std::list<const PrimaryKey *> deleted_vertices_;
 
   // Edges that are logically deleted and wait to be removed from the main
   // storage.

src/storage/v3/shard_rsm.cpp

@@ -10,12 +10,16 @@
 // licenses/APL.txt.
 
 #include <algorithm>
+#include <exception>
+#include <experimental/source_location>
 #include <functional>
 #include <iterator>
 #include <optional>
 #include <unordered_set>
 #include <utility>
+#include <variant>
 
+#include "common/errors.hpp"
 #include "parser/opencypher/parser.hpp"
 #include "query/v2/requests.hpp"
 #include "storage/v2/vertex.hpp"
@@ -29,6 +33,7 @@
 #include "storage/v3/bindings/symbol_generator.hpp"
 #include "storage/v3/bindings/symbol_table.hpp"
 #include "storage/v3/bindings/typed_value.hpp"
+#include "storage/v3/conversions.hpp"
 #include "storage/v3/expr.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/key_store.hpp"
@@ -326,7 +331,7 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) {
     std::vector<Value> expression_results;
     if (!req.filter_expressions.empty()) {
       // NOTE - DbAccessor might get removed in the future.
-      const bool eval = FilterOnVertex(dba, vertex, req.filter_expressions, expr::identifier_node_symbol);
+      const bool eval = FilterOnVertex(dba, vertex, req.filter_expressions);
       if (!eval) {
         return;
       }
@@ -431,7 +436,7 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ExpandOneRequest &&req) {
     }
     if (!req.filters.empty()) {
       // NOTE - DbAccessor might get removed in the future.
-      const bool eval = FilterOnVertex(dba, src_vertex_acc_opt.value(), req.filters, expr::identifier_node_symbol);
+      const bool eval = FilterOnVertex(dba, src_vertex_acc_opt.value(), req.filters);
       if (!eval) {
         continue;
       }
@@ -510,9 +515,191 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CommitRequest &&req) {
   return msgs::CommitResponse{};
 };
 
-// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest && /*req*/) {
-  return msgs::GetPropertiesResponse{};
+msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest &&req) {
+  if (!req.vertex_ids.empty() && !req.vertices_and_edges.empty()) {
+    auto shard_error = SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
+    auto error = CreateErrorResponse(shard_error, req.transaction_id, "");
+    return msgs::GetPropertiesResponse{.error = {}};
+  }
+
+  auto shard_acc = shard_->Access(req.transaction_id);
+  auto dba = DbAccessor{&shard_acc};
+  const auto view = storage::v3::View::NEW;
+
+  auto transform_props = [](std::map<PropertyId, Value> &&value) {
+    std::vector<std::pair<PropertyId, Value>> result;
+    result.reserve(value.size());
+    for (auto &[id, val] : value) {
+      result.emplace_back(std::make_pair(id, std::move(val)));
+    }
+    return result;
+  };
+
+  auto collect_props = [&req](const VertexAccessor &v_acc,
+                              const std::optional<EdgeAccessor> &e_acc) -> ShardResult<std::map<PropertyId, Value>> {
+    if (!req.property_ids) {
+      if (e_acc) {
+        return CollectAllPropertiesFromAccessor(*e_acc, view);
+      }
+      return CollectAllPropertiesFromAccessor(v_acc, view);
+    }
+
+    if (e_acc) {
+      return CollectSpecificPropertiesFromAccessor(*e_acc, *req.property_ids, view);
+    }
+    return CollectSpecificPropertiesFromAccessor(v_acc, *req.property_ids, view);
+  };
+
+  auto find_edge = [](const VertexAccessor &v, msgs::EdgeId e) -> std::optional<EdgeAccessor> {
+    auto in = v.InEdges(view);
+    MG_ASSERT(in.HasValue());
+    for (auto &edge : in.GetValue()) {
+      if (edge.Gid().AsUint() == e.gid) {
+        return edge;
+      }
+    }
+    auto out = v.OutEdges(view);
+    MG_ASSERT(out.HasValue());
+    for (auto &edge : out.GetValue()) {
+      if (edge.Gid().AsUint() == e.gid) {
+        return edge;
+      }
+    }
+    return std::nullopt;
+  };
+
+  const auto has_expr_to_evaluate = !req.expressions.empty();
+  auto emplace_result_row =
+      [dba, transform_props, collect_props, has_expr_to_evaluate, &req](
+          const VertexAccessor &v_acc,
+          const std::optional<EdgeAccessor> e_acc) mutable -> ShardResult<msgs::GetPropertiesResultRow> {
+    auto maybe_id = v_acc.Id(view);
+    if (maybe_id.HasError()) {
+      return {maybe_id.GetError()};
+    }
+    const auto &id = maybe_id.GetValue();
+    std::optional<msgs::EdgeId> e_id;
+    if (e_acc) {
+      e_id = msgs::EdgeId{e_acc->Gid().AsUint()};
+    }
+    msgs::VertexId v_id{msgs::Label{id.primary_label}, ConvertValueVector(id.primary_key)};
+    auto maybe_props = collect_props(v_acc, e_acc);
+    if (maybe_props.HasError()) {
+      return {maybe_props.GetError()};
+    }
+    auto props = transform_props(std::move(maybe_props.GetValue()));
+    auto result = msgs::GetPropertiesResultRow{.vertex = std::move(v_id), .edge = e_id, .props = std::move(props)};
+    if (has_expr_to_evaluate) {
+      std::vector<Value> e_results;
+      if (e_acc) {
+        e_results =
+            ConvertToValueVectorFromTypedValueVector(EvaluateEdgeExpressions(dba, v_acc, *e_acc, req.expressions));
+      } else {
+        e_results = ConvertToValueVectorFromTypedValueVector(
+            EvaluateVertexExpressions(dba, v_acc, req.expressions, expr::identifier_node_symbol));
+      }
+      result.evaluated_expressions = std::move(e_results);
+    }
+    return {std::move(result)};
+  };
+
+  auto get_limit = [&req](const auto &elements) {
+    size_t limit = elements.size();
+    if (req.limit && *req.limit < elements.size()) {
+      limit = *req.limit;
+    }
+    return limit;
+  };
+
+  auto collect_response = [get_limit, &req](auto &elements, auto create_result_row) -> msgs::ReadResponses {
+    msgs::GetPropertiesResponse response;
+    const auto limit = get_limit(elements);
+    for (size_t index = 0; index != limit; ++index) {
+      auto result_row = create_result_row(elements[index]);
+      if (result_row.HasError()) {
+        return msgs::GetPropertiesResponse{.error = CreateErrorResponse(result_row.GetError(), req.transaction_id, "")};
+      }
+      response.result_row.push_back(std::move(result_row.GetValue()));
+    }
+    return response;
+  };
+
+  std::vector<VertexAccessor> vertices;
+  std::vector<EdgeAccessor> edges;
+
+  auto parse_and_filter = [dba, &vertices](auto &container, auto projection, auto filter, auto maybe_get_edge) mutable {
+    for (const auto &elem : container) {
+      const auto &[label, pk_v] = projection(elem);
+      auto pk = ConvertPropertyVector(pk_v);
+      auto v_acc = dba.FindVertex(pk, view);
+      if (!v_acc || filter(*v_acc, maybe_get_edge(elem))) {
+        continue;
+      }
+      vertices.push_back(*v_acc);
+    }
+  };
+  auto identity = [](auto &elem) { return elem; };
+
+  auto filter_vertex = [dba, req](const auto &acc, const auto & /*edge*/) mutable {
+    if (!req.filter) {
+      return false;
+    }
+    return !FilterOnVertex(dba, acc, {*req.filter});
+  };
+
+  auto filter_edge = [dba, &edges, &req, find_edge](const auto &acc, const auto &edge) mutable {
+    auto e_acc = find_edge(acc, edge);
+    if (!e_acc) {
+      return true;
+    }
+
+    if (req.filter && !FilterOnEdge(dba, acc, *e_acc, {*req.filter})) {
+      return true;
+    }
+    edges.push_back(*e_acc);
+    return false;
+  };
+
+  // Handler logic here
+  if (!req.vertex_ids.empty()) {
+    parse_and_filter(req.vertex_ids, identity, filter_vertex, identity);
+  } else {
+    parse_and_filter(
+        req.vertices_and_edges, [](auto &e) { return e.first; }, filter_edge, [](auto &e) { return e.second; });
+  }
+
+  if (!req.vertex_ids.empty()) {
+    if (!req.order_by.empty()) {
+      auto elements = OrderByVertices(dba, vertices, req.order_by);
+      return collect_response(elements, [emplace_result_row](auto &element) mutable {
+        return emplace_result_row(element.object_acc, std::nullopt);
+      });
+    }
+    return collect_response(vertices,
+                            [emplace_result_row](auto &acc) mutable { return emplace_result_row(acc, std::nullopt); });
+  }
+
+  if (!req.order_by.empty()) {
+    auto elements = OrderByEdges(dba, edges, req.order_by, vertices);
+    return collect_response(elements, [emplace_result_row](auto &element) mutable {
+      return emplace_result_row(element.object_acc.first, element.object_acc.second);
+    });
+  }
+
+  struct ZipView {
+    ZipView(std::vector<VertexAccessor> &v, std::vector<EdgeAccessor> &e) : v(v), e(e) {}
+    size_t size() const { return v.size(); }
+    auto operator[](size_t index) { return std::make_pair(v[index], e[index]); }
+
+   private:
+    std::vector<VertexAccessor> &v;
+    std::vector<EdgeAccessor> &e;
+  };
+
+  ZipView vertices_and_edges(vertices, edges);
+  return collect_response(vertices_and_edges, [emplace_result_row](const auto &acc) mutable {
+    return emplace_result_row(acc.first, acc.second);
+  });
 }
 
 }  // namespace memgraph::storage::v3

src/storage/v3/vertex.cpp

@@ -9,11 +9,22 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-#pragma once
+#include "storage/v3/vertex.hpp"
 
-#include "storage/v3/lexicographically_ordered_vertex.hpp"
-#include "utils/skip_list.hpp"
+#include <limits>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+
+#include "storage/v3/delta.hpp"
 
 namespace memgraph::storage::v3 {
-using VerticesSkipList = utils::SkipList<LexicographicallyOrderedVertex>;
+
+VertexData::VertexData(Delta *delta) : delta{delta} {
+  MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
+            "Vertex must be created with an initial DELETE_OBJECT delta!");
+}
+
+bool VertexHasLabel(const Vertex &vertex, const LabelId label) { return utils::Contains(vertex.second.labels, label); }
+
 }  // namespace memgraph::storage::v3

src/storage/v3/vertex.hpp

@@ -16,7 +16,6 @@
 #include <type_traits>
 #include <vector>
 
-#include "storage/v3/delta.hpp"
 #include "storage/v3/edge_ref.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/key_store.hpp"
@@ -28,17 +27,12 @@
 
 namespace memgraph::storage::v3 {
 
-struct Vertex {
+struct Delta;
+
+struct VertexData {
   using EdgeLink = std::tuple<EdgeTypeId, VertexId, EdgeRef>;
 
-  Vertex(Delta *delta, const std::vector<PropertyValue> &primary_properties) : keys{primary_properties}, delta{delta} {
-    MG_ASSERT(delta == nullptr || delta->action == Delta::Action::DELETE_OBJECT,
-              "Vertex must be created with an initial DELETE_OBJECT delta!");
-  }
-
-  friend bool operator==(const Vertex &vertex, const PrimaryKey &primary_key) { return vertex.keys == primary_key; }
-
-  KeyStore keys;
+  explicit VertexData(Delta *delta);
 
   std::vector<LabelId> labels;
   PropertyStore properties;
@@ -52,8 +46,11 @@ struct Vertex {
   Delta *delta;
 };
 
-static_assert(alignof(Vertex) >= 8, "The Vertex should be aligned to at least 8!");
+static_assert(alignof(VertexData) >= 8, "The Vertex should be aligned to at least 8!");
 
-inline bool VertexHasLabel(const Vertex &vertex, const LabelId label) { return utils::Contains(vertex.labels, label); }
+using VertexContainer = std::map<PrimaryKey, VertexData>;
+using Vertex = VertexContainer::value_type;
+
+bool VertexHasLabel(const Vertex &vertex, LabelId label);
 
 }  // namespace memgraph::storage::v3

src/storage/v3/vertex_accessor.cpp

@@ -36,8 +36,8 @@ std::pair<bool, bool> IsVisible(Vertex *vertex, Transaction *transaction, View v
   bool deleted = false;
   Delta *delta = nullptr;
   {
-    deleted = vertex->deleted;
-    delta = vertex->delta;
+    deleted = vertex->second.deleted;
+    delta = vertex->second.delta;
   }
   ApplyDeltasForRead(transaction, delta, view, [&](const Delta &delta) {
     switch (delta.action) {
@@ -85,13 +85,14 @@ ShardResult<bool> VertexAccessor::AddLabel(LabelId label) {
 
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  if (std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end()) return false;
+  if (std::find(vertex_->second.labels.begin(), vertex_->second.labels.end(), label) != vertex_->second.labels.end())
+    return false;
 
   CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label);
 
-  vertex_->labels.push_back(label);
+  vertex_->second.labels.push_back(label);
 
   UpdateOnAddLabel(indices_, label, vertex_, *transaction_);
 
@@ -106,13 +107,14 @@ ShardResult<bool> VertexAccessor::AddLabelAndValidate(LabelId label) {
 
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  if (std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end()) return false;
+  if (std::find(vertex_->second.labels.begin(), vertex_->second.labels.end(), label) != vertex_->second.labels.end())
+    return false;
 
   CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label);
 
-  vertex_->labels.push_back(label);
+  vertex_->second.labels.push_back(label);
 
   UpdateOnAddLabel(indices_, label, vertex_, *transaction_);
 
@@ -122,15 +124,15 @@ ShardResult<bool> VertexAccessor::AddLabelAndValidate(LabelId label) {
 ShardResult<bool> VertexAccessor::RemoveLabel(LabelId label) {
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  auto it = std::find(vertex_->labels.begin(), vertex_->labels.end(), label);
-  if (it == vertex_->labels.end()) return false;
+  auto it = std::find(vertex_->second.labels.begin(), vertex_->second.labels.end(), label);
+  if (it == vertex_->second.labels.end()) return false;
 
   CreateAndLinkDelta(transaction_, vertex_, Delta::AddLabelTag(), label);
 
-  std::swap(*it, *vertex_->labels.rbegin());
-  vertex_->labels.pop_back();
+  std::swap(*it, *vertex_->second.labels.rbegin());
+  vertex_->second.labels.pop_back();
   return true;
 }
 
@@ -142,15 +144,15 @@ ShardResult<bool> VertexAccessor::RemoveLabelAndValidate(LabelId label) {
 
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  auto it = std::find(vertex_->labels.begin(), vertex_->labels.end(), label);
-  if (it == vertex_->labels.end()) return false;
+  auto it = std::find(vertex_->second.labels.begin(), vertex_->second.labels.end(), label);
+  if (it == vertex_->second.labels.end()) return false;
 
   CreateAndLinkDelta(transaction_, vertex_, Delta::AddLabelTag(), label);
 
-  std::swap(*it, *vertex_->labels.rbegin());
-  vertex_->labels.pop_back();
+  std::swap(*it, *vertex_->second.labels.rbegin());
+  vertex_->second.labels.pop_back();
   return true;
 }
 
@@ -162,9 +164,9 @@ ShardResult<bool> VertexAccessor::HasLabel(LabelId label, View view) const {
   bool has_label = false;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
+    deleted = vertex_->second.deleted;
     has_label = label == vertex_validator_->primary_label_ || VertexHasLabel(*vertex_, label);
-    delta = vertex_->delta;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &has_label, label](const Delta &delta) {
     switch (delta.action) {
@@ -215,14 +217,14 @@ ShardResult<PrimaryKey> VertexAccessor::PrimaryKey(const View view) const {
   if (const auto result = CheckVertexExistence(view); result.HasError()) {
     return result.GetError();
   }
-  return vertex_->keys.Keys();
+  return vertex_->first;
 }
 
 ShardResult<VertexId> VertexAccessor::Id(View view) const {
   if (const auto result = CheckVertexExistence(view); result.HasError()) {
     return result.GetError();
   }
-  return VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
+  return VertexId{vertex_validator_->primary_label_, vertex_->first};
 };
 
 ShardResult<std::vector<LabelId>> VertexAccessor::Labels(View view) const {
@@ -231,9 +233,9 @@ ShardResult<std::vector<LabelId>> VertexAccessor::Labels(View view) const {
   std::vector<LabelId> labels;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
-    labels = vertex_->labels;
-    delta = vertex_->delta;
+    deleted = vertex_->second.deleted;
+    labels = vertex_->second.labels;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &labels](const Delta &delta) {
     switch (delta.action) {
@@ -278,9 +280,9 @@ ShardResult<PropertyValue> VertexAccessor::SetProperty(PropertyId property, cons
 
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  auto current_value = vertex_->properties.GetProperty(property);
+  auto current_value = vertex_->second.properties.GetProperty(property);
   // We could skip setting the value if the previous one is the same to the new
   // one. This would save some memory as a delta would not be created as well as
   // avoid copying the value. The reason we are not doing that is because the
@@ -288,7 +290,7 @@ ShardResult<PropertyValue> VertexAccessor::SetProperty(PropertyId property, cons
   // "modify in-place". Additionally, the created delta will make other
   // transactions get a SERIALIZATION_ERROR.
   CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, current_value);
-  vertex_->properties.SetProperty(property, value);
+  vertex_->second.properties.SetProperty(property, value);
 
   UpdateOnSetProperty(indices_, property, value, vertex_, *transaction_);
 
@@ -300,8 +302,8 @@ ShardResult<void> VertexAccessor::CheckVertexExistence(View view) const {
   bool deleted = false;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
-    delta = vertex_->delta;
+    deleted = vertex_->second.deleted;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted](const Delta &delta) {
     switch (delta.action) {
@@ -343,11 +345,11 @@ ShardResult<PropertyValue> VertexAccessor::SetPropertyAndValidate(PropertyId pro
     return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
   }
 
-  if (vertex_->deleted) {
+  if (vertex_->second.deleted) {
     return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
   }
 
-  auto current_value = vertex_->properties.GetProperty(property);
+  auto current_value = vertex_->second.properties.GetProperty(property);
   // We could skip setting the value if the previous one is the same to the new
   // one. This would save some memory as a delta would not be created as well as
   // avoid copying the value. The reason we are not doing that is because the
@@ -355,7 +357,7 @@ ShardResult<PropertyValue> VertexAccessor::SetPropertyAndValidate(PropertyId pro
   // "modify in-place". Additionally, the created delta will make other
   // transactions get a SERIALIZATION_ERROR.
   CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, current_value);
-  vertex_->properties.SetProperty(property, value);
+  vertex_->second.properties.SetProperty(property, value);
 
   UpdateOnSetProperty(indices_, property, value, vertex_, *transaction_);
 
@@ -365,15 +367,15 @@ ShardResult<PropertyValue> VertexAccessor::SetPropertyAndValidate(PropertyId pro
 ShardResult<std::map<PropertyId, PropertyValue>> VertexAccessor::ClearProperties() {
   if (!PrepareForWrite(transaction_, vertex_)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (vertex_->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (vertex_->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
-  auto properties = vertex_->properties.Properties();
+  auto properties = vertex_->second.properties.Properties();
   for (const auto &property : properties) {
     CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property.first, property.second);
     UpdateOnSetProperty(indices_, property.first, PropertyValue(), vertex_, *transaction_);
   }
 
-  vertex_->properties.ClearProperties();
+  vertex_->second.properties.ClearProperties();
 
   return std::move(properties);
 }
@@ -394,18 +396,13 @@ PropertyValue VertexAccessor::GetPropertyValue(PropertyId property, View view) c
     return value;
   }
   // Find PropertyId index in keystore
-  size_t property_index{0};
-  for (; property_index < schema->second.size(); ++property_index) {
+  for (size_t property_index{0}; property_index < schema->second.size(); ++property_index) {
     if (schema->second[property_index].property_id == property) {
-      break;
+      return vertex_->first[property_index];
     }
   }
 
-  value = vertex_->keys.GetKey(property_index);
-  if (value.IsNull()) {
-    value = vertex_->properties.GetProperty(property);
-  }
-  return value;
+  return value = vertex_->second.properties.GetProperty(property);
 }
 
 ShardResult<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View view) const {
@@ -414,9 +411,9 @@ ShardResult<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View
   PropertyValue value;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
+    deleted = vertex_->second.deleted;
     value = GetPropertyValue(property, view);
-    delta = vertex_->delta;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &value, property](const Delta &delta) {
     switch (delta.action) {
@@ -443,8 +440,12 @@ ShardResult<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View
         break;
     }
   });
-  if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
-  if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  if (!exists) {
+    return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
+  }
+  if (!for_deleted_ && deleted) {
+    return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
+  }
   return std::move(value);
 }
 
@@ -454,10 +455,10 @@ ShardResult<std::map<PropertyId, PropertyValue>> VertexAccessor::Properties(View
   std::map<PropertyId, PropertyValue> properties;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
+    deleted = vertex_->second.deleted;
     // TODO(antaljanosbenjamin): Should this also return the primary key?
-    properties = vertex_->properties.Properties();
-    delta = vertex_->delta;
+    properties = vertex_->second.properties.Properties();
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
     switch (delta.action) {
@@ -502,14 +503,14 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const
                                                                const VertexId *destination_id) const {
   bool exists = true;
   bool deleted = false;
-  std::vector<Vertex::EdgeLink> in_edges;
+  std::vector<VertexData::EdgeLink> in_edges;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
+    deleted = vertex_->second.deleted;
     if (edge_types.empty() && nullptr == destination_id) {
-      in_edges = vertex_->in_edges;
+      in_edges = vertex_->second.in_edges;
     } else {
-      for (const auto &item : vertex_->in_edges) {
+      for (const auto &item : vertex_->second.in_edges) {
         const auto &[edge_type, from_vertex, edge] = item;
         if (nullptr != destination_id && from_vertex != *destination_id) {
           continue;
@@ -519,7 +520,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const
         in_edges.push_back(item);
       }
     }
-    delta = vertex_->delta;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(
       transaction_, delta, view, [&exists, &deleted, &in_edges, &edge_types, destination_id](const Delta &delta) {
@@ -530,7 +531,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const
                 std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
               break;
             // Add the edge because we don't see the removal.
-            Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
+            VertexData::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
             auto it = std::find(in_edges.begin(), in_edges.end(), link);
             MG_ASSERT(it == in_edges.end(), "Invalid database state!");
             in_edges.push_back(link);
@@ -542,7 +543,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const
                 std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
               break;
             // Remove the label because we don't see the addition.
-            Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
+            VertexData::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
             auto it = std::find(in_edges.begin(), in_edges.end(), link);
             MG_ASSERT(it != in_edges.end(), "Invalid database state!");
             std::swap(*it, *in_edges.rbegin());
@@ -572,7 +573,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::InEdges(View view, const
     return ret;
   }
   ret.reserve(in_edges.size());
-  const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
+  const auto id = VertexId{vertex_validator_->primary_label_, vertex_->first};
   for (const auto &item : in_edges) {
     const auto &[edge_type, from_vertex, edge] = item;
     ret.emplace_back(edge, edge_type, from_vertex, id, transaction_, indices_, config_);
@@ -584,14 +585,14 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const
                                                                 const VertexId *destination_id) const {
   bool exists = true;
   bool deleted = false;
-  std::vector<Vertex::EdgeLink> out_edges;
+  std::vector<VertexData::EdgeLink> out_edges;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
+    deleted = vertex_->second.deleted;
     if (edge_types.empty() && nullptr == destination_id) {
-      out_edges = vertex_->out_edges;
+      out_edges = vertex_->second.out_edges;
     } else {
-      for (const auto &item : vertex_->out_edges) {
+      for (const auto &item : vertex_->second.out_edges) {
         const auto &[edge_type, to_vertex, edge] = item;
         if (nullptr != destination_id && to_vertex != *destination_id) continue;
         if (!edge_types.empty() && std::find(edge_types.begin(), edge_types.end(), edge_type) == edge_types.end())
@@ -599,7 +600,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const
         out_edges.push_back(item);
       }
     }
-    delta = vertex_->delta;
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(
       transaction_, delta, view, [&exists, &deleted, &out_edges, &edge_types, destination_id](const Delta &delta) {
@@ -610,7 +611,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const
                 std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
               break;
             // Add the edge because we don't see the removal.
-            Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
+            VertexData::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
             auto it = std::find(out_edges.begin(), out_edges.end(), link);
             MG_ASSERT(it == out_edges.end(), "Invalid database state!");
             out_edges.push_back(link);
@@ -622,7 +623,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const
                 std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
               break;
             // Remove the label because we don't see the addition.
-            Vertex::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
+            VertexData::EdgeLink link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id, delta.vertex_edge.edge};
             auto it = std::find(out_edges.begin(), out_edges.end(), link);
             MG_ASSERT(it != out_edges.end(), "Invalid database state!");
             std::swap(*it, *out_edges.rbegin());
@@ -652,7 +653,7 @@ ShardResult<std::vector<EdgeAccessor>> VertexAccessor::OutEdges(View view, const
     return ret;
   }
   ret.reserve(out_edges.size());
-  const auto id = VertexId{vertex_validator_->primary_label_, vertex_->keys.Keys()};
+  const auto id = VertexId{vertex_validator_->primary_label_, vertex_->first};
   for (const auto &item : out_edges) {
     const auto &[edge_type, to_vertex, edge] = item;
     ret.emplace_back(edge, edge_type, id, to_vertex, transaction_, indices_, config_);
@@ -666,9 +667,9 @@ ShardResult<size_t> VertexAccessor::InDegree(View view) const {
   size_t degree = 0;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
-    degree = vertex_->in_edges.size();
-    delta = vertex_->delta;
+    deleted = vertex_->second.deleted;
+    degree = vertex_->second.in_edges.size();
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &degree](const Delta &delta) {
     switch (delta.action) {
@@ -703,9 +704,9 @@ ShardResult<size_t> VertexAccessor::OutDegree(View view) const {
   size_t degree = 0;
   Delta *delta = nullptr;
   {
-    deleted = vertex_->deleted;
-    degree = vertex_->out_edges.size();
-    delta = vertex_->delta;
+    deleted = vertex_->second.deleted;
+    degree = vertex_->second.out_edges.size();
+    delta = vertex_->second.delta;
   }
   ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &degree](const Delta &delta) {
     switch (delta.action) {

tests/benchmark/CMakeLists.txt

@@ -9,11 +9,13 @@ function(add_benchmark test_cpp)
   get_filename_component(exec_name ${test_cpp} NAME_WE)
   set(target_name ${test_prefix}${exec_name})
   add_executable(${target_name} ${test_cpp} ${ARGN})
+
   # OUTPUT_NAME sets the real name of a target when it is built and can be
   # used to help create two targets of the same name even though CMake
   # requires unique logical target names
   set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name})
   target_link_libraries(${target_name} benchmark gflags)
+
   # register test
   add_test(${target_name} ${exec_name})
   add_dependencies(memgraph__benchmark ${target_name})
@@ -37,9 +39,9 @@ target_link_libraries(${test_prefix}profile mg-query)
 add_benchmark(query/stripped.cpp)
 target_link_libraries(${test_prefix}stripped mg-query)
 
-if (MG_ENTERPRISE)
-add_benchmark(rpc.cpp)
-target_link_libraries(${test_prefix}rpc mg-rpc)
+if(MG_ENTERPRISE)
+  add_benchmark(rpc.cpp)
+  target_link_libraries(${test_prefix}rpc mg-rpc)
 endif()
 
 add_benchmark(skip_list_random.cpp)
@@ -65,3 +67,15 @@ target_link_libraries(${test_prefix}storage_v2_property_store mg-storage-v2)
 
 add_benchmark(future.cpp)
 target_link_libraries(${test_prefix}future mg-io)
+
+add_benchmark(data_structures_insert.cpp)
+target_link_libraries(${test_prefix}data_structures_insert mg-utils mg-storage-v3)
+
+add_benchmark(data_structures_find.cpp)
+target_link_libraries(${test_prefix}data_structures_find mg-utils mg-storage-v3)
+
+add_benchmark(data_structures_contains.cpp)
+target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage-v3)
+
+add_benchmark(data_structures_remove.cpp)
+target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3)

tests/benchmark/data_structures_common.hpp

@@ -0,0 +1,56 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <vector>
+
+#include "coordinator/hybrid_logical_clock.hpp"
+#include "storage/v3/key_store.hpp"
+#include "storage/v3/mvcc.hpp"
+#include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
+#include "utils/skip_list.hpp"
+
+namespace memgraph::benchmark {
+
+template <typename T>
+inline void PrepareData(utils::SkipList<T> &skip_list, const int64_t num_elements) {
+  coordinator::Hlc start_timestamp;
+  storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
+  for (auto i{0}; i < num_elements; ++i) {
+    auto acc = skip_list.access();
+    acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}});
+  }
+}
+
+template <typename TKey, typename TValue>
+inline void PrepareData(std::map<TKey, TValue> &std_map, const int64_t num_elements) {
+  coordinator::Hlc start_timestamp;
+  storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
+  auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction);
+  for (auto i{0}; i < num_elements; ++i) {
+    std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, storage::v3::VertexData{delta}});
+  }
+}
+
+template <typename T>
+inline void PrepareData(std::set<T> &std_set, const int64_t num_elements) {
+  coordinator::Hlc start_timestamp;
+  storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
+  for (auto i{0}; i < num_elements; ++i) {
+    std_set.insert(std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}});
+  }
+}
+
+}  // namespace memgraph::benchmark

tests/benchmark/data_structures_contains.cpp

@@ -0,0 +1,104 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <atomic>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <gflags/gflags.h>
+
+#include "data_structures_common.hpp"
+#include "storage/v3/key_store.hpp"
+#include "storage/v3/mvcc.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
+#include "utils/skip_list.hpp"
+
+namespace memgraph::benchmark {
+
+///////////////////////////////////////////////////////////////////////////////
+// Testing Contains Operation
+///////////////////////////////////////////////////////////////////////////////
+static void BM_BenchmarkContainsSkipList(::benchmark::State &state) {
+  utils::SkipList<storage::v3::PrimaryKey> skip_list;
+  PrepareData(skip_list, state.range(0));
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      auto acc = skip_list.access();
+      if (acc.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+static void BM_BenchmarkContainsStdMap(::benchmark::State &state) {
+  std::map<storage::v3::PrimaryKey, storage::v3::VertexData> std_map;
+  PrepareData(std_map, state.range(0));
+
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_map.contains(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}})) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+static void BM_BenchmarkContainsStdSet(::benchmark::State &state) {
+  std::set<storage::v3::PrimaryKey> std_set;
+  PrepareData(std_set, state.range(0));
+
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_set.contains(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}})) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+BENCHMARK(BM_BenchmarkContainsSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkContainsStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkContainsStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+}  // namespace memgraph::benchmark
+
+BENCHMARK_MAIN();

tests/benchmark/data_structures_find.cpp

@@ -0,0 +1,103 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <atomic>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <gflags/gflags.h>
+
+#include "data_structures_common.hpp"
+#include "storage/v3/key_store.hpp"
+#include "storage/v3/mvcc.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
+#include "utils/skip_list.hpp"
+
+namespace memgraph::benchmark {
+
+///////////////////////////////////////////////////////////////////////////////
+// Testing Find Operation
+///////////////////////////////////////////////////////////////////////////////
+static void BM_BenchmarkFindSkipList(::benchmark::State &state) {
+  utils::SkipList<storage::v3::PrimaryKey> skip_list;
+  PrepareData(skip_list, state.range(0));
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      auto acc = skip_list.access();
+      if (acc.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != acc.end()) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+static void BM_BenchmarkFindStdMap(::benchmark::State &state) {
+  std::map<storage::v3::PrimaryKey, storage::v3::VertexData> std_map;
+  PrepareData(std_map, state.range(0));
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_map.find(storage::v3::PrimaryKey{{storage::v3::PropertyValue(value)}}) != std_map.end()) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+static void BM_BenchmarkFindStdSet(::benchmark::State &state) {
+  std::set<storage::v3::PrimaryKey> std_set;
+  PrepareData(std_set, state.range(0));
+
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t found_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_set.find(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) != std_set.end()) {
+        found_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(found_elems);
+}
+
+BENCHMARK(BM_BenchmarkFindSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkFindStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkFindStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+}  // namespace memgraph::benchmark
+
+BENCHMARK_MAIN();

tests/benchmark/data_structures_insert.cpp

@@ -0,0 +1,82 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <atomic>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <gflags/gflags.h>
+
+#include "storage/v3/key_store.hpp"
+#include "storage/v3/mvcc.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
+#include "utils/skip_list.hpp"
+
+namespace memgraph::benchmark {
+
+///////////////////////////////////////////////////////////////////////////////
+// Testing Insert Operation
+///////////////////////////////////////////////////////////////////////////////
+static void BM_BenchmarkInsertSkipList(::benchmark::State &state) {
+  utils::SkipList<storage::v3::PrimaryKey> skip_list;
+  coordinator::Hlc start_timestamp;
+  storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
+
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      auto acc = skip_list.access();
+      acc.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{true}}});
+    }
+  }
+}
+
+static void BM_BenchmarkInsertStdMap(::benchmark::State &state) {
+  std::map<storage::v3::PrimaryKey, storage::v3::VertexData> std_map;
+  coordinator::Hlc start_timestamp;
+  storage::v3::Transaction transaction{start_timestamp, storage::v3::IsolationLevel::SNAPSHOT_ISOLATION};
+  auto *delta = storage::v3::CreateDeleteObjectDelta(&transaction);
+
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      std_map.insert({storage::v3::PrimaryKey{storage::v3::PropertyValue{i}}, storage::v3::VertexData{delta}});
+    }
+  }
+}
+
+static void BM_BenchmarkInsertStdSet(::benchmark::State &state) {
+  std::set<storage::v3::PrimaryKey> std_set;
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      std_set.insert(
+          storage::v3::PrimaryKey{std::vector<storage::v3::PropertyValue>{storage::v3::PropertyValue{true}}});
+    }
+  }
+}
+
+BENCHMARK(BM_BenchmarkInsertSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkInsertStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkInsertStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+}  // namespace memgraph::benchmark
+
+BENCHMARK_MAIN();

tests/benchmark/data_structures_remove.cpp

@@ -0,0 +1,105 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <atomic>
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <gflags/gflags.h>
+
+#include "data_structures_common.hpp"
+#include "storage/v3/key_store.hpp"
+#include "storage/v3/mvcc.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
+#include "utils/skip_list.hpp"
+
+namespace memgraph::benchmark {
+
+///////////////////////////////////////////////////////////////////////////////
+// Testing Remove Operation
+///////////////////////////////////////////////////////////////////////////////
+static void BM_BenchmarkRemoveSkipList(::benchmark::State &state) {
+  utils::SkipList<storage::v3::PrimaryKey> skip_list;
+  PrepareData(skip_list, state.range(0));
+
+  // So we can also have elements that don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t removed_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      auto acc = skip_list.access();
+      if (acc.remove(storage::v3::PrimaryKey{storage::v3::PropertyValue(value)})) {
+        removed_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(removed_elems);
+}
+
+static void BM_BenchmarkRemoveStdMap(::benchmark::State &state) {
+  std::map<storage::v3::PrimaryKey, storage::v3::VertexData> std_map;
+  PrepareData(std_map, state.range(0));
+
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t removed_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_map.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) {
+        removed_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(removed_elems);
+}
+
+static void BM_BenchmarkRemoveStdSet(::benchmark::State &state) {
+  std::set<storage::v3::PrimaryKey> std_set;
+  PrepareData(std_set, state.range(0));
+
+  // So we can also have elements that does don't exist
+  std::mt19937 i_generator(std::random_device{}());
+  std::uniform_int_distribution<int64_t> i_distribution(0, state.range(0) * 2);
+  int64_t removed_elems{0};
+  for (auto _ : state) {
+    for (auto i{0}; i < state.range(0); ++i) {
+      int64_t value = i_distribution(i_generator);
+      if (std_set.erase(storage::v3::PrimaryKey{storage::v3::PropertyValue{value}}) > 0) {
+        removed_elems++;
+      }
+    }
+  }
+  state.SetItemsProcessed(removed_elems);
+}
+
+BENCHMARK(BM_BenchmarkRemoveSkipList)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkRemoveStdMap)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+BENCHMARK(BM_BenchmarkRemoveStdSet)->RangeMultiplier(10)->Range(1000, 10000000)->Unit(::benchmark::kMillisecond);
+
+}  // namespace memgraph::benchmark
+
+BENCHMARK_MAIN();

tests/benchmark/skip_list_common.hpp

@@ -1,4 +1,4 @@
-// Copyright 2021 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,11 +11,14 @@
 
 #pragma once
 
+#include <array>
 #include <atomic>
 #include <chrono>
+#include <cstdint>
 #include <functional>
 #include <iostream>
 #include <memory>
+#include <numeric>
 #include <thread>
 #include <vector>
 
@@ -26,7 +29,7 @@ DEFINE_int32(duration, 10, "Duration of test (in seconds)");
 
 struct Stats {
   uint64_t total{0};
-  uint64_t succ[4] = {0, 0, 0, 0};
+  std::array<uint64_t, 4> succ = {0, 0, 0, 0};
 };
 
 const int OP_INSERT = 0;
@@ -81,3 +84,27 @@ inline void RunConcurrentTest(std::function<void(std::atomic<bool> *, Stats *)>
   std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl;
   std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl;
 }
+
+inline void RunTest(std::function<void(const std::atomic<bool> &, Stats &)> test_func) {
+  Stats stats;
+  std::atomic<bool> run{true};
+
+  {
+    std::jthread bg_thread(test_func, std::cref(run), std::ref(stats));
+    std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration));
+    run.store(false, std::memory_order_relaxed);
+  }
+
+  std::cout << "    Operations: " << stats.total << std::endl;
+  std::cout << "    Successful insert: " << stats.succ[0] << std::endl;
+  std::cout << "    Successful contains: " << stats.succ[1] << std::endl;
+  std::cout << "    Successful remove: " << stats.succ[2] << std::endl;
+  std::cout << "    Successful find: " << stats.succ[3] << std::endl;
+  std::cout << std::endl;
+
+  const auto tot = std::accumulate(stats.succ.begin(), +stats.succ.begin() + 3, 0);
+  const auto tops = stats.total;
+
+  std::cout << "Total successful: " << tot << " (" << tot / FLAGS_duration << " calls/s)" << std::endl;
+  std::cout << "Total ops: " << tops << " (" << tops / FLAGS_duration << " calls/s)" << std::endl;
+}

tests/mgbench/README.md

@@ -0,0 +1,291 @@
+# :fire: mgBench: Benchmark for graph databases
+
+## :clipboard: Benchmark Overview
+
+mgBench is primarily designed to benchmark graph databases. To test graph database performance, this benchmark executes Cypher queries (write, read, update, aggregate, and analyze) on a given dataset. Queries are general and represent a typical workload that would be used to analyze any graph dataset. [BenchGraph](https://memgraph.com/benchgraph/) platform shows the results of running these queries on supported vendors. It shows the overall performance of each system relative to others.
+
+Three workload types can be executed:
+- Isolated - Concurrent execution of a single type of query,
+- Mixed - Concurrent execution of a single type of query mixed with a certain percentage of queries from a designated query group,
+- Realistic - Concurrent execution of queries from write, read, update and analyze groups.
+
+Currently, the benchmark is executed on the social media dataset Pokec, available in different sizes. The full list of queries and their grouping is available as [query list](#query-list).
+
+This methodology is designed to be read from top to bottom to understand what is being tested and how, but feel free to jump to parts that interest you.
+
+- [:fire: mgBench: Benchmark for graph databases](#fire-mgbench-benchmark-for-graph-databases)
+  - [:clipboard: Benchmark Overview](#clipboard-benchmark-overview)
+  - [:dart: Design goals](#dart-design-goals)
+    - [Reproducibility and validation](#reproducibility-and-validation)
+    - [Database compatibility](#database-compatibility)
+    - [Workloads](#workloads)
+    - [Fine-tuning](#fine-tuning)
+    - [Limitations](#limitations)
+  - [:wrench: mgBench](#wrench-mgbench)
+    - [Important files](#important-files)
+    - [Prerequisites](#prerequisites)
+    - [Running the benchmark](#running-the-benchmark)
+    - [Database conditions](#database-conditions)
+    - [Comparing results](#comparing-results)
+  - [:bar\_chart: Results](#bar_chart-results)
+  - [:books: Datasets](#books-datasets)
+    - [Pokec](#pokec)
+      - [Query list](#query-list)
+  - [:computer: Platform](#computer-platform)
+    - [Intel - HP](#intel---hp)
+  - [:nut\_and\_bolt: Supported databases](#nut_and_bolt-supported-databases)
+    - [Database notes](#database-notes)
+  - [:raised\_hands: Contributions](#raised_hands-contributions)
+  - [:mega: History and Future of mgBench](#mega-history-and-future-of-mgbench)
+    - [History of mgBench](#history-of-mgbench)
+    - [Future of mgBench](#future-of-mgbench)
+
+## :dart: Design goals
+
+### Reproducibility and validation
+
+Running this benchmark is automated, and the code used to run benchmarks is publicly available. You can  [run mgBench](#running-the-benchmark) with default settings to validate the results at [BenchGraph platform](https://memgraph.com/benchgraph). The results may differ depending on the hardware, database configuration, and other variables involved in your setup.  But if the results you get are significantly different, feel free to [open a GitHub issue](https://github.com/memgraph/memgraph/issues).
+
+In the future, the project will be expanded to include more platforms to see how systems perform on different OS and hardware configurations. If you are interested in what will be added and tested, read the section about [the future of mgBench](#future-of-mgbench)
+
+
+### Database compatibility
+
+At the moment, support for graph databases is limited. To run the benchmarks, the graph database must support Cypher query language and the Bolt protocol.
+
+Using Cypher ensures that executed queries are identical or similar on every supported system. Possible differences are noted in [database notes](#database-notes). A single C++ client queries all database systems, and it is based on the Bolt protocol. Using a single client ensures minimal performance penalties from the client side and ensures fairness across different vendors.
+
+If your database supports the given requirements, feel free to contribute and add your database to mgBench.
+If your database does not support the mentioned requirements, follow the project because support for other languages and protocols in graph database space will be added.
+
+
+### Workloads
+Running queries as standalone units is simple and relatively easy to measure, but vendors often apply various caching and pre-aggregations that influence the results in these kinds of scenarios.  Results from running single queries can hint at the database's general performance, but in real life, a database is queried by multiple clients from multiple sides. That is why the mgBench client supports the consecutive execution of various queries. Concurrently writing, reading, updating and executing aggregational and analytical queries provides a better view of overall system performance than executing and measuring a single query. Queries that the mgBench executes are grouped into 5 groups - write, read, update, aggregate and analytical.
+
+The [BenchGraph platform](https://memgraph.com/benchgraph) shows results made by mgBench by executing three types of workloads:
+- ***Isolated workload***
+- ***Mixed workload***
+- ***Realistic workload***
+
+Each of these workloads has a specific purpose:
+
+***Isolated*** workload is the simplest test. An isolated workload goes through all the queries individually, concurrently executing a single query a predefined number of times. It is similar to executing a single query and measuring time but more complex due to concurrency. How many times a specific query will be executed depends on the approximation of the query’s latency. If a query is slower, it will be executed fewer times, if a query is faster, it will be executed more times. The approximation is based on the duration of execution for several concurrent threads, and it varies between vendors.
+If a query takes arguments, the argument value is changed for each execution. Arguments are generated non-randomly, so each vendor gets the same sequence of queries with the same arguments. This enables a deterministic workload for both vendors.
+The good thing about isolated workload is that it yields a better picture of single query performance. There is also a negative side, executing the same queries multiple times can trigger strong results caching on the vendor's side, which can result in false query times.
+
+
+***Mixed*** workload executes a fixed number of queries that read, update, aggregate, or analyze the data concurrently with a certain percentage of write queries because writing from the database can prevent aggressive caching and thus represent a more realistic performance of a single query. The negative side is that there is an added influence of write performance on the results. Currently, mgBench client does not support per-thread performance measurements, but this will be added in future iterations.
+
+
+***Realistic*** workload represents real-life use cases because queries write, read, update, and perform analytics in a mixed ratio like they would in real projects. The test executes a fixed number of queries, the distribution of which is defined by defining a percentage of queries performing one of four operations. The queries are selected non-randomly, so the workload is identical between different vendors. As with the rest of the workloads, all queries are executed concurrently.
+
+### Fine-tuning
+
+Each database system comes with a wide variety of possible configurations. Changing each of those configuration settings can introduce performance improvements or penalties. The focus of this benchmark is "out-of-the-box" performance without fine-tuning with the goal of having the fairest possible comparison. Fine-tuning can make some systems perform magnitudes faster, but this makes general benchmark systems hard to manage because all systems are configured differently, and fine-tuning requires vendor DB experts.
+
+Some configurational changes are necessary for test execution and are not considered fine-tuning. For example, configuring the database to avoid Bolt client login is valid since the tests are not performed under any type of authorization. All non-default configurations are mentioned in [database notes](#database-notes)
+
+### Limitations
+
+Benchmarking different systems is challenging because the setup, environment, queries, workload, and dataset can benefit specific database vendors. Each vendor may have a particularly strong use-case scenario. This benchmark aims to be neutral and fair to all database vendors. Acknowledging some of the current limitations can help understand the issues you might notice:
+1. mgBench measures and tracks just a tiny subset of everything that can be tracked and compared during testing. Active benchmarking is strenuous because it requires a lot of time to set up and validate. Passive benchmarking is much faster to iterate on but can have a few bugs.
+2. Datasets and queries used for testing are simple. Datasets and queries in real-world environments can become quite complex. To avoid Cypher specifics, mgBench uses simple queries of different variates. Future versions will include more complex datasets and queries.
+3. The scale of the dataset used is miniature for production environments. Production environments can have up to trillions of nodes and edges.
+Query results are not verified or important. The queries might return different results, but only the performance is measured, not correctness.
+4. All tests are performed on single-node databases.
+5. Architecturally different systems can be set up and measured biasedly.
+
+
+## :wrench: mgBench
+### Important files
+
+Listed below are the main scripts used to run the benchmarks:
+
+- `benchmark.py` - Script that runs the queries and workloads.
+- `datasets.py` - Script that handles datasets and queries for workloads.
+- `runners.py` - Script holding the configuration for different DB vendors.
+- `client.cpp` - Client for querying the database.
+- `graph_bench.py` - Script that starts all predefined and custom-defined workloads.
+-` compare_results.py` - Script that visually compares benchmark results.
+
+Except for these scripts, the project also includes dataset files and index configuration files. Once the first test is executed, those files can be located in the newly generated .cache folder.
+
+### Prerequisites
+
+To execute a benchmark, you need to download a binary version of supported databases and install Python on your system. Each database vendor can depend on external dependencies, such as Cmake, JVM, etc., so make sure to check specific vendor prerequisites.
+
+### Running the benchmark
+To run benchmarks, use the `graph_bench.py` script, which calls all the other necessary scripts. You can start the benchmarks by executing the following command:
+
+```
+graph_bench.py
+--vendor memgraph /home/memgraph/binary
+--dataset-group basic
+--dataset-size small
+--realistic 100 30 70 0 0
+--realistic 100 50 50 0 0
+--realistic 100 70 30 0 0
+--realistic 100 30 40 10 20
+--mixed 100 30 0 0 0 70
+```
+
+
+Isolated workload are always executed, and this commands calls for the execution of four realistic workloads with different distribution of queries and one mixed workload on a small size dataset.
+
+The distribution of queries from write, read, update and aggregate groups are defined in percentages and stated as arguments following the `--realistic` or `--mixed` flags.
+
+In the example of `--realistic 100 30 40 10 20` the distribution is as follows:
+
+- 100 - The number of queries to be executed.
+- 30 - The percentage of write queries to be executed.
+- 40 - The percentage of read queries to be executed.
+- 10 - The percentage of update queries to be executed.
+- 20 - The percentage of analytical queries to be executed.
+
+
+For `--mixed` workload argument, the first five parameters are the same, with an addition of a parameter for defining the percentage of individual queries.
+
+Feel free to add different configurations if you want. Results from the above benchmark run are visible on [BenchGraph platform](https://memgraph.com/benchgraph)
+
+### Database conditions
+In a production environment, database query caches are usually warmed from usage or pre-warm procedure to provide the best possible performance. Each workload in mgBench will be executed under the following conditions:
+- ***Hot run*** - before executing any benchmark query and taking measurements, a set of defined queries is executed to pre-warm the database.
+- ***Cold run*** - no warm-up was performed on the database before taking benchmark measurements.
+
+List of queries used for pre-warm up:
+```
+CREATE ();
+CREATE ()-[:TempEdge]->();
+MATCH (n) RETURN n LIMIT 1;
+```
+
+### Comparing results
+
+Once the benchmark has been run for a single vendor, all the results are saved in appropriately named `.json` files. A summary file is also created for that vendor and it contains all results combined. These summary files are used to compare results against other vendor results via the `compare_results.py` script:
+
+```
+compare_results.py
+--compare
+“path_to/neo4j_summary.json”
+“path_to/memgraph_summary.json”
+--output neo4j_vs_memgraph.html
+--different-vendors
+```
+
+The output is an HTML file with the visual representation of the performance differences between two compared vendors. The first passed summary JSON file is the reference point.
+
+## :bar_chart: Results
+Results visible in the HTML file or at [BenchGraph](https://memgraph.com/benchgraph) are throughput, memory, and latency. Database throughput and memory usage directly impact database usability and cost, while the latency of the query shows the base query execution duration.
+
+***Throughput*** directly defines how performant the database is and how much query traffic it can handle in a fixed time interval. It is expressed in queries per second. In each concurrent workload, execution is split across multiple clients. Each client executes queries concurrently. The duration of total execution is the sum of all concurrent clients' execution duration in seconds. In mgBench, the total count of executed queries and the total duration defines throughput per second across concurrent execution.
+
+Here is the code snippet from the client, that calculates ***throughput*** and metadata:
+```
+  // Create and output summary.
+  Metadata final_metadata;
+  uint64_t final_retries = 0;
+  double final_duration = 0.0;
+  for (int i = 0; i < FLAGS_num_workers; ++i) {
+    final_metadata += worker_metadata[i];
+    final_retries += worker_retries[i];
+    final_duration += worker_duration[i];
+  }
+  final_duration /= FLAGS_num_workers;
+  nlohmann::json summary = nlohmann::json::object();
+  summary["count"] = queries.size();
+  summary["duration"] = final_duration;
+  summary["throughput"] = static_cast<double>(queries.size()) / final_duration;
+  summary["retries"] = final_retries;
+  summary["metadata"] = final_metadata.Export();
+  summary["num_workers"] = FLAGS_num_workers;
+  (*stream) << summary.dump() << std::endl;
+```
+
+***Memory*** usage is calculated as ***peak RES*** (resident size) memory for each query or workload execution within mgBench. The result includes starting the database, executing the query/workload, and stopping the database. The peak RES is extracted from process PID as VmHVM (peak resident set size) before the process is stopped. The peak memory usage defines the worst-case scenario for a given query or workload, while on average, RAM footprint is lower. Measuring RES over time is supported by `runners.py`. For each vendor, it is possible to add RES tracking across workload execution, but it is not reported in the results.
+***Latency*** is calculated as the serial execution of 100 identical queries on a single thread. Each query has standard query statistics and tail latency data. The result includes query execution times: max, min, mean, p99, p95, p90, p75, and p50 in seconds.
+
+Each workload and all the results are based on concurrent query execution, except ***latency***.  As stated in [limitations](#limitations) section, mgBench tracks just a subset of resources, but the chapter on [mgBench future](#future-of-mgbench) explains the expansion plans.
+
+## :books: Datasets
+Before workload execution, appropriate dataset indexes are set.  Each vendor can have a specific syntax for setting up indexes, but those indexes should be schematically as similar as possible.
+
+After each workload is executed, the database is cleaned, and a new dataset is imported to provide a clean start for the following workload run. When executing isolated and mixed workloads, the database is also restarted after executing each query to minimize the impact on the following query execution.
+
+### Pokec
+
+Currently, the only available dataset to run the benchmarks on is the Slovenian social network, Pokec. It’s available in three different sizes, small, medium, and large.
+- [small](https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_small_import.cypher) - vertices 10,000, edges 121,716
+- [medium](https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_medium_import.cypher) - vertices 100,000, edges 1,768,515
+- [large](https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_large.setup.cypher.gz) - vertices 1,632,803, edges 30,622,564.
+
+Dataset is imported as a CYPHERL file of Cypher queries. Feel free to check dataset links for complete Cypher queries.
+Once the script is started, a single index is configured on (:User{id}). Only then are queries executed.
+Index queries for each supported vendor can be downloaded from “https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/vendor_name.cypher”, just make sure to use the proper vendor name such as `memgraph.cypher`.
+
+#### Query list
+
+| |Name     | Group | Query |
+|-|-----| -- | ------------ |
+|Q1|aggregate | aggregate | MATCH (n:User) RETURN n.age, COUNT(*)|
+|Q2|aggregate_count | aggregate | MATCH (n) RETURN count(n), count(n.age)|
+|Q3|aggregate_with_filter | aggregate | MATCH (n:User) WHERE n.age >= 18 RETURN n.age, COUNT(*)|
+|Q4|min_max_avg | aggregate | MATCH (n) RETURN min(n.age), max(n.age), avg(n.age)|
+|Q5|expansion_1 | analytical | MATCH (s:User {id: $id})-->(n:User) RETURN n.id|
+|Q6|expansion_1_with_filter| analytical | MATCH (s:User {id: $id})-->(n:User) WHERE n.age >= 18 RETURN n.id|
+|Q7|expansion_2| analytical | MATCH (s:User {id: $id})-->()-->(n:User) RETURN DISTINCT n.id|
+|Q8|expansion_2_with_filter| analytical | MATCH (s:User {id: $id})-->()-->(n:User) WHERE n.age >= 18 RETURN DISTINCT n.id|
+|Q9|expansion_3| analytical | MATCH (s:User {id: $id})-->()-->()-->(n:User) RETURN DISTINCT n.id|
+|Q10|expansion_3_with_filter| analytical | MATCH (s:User {id: $id})-->()-->()-->(n:User) WHERE n.age >= 18 RETURN DISTINCT n.id|
+|Q11|expansion_4| analytical | MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) RETURN DISTINCT n.id|
+|Q12|expansion_4_with_filter| analytical | MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) WHERE n.age >= 18 RETURN DISTINCT n.id|
+|Q13|neighbours_2| analytical | MATCH (s:User {id: $id})-[*1..2]->(n:User) RETURN DISTINCT n.id|
+|Q14|neighbours_2_with_filter| analytical | MATCH (s:User {id: $id})-[*1..2]->(n:User) WHERE n.age >= 18 RETURN DISTINCT n.id|
+|Q15|neighbours_2_with_data| analytical | MATCH (s:User {id: $id})-[*1..2]->(n:User) RETURN DISTINCT n.id, n|
+|Q16|neighbours_2_with_data_and_filter| analytical | MATCH (s:User {id: $id})-[*1..2]->(n:User) WHERE n.age >= 18 RETURN DISTINCT n.id, n|
+|Q17|pattern_cycle| analytical | MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) RETURN e1, m, e2|
+|Q18|pattern_long| analytical | MATCH (n1:User {id: $id})-[e1]->(n2)-[e2]->(n3)-[e3]->(n4)<-[e4]-(n5) RETURN n5 LIMIT 1|
+|Q19|pattern_short| analytical | MATCH (n:User {id: $id})-[e]->(m) RETURN m LIMIT 1|
+|Q20|single_edge_write| write | MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m CREATE (n)-[e:Temp]->(m) RETURN e|
+|Q21|single_vertex_write| write |CREATE (n:UserTemp {id : $id}) RETURN n|
+|Q22|single_vertex_property_update| update | MATCH (n:User {id: $id})-[e]->(m) RETURN m LIMIT 1|
+|Q23|single_vertex_read| read | MATCH (n:User {id : $id}) RETURN n|
+
+## :computer: Platform
+
+Testing on different hardware platforms and cloudVMs is essential for validating benchmark results. Currently, the tests are run on two different platforms.
+
+### Intel - HP
+
+- Server: HP DL360 G6
+- CPU: 2 x Intel Xeon X5650 6C12T @ 2.67GHz
+- RAM: 144GB
+- OS: Debian 4.19
+
+## :nut_and_bolt: Supported databases
+
+Due to current [database compatibility](link) requirements, the only supported database systems at the moment are:
+1. Memgraph v2.4
+2. Neo4j Community Edition v5.1.
+
+### Database notes
+
+Running configurations that differ from default configuration:
+
+- Memgraph - `storage_snapshot_on_exit=true`, `storage_recover_on_startup=true`
+- Neo4j - `dbms.security.auth_enabled=false`
+
+## :raised_hands: Contributions
+
+As previously stated, mgBench will expand, and we will need help adding more datasets, queries, databases, and support for protocols in mgBench. Feel free to contribute to any of those, and throw us a start :star:!
+
+## :mega: History and Future of mgBench
+### History of mgBench
+
+Infrastructure around mgBench was developed to test and maintain Memgraph performance. When critical code is changed, a performance test is run on Memgraph’s CI/CD infrastructure to ensure performance is not impacted. Due to the usage of mgBench for internal testing, some parts of the code are still tightly connected to Memgraph’s CI/CD infrastructure. The remains of that code do not impact benchmark setup or performance in any way.
+
+### Future of mgBench
+We have big plans for mgBench infrastructure that refers to the above mentioned [limitations](#limitations). Even though a basic dataset can give a solid indication of performance, adding bigger and more complex datasets is a priority to enable the execution of complex analytical queries.
+
+Also high on the list is expanding the list of vendors and providing support for different protocols and languages. The goal is to use mgBench to see how well Memgraph performs on various benchmarks tasks and publicly commit to improving.
+
+mgBench is currently a passive benchmark since resource usage and saturation across execution are not tracked. Sanity checks were performed, but these values are needed to get the full picture after each test. mgBench also deserves its own repository, and it will be decoupled from Memgraph’s testing infrastructure.

tests/mgbench/benchmark.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2021 Memgraph Ltd.
+# Copyright 2022 Memgraph Ltd.
 #
 # Use of this software is governed by the Business Source License
 # included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,9 +18,11 @@ import fnmatch
 import importlib
 import inspect
 import json
+import math
 import multiprocessing
 import os
 import random
+import statistics
 import sys
 import time
 
@@ -29,53 +31,8 @@ import helpers
 import log
 import runners
 
-
-def get_queries(gen, count):
-    # Make the generator deterministic.
-    random.seed(gen.__name__)
-    # Generate queries.
-    ret = []
-    for i in range(count):
-        ret.append(gen())
-    return ret
-
-
-def match_patterns(dataset, variant, group, test, is_default_variant, patterns):
-    for pattern in patterns:
-        verdict = [fnmatch.fnmatchcase(dataset, pattern[0])]
-        if pattern[1] != "":
-            verdict.append(fnmatch.fnmatchcase(variant, pattern[1]))
-        else:
-            verdict.append(is_default_variant)
-        verdict.append(fnmatch.fnmatchcase(group, pattern[2]))
-        verdict.append(fnmatch.fnmatchcase(test, pattern[3]))
-        if all(verdict):
-            return True
-    return False
-
-
-def filter_benchmarks(generators, patterns):
-    patterns = copy.deepcopy(patterns)
-    for i in range(len(patterns)):
-        pattern = patterns[i].split("/")
-        if len(pattern) > 4 or len(pattern) == 0:
-            raise Exception("Invalid benchmark description '" + pattern + "'!")
-        pattern.extend(["", "*", "*"][len(pattern) - 1 :])
-        patterns[i] = pattern
-    filtered = []
-    for dataset in sorted(generators.keys()):
-        generator, tests = generators[dataset]
-        for variant in generator.VARIANTS:
-            is_default_variant = variant == generator.DEFAULT_VARIANT
-            current = collections.defaultdict(list)
-            for group in tests:
-                for test_name, test_func in tests[group]:
-                    if match_patterns(dataset, variant, group, test_name, is_default_variant, patterns):
-                        current[group].append((test_name, test_func))
-            if len(current) > 0:
-                filtered.append((generator(variant), dict(current)))
-    return filtered
-
+WITH_FINE_GRAINED_AUTHORIZATION = "with_fine_grained_authorization"
+WITHOUT_FINE_GRAINED_AUTHORIZATION = "without_fine_grained_authorization"
 
 # Parse options.
 parser = argparse.ArgumentParser(
@@ -88,20 +45,28 @@ parser.add_argument(
     help="descriptions of benchmarks that should be run; "
     "multiple descriptions can be specified to run multiple "
     "benchmarks; the description is specified as "
-    "dataset/variant/group/test; Unix shell-style wildcards "
-    "can be used in the descriptions; variant, group and test "
+    "dataset/variant/group/query; Unix shell-style wildcards "
+    "can be used in the descriptions; variant, group and query "
     "are optional and they can be left out; the default "
     "variant is '' which selects the default dataset variant; "
-    "the default group is '*' which selects all groups; the "
-    "default test is '*' which selects all tests",
+    "the default group is '*' which selects all groups; the"
+    "default query is '*' which selects all queries",
 )
 parser.add_argument(
-    "--memgraph-binary", default=helpers.get_binary_path("memgraph"), help="Memgraph binary used for benchmarking"
+    "--vendor-binary",
+    help="Vendor binary used for benchmarking, by defuault it is memgraph",
+    default=helpers.get_binary_path("memgraph"),
+)
+
+parser.add_argument(
+    "--vendor-name",
+    default="memgraph",
+    help="Input vendor binary name (memgraph, neo4j)",
 )
 parser.add_argument(
     "--client-binary",
     default=helpers.get_binary_path("tests/mgbench/client"),
-    help="client binary used for benchmarking",
+    help="Client binary used for benchmarking",
 )
 parser.add_argument(
     "--num-workers-for-import",
@@ -112,18 +77,415 @@ parser.add_argument(
 parser.add_argument(
     "--num-workers-for-benchmark", type=int, default=1, help="number of workers used to execute the benchmark"
 )
-parser.add_argument("--single-threaded-runtime-sec", type=int, default=10, help="single threaded duration of each test")
-parser.add_argument("--no-load-query-counts", action="store_true", help="disable loading of cached query counts")
-parser.add_argument("--no-save-query-counts", action="store_true", help="disable storing of cached query counts")
-parser.add_argument("--export-results", default="", help="file path into which results should be exported")
+
 parser.add_argument(
-    "--temporary-directory", default="/tmp", help="directory path where temporary data should " "be stored"
+    "--single-threaded-runtime-sec",
+    type=int,
+    default=10,
+    help="single threaded duration of each query",
+)
+parser.add_argument(
+    "--no-load-query-counts",
+    action="store_true",
+    help="disable loading of cached query counts",
+)
+parser.add_argument(
+    "--no-save-query-counts",
+    action="store_true",
+    help="disable storing of cached query counts",
+)
+parser.add_argument(
+    "--export-results",
+    default="",
+    help="file path into which results should be exported",
+)
+parser.add_argument(
+    "--temporary-directory",
+    default="/tmp",
+    help="directory path where temporary data should " "be stored",
 )
 parser.add_argument("--no-properties-on-edges", action="store_true", help="disable properties on edges")
-parser.add_argument("--datasets-path", default="datasets", help="path to datasets to scan")
+
+parser.add_argument("--bolt-port", default=7687, help="memgraph bolt port")
+
+parser.add_argument(
+    "--datasets-path",
+    default="datasets",
+    help="path to a single Python file where additional datasets are located"
+    "NOTE: give the file unique name compared to the core module names",
+)
 parser.add_argument("--test-system-args", default="")
+
+parser.add_argument(
+    "--no-authorization",
+    action="store_false",
+    default=True,
+    help="Run each query with authorization",
+)
+
+parser.add_argument(
+    "--warmup-run",
+    action="store_true",
+    default=False,
+    help="Run warmup before benchmarks",
+)
+
+parser.add_argument(
+    "--mixed-workload",
+    nargs="*",
+    type=int,
+    default=[],
+    help="""Define combination that defines the mixed workload.
+    Mixed workload can be run as a single configuration for all groups of queries,
+    Pass the positional arguments as values of what percentage of
+    write/read/update/analytical queries you want to have in your workload.
+    Example:  --mixed-workload 1000 20 70 10 0 will execute 1000 queries, 20% write,
+    70% read, 10% update and 0% analytical.
+
+    Mixed workload can also be run on each query under some defined load.
+    By passing one more positional argument, you are defining what percentage of that query
+    will be in mixed workload, and this is executed for each query. The rest of the queries will be
+    selected from the appropriate groups
+    Running --mixed-workload 1000 30 0 0 0 70, will execute each query 700 times or 70%,
+    with the presence of 300 write queries from write type or 30%""",
+)
+
+parser.add_argument("--tail-latency", type=int, default=0, help="Number of queries for the tail latency statistics")
+
+parser.add_argument(
+    "--performance-tracking",
+    action="store_true",
+    default=False,
+    help="Flag for runners performance tracking, this logs RES through time and vendor specific performance tracking.",
+)
+
 args = parser.parse_args()
 
+
+class Workload:
+    def __init__(self, config):
+        config_len = len(config)
+        if config_len == 0:
+            self.name = "Isolated"
+            self.config = config
+        elif config_len >= 5:
+            if sum(config[1:]) != 100:
+                raise Exception(
+                    "Please make sure that passed arguments % sum to 100% percent!, passed: ",
+                    config,
+                )
+            if config_len == 5:
+                self.name = "Realistic"
+                self.config = config
+            else:
+                self.name = "Mixed"
+                self.config = config
+
+
+def get_queries(gen, count):
+    # Make the generator deterministic.
+    random.seed(gen.__name__)
+    # Generate queries.
+    ret = []
+    for i in range(count):
+        ret.append(gen())
+    return ret
+
+
+def match_patterns(dataset, variant, group, query, is_default_variant, patterns):
+    for pattern in patterns:
+        verdict = [fnmatch.fnmatchcase(dataset, pattern[0])]
+        if pattern[1] != "":
+            verdict.append(fnmatch.fnmatchcase(variant, pattern[1]))
+        else:
+            verdict.append(is_default_variant)
+        verdict.append(fnmatch.fnmatchcase(group, pattern[2]))
+        verdict.append(fnmatch.fnmatchcase(query, pattern[3]))
+        if all(verdict):
+            return True
+    return False
+
+
+def filter_benchmarks(generators, patterns):
+    patterns = copy.deepcopy(patterns)
+    for i in range(len(patterns)):
+        pattern = patterns[i].split("/")
+        if len(pattern) > 5 or len(pattern) == 0:
+            raise Exception("Invalid benchmark description '" + pattern + "'!")
+        pattern.extend(["", "*", "*"][len(pattern) - 1 :])
+        patterns[i] = pattern
+    filtered = []
+    for dataset in sorted(generators.keys()):
+        generator, queries = generators[dataset]
+        for variant in generator.VARIANTS:
+            is_default_variant = variant == generator.DEFAULT_VARIANT
+            current = collections.defaultdict(list)
+            for group in queries:
+                for query_name, query_func in queries[group]:
+                    if match_patterns(
+                        dataset,
+                        variant,
+                        group,
+                        query_name,
+                        is_default_variant,
+                        patterns,
+                    ):
+                        current[group].append((query_name, query_func))
+            if len(current) == 0:
+                continue
+
+            # Ignore benchgraph "basic" queries in standard CI/CD run
+            for pattern in patterns:
+                res = pattern.count("*")
+                key = "basic"
+                if res >= 2 and key in current.keys():
+                    current.pop(key)
+
+            filtered.append((generator(variant, args.vendor_name), dict(current)))
+    return filtered
+
+
+def warmup(client):
+    print("Executing warm-up queries")
+    client.execute(
+        queries=[
+            ("CREATE ();", {}),
+            ("CREATE ()-[:TempEdge]->();", {}),
+            ("MATCH (n) RETURN n LIMIT 1;", {}),
+        ],
+        num_workers=1,
+    )
+
+
+def tail_latency(vendor, client, func):
+    iteration = args.tail_latency
+    if iteration >= 10:
+        vendor.start_benchmark("tail_latency")
+        if args.warmup_run:
+            warmup(client)
+        latency = []
+
+        query_list = get_queries(func, iteration)
+        for i in range(0, iteration):
+            ret = client.execute(queries=[query_list[i]], num_workers=1)
+            latency.append(ret[0]["duration"])
+        latency.sort()
+        query_stats = {
+            "iterations": iteration,
+            "min": latency[0],
+            "max": latency[iteration - 1],
+            "mean": statistics.mean(latency),
+            "p99": latency[math.floor(iteration * 0.99) - 1],
+            "p95": latency[math.floor(iteration * 0.95) - 1],
+            "p90": latency[math.floor(iteration * 0.90) - 1],
+            "p75": latency[math.floor(iteration * 0.75) - 1],
+            "p50": latency[math.floor(iteration * 0.50) - 1],
+        }
+        print("Query statistics for tail latency: ")
+        print(query_stats)
+        vendor.stop("tail_latency")
+    else:
+        query_stats = {}
+    return query_stats
+
+
+def mixed_workload(vendor, client, dataset, group, queries, workload):
+
+    num_of_queries = workload.config[0]
+    percentage_distribution = workload.config[1:]
+    if sum(percentage_distribution) != 100:
+        raise Exception(
+            "Please make sure that passed arguments % sum to 100% percent!, passed: ",
+            percentage_distribution,
+        )
+    s = [str(i) for i in workload.config]
+
+    config_distribution = "_".join(s)
+
+    print("Generating mixed workload.")
+
+    percentages_by_type = {
+        "write": percentage_distribution[0],
+        "read": percentage_distribution[1],
+        "update": percentage_distribution[2],
+        "analytical": percentage_distribution[3],
+    }
+
+    queries_by_type = {
+        "write": [],
+        "read": [],
+        "update": [],
+        "analytical": [],
+    }
+
+    for (_, funcname) in queries[group]:
+        for key in queries_by_type.keys():
+            if key in funcname:
+                queries_by_type[key].append(funcname)
+
+    for key, percentage in percentages_by_type.items():
+        if percentage != 0 and len(queries_by_type[key]) == 0:
+            raise Exception(
+                "There is a missing query in group (write, read, update or analytical) for given workload distribution."
+            )
+
+    random.seed(config_distribution)
+
+    # Executing mixed workload for each test
+    if workload.name == "Mixed":
+        for query, funcname in queries[group]:
+            full_workload = []
+
+            log.info(
+                "Running query in mixed workload:",
+                "{}/{}/{}".format(
+                    group,
+                    query,
+                    funcname,
+                ),
+            )
+            base_query = getattr(dataset, funcname)
+
+            base_query_type = funcname.rsplit("_", 1)[1]
+
+            if percentages_by_type.get(base_query_type, 0) > 0:
+                continue
+
+            options = ["write", "read", "update", "analytical", "query"]
+            function_type = random.choices(population=options, weights=percentage_distribution, k=num_of_queries)
+
+            for t in function_type:
+                # Get the apropropriate functions with same probabilty
+                if t == "query":
+                    full_workload.append(base_query())
+                else:
+                    funcname = random.choices(queries_by_type[t], k=1)[0]
+                    aditional_query = getattr(dataset, funcname)
+                    full_workload.append(aditional_query())
+
+            vendor.start_benchmark(
+                dataset.NAME + dataset.get_variant() + "_" + "mixed" + "_" + query + "_" + config_distribution
+            )
+            if args.warmup_run:
+                warmup(client)
+            ret = client.execute(
+                queries=full_workload,
+                num_workers=args.num_workers_for_benchmark,
+            )[0]
+            usage_workload = vendor.stop(
+                dataset.NAME + dataset.get_variant() + "_" + "mixed" + "_" + query + "_" + config_distribution
+            )
+
+            ret["database"] = usage_workload
+
+            results_key = [
+                dataset.NAME,
+                dataset.get_variant(),
+                group,
+                query + "_" + config_distribution,
+                WITHOUT_FINE_GRAINED_AUTHORIZATION,
+            ]
+            results.set_value(*results_key, value=ret)
+
+    else:
+        # Executing mixed workload from groups of queries
+        full_workload = []
+        options = ["write", "read", "update", "analytical"]
+        function_type = random.choices(population=options, weights=percentage_distribution, k=num_of_queries)
+
+        for t in function_type:
+            # Get the apropropriate functions with same probabilty
+            funcname = random.choices(queries_by_type[t], k=1)[0]
+            aditional_query = getattr(dataset, funcname)
+            full_workload.append(aditional_query())
+
+        vendor.start_benchmark(dataset.NAME + dataset.get_variant() + "_" + workload.name + "_" + config_distribution)
+        if args.warmup_run:
+            warmup(client)
+        ret = client.execute(
+            queries=full_workload,
+            num_workers=args.num_workers_for_benchmark,
+        )[0]
+        usage_workload = vendor.stop(
+            dataset.NAME + dataset.get_variant() + "_" + workload.name + "_" + config_distribution
+        )
+        mixed_workload = {
+            "count": ret["count"],
+            "duration": ret["duration"],
+            "retries": ret["retries"],
+            "throughput": ret["throughput"],
+            "num_workers": ret["num_workers"],
+            "database": usage_workload,
+        }
+        results_key = [
+            dataset.NAME,
+            dataset.get_variant(),
+            group,
+            config_distribution,
+            WITHOUT_FINE_GRAINED_AUTHORIZATION,
+        ]
+        results.set_value(*results_key, value=mixed_workload)
+
+        print(mixed_workload)
+
+
+def get_query_cache_count(vendor, client, func, config_key):
+    cached_count = config.get_value(*config_key)
+
+    if cached_count is None:
+        print(
+            "Determining the number of queries necessary for",
+            args.single_threaded_runtime_sec,
+            "seconds of single-threaded runtime...",
+        )
+        # First run to prime the query caches.
+        vendor.start_benchmark("cache")
+        if args.warmup_run:
+            warmup(client)
+        client.execute(queries=get_queries(func, 1), num_workers=1)
+        # Get a sense of the runtime.
+        count = 1
+        while True:
+            ret = client.execute(queries=get_queries(func, count), num_workers=1)
+            duration = ret[0]["duration"]
+            should_execute = int(args.single_threaded_runtime_sec / (duration / count))
+            print(
+                "executed_queries={}, total_duration={}, "
+                "query_duration={}, estimated_count={}".format(count, duration, duration / count, should_execute)
+            )
+            # We don't have to execute the next iteration when
+            # `should_execute` becomes the same order of magnitude as
+            # `count * 10`.
+            if should_execute / (count * 10) < 10:
+                count = should_execute
+                break
+            else:
+                count = count * 10
+        vendor.stop("cache")
+
+        # Lower bound for count
+        if count < 20:
+            count = 20
+
+        config.set_value(
+            *config_key,
+            value={
+                "count": count,
+                "duration": args.single_threaded_runtime_sec,
+            },
+        )
+    else:
+        print(
+            "Using cached query count of",
+            cached_count["count"],
+            "queries for",
+            cached_count["duration"],
+            "seconds of single-threaded runtime.",
+        )
+        count = int(cached_count["count"] * args.single_threaded_runtime_sec / cached_count["duration"])
+    return count
+
+
 head_tail = os.path.split(args.datasets_path)
 path_without_dataset_name = head_tail[0]
 dataset_name = head_tail[1].split(".")[0]
@@ -138,13 +500,13 @@ for key in dir(dataset_to_use):
     dataset = getattr(dataset_to_use, key)
     if not inspect.isclass(dataset) or dataset == datasets.Dataset or not issubclass(dataset, datasets.Dataset):
         continue
-    tests = collections.defaultdict(list)
+    queries = collections.defaultdict(list)
     for funcname in dir(dataset):
         if not funcname.startswith("benchmark__"):
             continue
-        group, test = funcname.split("__")[1:]
-        tests[group].append((test, funcname))
-    generators[dataset.NAME] = (dataset, dict(tests))
+        group, query = funcname.split("__")[1:]
+        queries[group].append((query, funcname))
+    generators[dataset.NAME] = (dataset, dict(queries))
     if dataset.PROPERTIES_ON_EDGES and args.no_properties_on_edges:
         raise Exception(
             'The "{}" dataset requires properties on edges, ' "but you have disabled them!".format(dataset.NAME)
@@ -152,15 +514,19 @@ for key in dir(dataset_to_use):
 
 # List datasets if there is no specified dataset.
 if len(args.benchmarks) == 0:
-    log.init("Available tests")
+    log.init("Available queries")
     for name in sorted(generators.keys()):
         print("Dataset:", name)
-        dataset, tests = generators[name]
-        print("    Variants:", ", ".join(dataset.VARIANTS), "(default: " + dataset.DEFAULT_VARIANT + ")")
-        for group in sorted(tests.keys()):
+        dataset, queries = generators[name]
+        print(
+            "    Variants:",
+            ", ".join(dataset.VARIANTS),
+            "(default: " + dataset.DEFAULT_VARIANT + ")",
+        )
+        for group in sorted(queries.keys()):
             print("    Group:", group)
-            for test_name, test_func in tests[group]:
-                print("        Test:", test_name)
+            for query_name, query_func in queries[group]:
+                print("        Query:", query_name)
     sys.exit(0)
 
 # Create cache, config and results objects.
@@ -175,136 +541,268 @@ results = helpers.RecursiveDict()
 benchmarks = filter_benchmarks(generators, args.benchmarks)
 
 # Run all specified benchmarks.
-for dataset, tests in benchmarks:
+for dataset, queries in benchmarks:
+
+    workload = Workload(args.mixed_workload)
+
+    run_config = {
+        "vendor": args.vendor_name,
+        "condition": "hot" if args.warmup_run else "cold",
+        "workload": workload.name,
+        "workload_config": workload.config,
+    }
+
+    results.set_value("__run_configuration__", value=run_config)
+
     log.init("Preparing", dataset.NAME + "/" + dataset.get_variant(), "dataset")
     dataset.prepare(cache.cache_directory("datasets", dataset.NAME, dataset.get_variant()))
 
-    # Prepare runners and import the dataset.
-    memgraph = runners.Memgraph(
-        args.memgraph_binary,
-        args.temporary_directory,
-        not args.no_properties_on_edges,
-        args.test_system_args,
-    )
-    client = runners.Client(args.client_binary, args.temporary_directory)
-    memgraph.start_preparation()
-    time.sleep(5.0)  # giving enough time to machine manager and all to start up
-    ret = client.execute(file_path=dataset.get_file(), num_workers=args.num_workers_for_import)
-    usage = memgraph.stop()
-
-    # Display import statistics.
-    print()
-    for row in ret:
-        print(
-            "Executed",
-            row["count"],
-            "queries in",
-            row["duration"],
-            "seconds using",
-            row["num_workers"],
-            "workers with a total throughput of",
-            row["throughput"],
-            "queries/second.",
+    # TODO: Create some abstract class for vendors, that will hold this data
+    if args.vendor_name == "neo4j":
+        vendor = runners.Neo4j(
+            args.vendor_binary,
+            args.temporary_directory,
+            args.bolt_port,
+            args.performance_tracking,
+            args.test_system_args,
+        )
+    else:
+        vendor = runners.Memgraph(
+            args.vendor_binary,
+            args.temporary_directory,
+            not args.no_properties_on_edges,
+            args.bolt_port,
+            args.performance_tracking,
+            args.test_system_args,
         )
-    print()
-    print(
-        "The database used",
-        usage["cpu"],
-        "seconds of CPU time and peaked at",
-        usage["memory"] / 1024 / 1024,
-        "MiB of RAM.",
-    )
 
-    # Save import results.
+    client = runners.Client(args.client_binary, args.temporary_directory, args.bolt_port)
+
+    ret = None
+    usage = None
+    if args.vendor_name == "neo4j":
+        vendor.start_preparation("preparation")
+        print("Executing database cleanup and index setup...")
+        ret = client.execute(file_path=dataset.get_index(), num_workers=args.num_workers_for_import)
+        usage = vendor.stop("preparation")
+        dump_dir = cache.cache_directory("datasets", dataset.NAME, dataset.get_variant())
+        dump_file, exists = dump_dir.get_file("neo4j.dump")
+        if exists:
+            vendor.load_db_from_dump(path=dump_dir.get_path())
+        else:
+            vendor.start_preparation("import")
+            print("Importing dataset...")
+
+            ret = client.execute(file_path=dataset.get_file(), num_workers=args.num_workers_for_import)
+            usage = vendor.stop("import")
+
+            vendor.dump_db(path=dump_dir.get_path())
+    else:
+        vendor.start_preparation("import")
+        time.sleep(5.0)  # giving enough time to machine manager and all to start up
+        print("Executing database cleanup and index setup...")
+        ret = client.execute(file_path=dataset.get_index(), num_workers=args.num_workers_for_import)
+        print("Importing dataset...")
+        ret = client.execute(file_path=dataset.get_file(), num_workers=args.num_workers_for_import)
+        usage = vendor.stop("import")
+        # Save import results.
     import_key = [dataset.NAME, dataset.get_variant(), "__import__"]
-    results.set_value(*import_key, value={"client": ret, "database": usage})
+    if ret != None and usage != None:
+        # Display import statistics.
+        print()
+        for row in ret:
+            print(
+                "Executed",
+                row["count"],
+                "queries in",
+                row["duration"],
+                "seconds using",
+                row["num_workers"],
+                "workers with a total throughput of",
+                row["throughput"],
+                "queries/second.",
+            )
+        print()
+        print(
+            "The database used",
+            usage["cpu"],
+            "seconds of CPU time and peaked at",
+            usage["memory"] / 1024 / 1024,
+            "MiB of RAM.",
+        )
 
-    # TODO: cache import data
+        results.set_value(*import_key, value={"client": ret, "database": usage})
+    else:
+        results.set_value(*import_key, value={"client": "dump_load", "database": "dump_load"})
 
     # Run all benchmarks in all available groups.
-    for group in sorted(tests.keys()):
-        for test, funcname in tests[group]:
-            log.info("Running test:", "{}/{}".format(group, test))
-            func = getattr(dataset, funcname)
+    for group in sorted(queries.keys()):
 
-            # Get number of queries to execute.
-            # TODO: implement minimum number of queries, `max(10, num_workers)`
-            config_key = [dataset.NAME, dataset.get_variant(), group, test]
-            cached_count = config.get_value(*config_key)
-            if cached_count is None:
+        # Running queries in mixed workload
+        if workload.name == "Mixed" or workload.name == "Realistic":
+            mixed_workload(vendor, client, dataset, group, queries, workload)
+        else:
+            for query, funcname in queries[group]:
+                log.info(
+                    "Running query:",
+                    "{}/{}/{}/{}".format(group, query, funcname, WITHOUT_FINE_GRAINED_AUTHORIZATION),
+                )
+                func = getattr(dataset, funcname)
+
+                query_statistics = tail_latency(vendor, client, func)
+
+                # Query count for each vendor
+                config_key = [
+                    dataset.NAME,
+                    dataset.get_variant(),
+                    args.vendor_name,
+                    group,
+                    query,
+                ]
+                count = get_query_cache_count(vendor, client, func, config_key)
+
+                # Benchmark run.
+                print("Sample query:", get_queries(func, 1)[0][0])
                 print(
-                    "Determining the number of queries necessary for",
+                    "Executing benchmark with",
+                    count,
+                    "queries that should " "yield a single-threaded runtime of",
                     args.single_threaded_runtime_sec,
-                    "seconds of single-threaded runtime...",
+                    "seconds.",
                 )
-                # First run to prime the query caches.
-                memgraph.start_benchmark()
-                client.execute(queries=get_queries(func, 1), num_workers=1)
-                # Get a sense of the runtime.
-                count = 1
-                while True:
-                    ret = client.execute(queries=get_queries(func, count), num_workers=1)
-                    duration = ret[0]["duration"]
-                    should_execute = int(args.single_threaded_runtime_sec / (duration / count))
+                print(
+                    "Queries are executed using",
+                    args.num_workers_for_benchmark,
+                    "concurrent clients.",
+                )
+                vendor.start_benchmark(dataset.NAME + dataset.get_variant() + "_" + workload.name + "_" + query)
+                if args.warmup_run:
+                    warmup(client)
+                ret = client.execute(
+                    queries=get_queries(func, count),
+                    num_workers=args.num_workers_for_benchmark,
+                )[0]
+                usage = vendor.stop(dataset.NAME + dataset.get_variant() + "_" + workload.name + "_" + query)
+                ret["database"] = usage
+                ret["query_statistics"] = query_statistics
+
+                # Output summary.
+                print()
+                print("Executed", ret["count"], "queries in", ret["duration"], "seconds.")
+                print("Queries have been retried", ret["retries"], "times.")
+                print("Database used {:.3f} seconds of CPU time.".format(usage["cpu"]))
+                print("Database peaked at {:.3f} MiB of memory.".format(usage["memory"] / 1024.0 / 1024.0))
+                print("{:<31} {:>20} {:>20} {:>20}".format("Metadata:", "min", "avg", "max"))
+                metadata = ret["metadata"]
+                for key in sorted(metadata.keys()):
                     print(
-                        "executed_queries={}, total_duration={}, "
-                        "query_duration={}, estimated_count={}".format(
-                            count, duration, duration / count, should_execute
-                        )
+                        "{name:>30}: {minimum:>20.06f} {average:>20.06f} "
+                        "{maximum:>20.06f}".format(name=key, **metadata[key])
                     )
-                    # We don't have to execute the next iteration when
-                    # `should_execute` becomes the same order of magnitude as
-                    # `count * 10`.
-                    if should_execute / (count * 10) < 10:
-                        count = should_execute
-                        break
-                    else:
-                        count = count * 10
-                memgraph.stop()
-                config.set_value(*config_key, value={"count": count, "duration": args.single_threaded_runtime_sec})
-            else:
-                print(
-                    "Using cached query count of",
-                    cached_count["count"],
-                    "queries for",
-                    cached_count["duration"],
-                    "seconds of single-threaded runtime.",
-                )
-                count = int(cached_count["count"] * args.single_threaded_runtime_sec / cached_count["duration"])
+                log.success("Throughput: {:02f} QPS".format(ret["throughput"]))
 
-            # Benchmark run.
-            print("Sample query:", get_queries(func, 1)[0][0])
-            print(
-                "Executing benchmark with",
-                count,
-                "queries that should yield a single-threaded runtime of",
-                args.single_threaded_runtime_sec,
-                "seconds.",
+                # Save results.
+                results_key = [
+                    dataset.NAME,
+                    dataset.get_variant(),
+                    group,
+                    query,
+                    WITHOUT_FINE_GRAINED_AUTHORIZATION,
+                ]
+                results.set_value(*results_key, value=ret)
+
+        ## If there is need for authorization testing.
+        if args.no_authorization:
+            print("Running query with authorization")
+            vendor.start_benchmark("authorization")
+            client.execute(
+                queries=[
+                    ("CREATE USER user IDENTIFIED BY 'test';", {}),
+                    ("GRANT ALL PRIVILEGES TO user;", {}),
+                    ("GRANT CREATE_DELETE ON EDGE_TYPES * TO user;", {}),
+                    ("GRANT CREATE_DELETE ON LABELS * TO user;", {}),
+                ]
             )
-            print("Queries are executed using", args.num_workers_for_benchmark, "concurrent clients.")
-            memgraph.start_benchmark()
-            ret = client.execute(queries=get_queries(func, count), num_workers=args.num_workers_for_benchmark)[0]
-            usage = memgraph.stop()
-            ret["database"] = usage
+            client = runners.Client(
+                args.client_binary,
+                args.temporary_directory,
+                args.bolt_port,
+                username="user",
+                password="test",
+            )
+            vendor.stop("authorization")
 
-            # Output summary.
-            print()
-            print("Executed", ret["count"], "queries in", ret["duration"], "seconds.")
-            print("Queries have been retried", ret["retries"], "times.")
-            print("Database used {:.3f} seconds of CPU time.".format(usage["cpu"]))
-            print("Database peaked at {:.3f} MiB of memory.".format(usage["memory"] / 1024.0 / 1024.0))
-            print("{:<31} {:>20} {:>20} {:>20}".format("Metadata:", "min", "avg", "max"))
-            metadata = ret["metadata"]
-            for key in sorted(metadata.keys()):
-                print(
-                    "{name:>30}: {minimum:>20.06f} {average:>20.06f} "
-                    "{maximum:>20.06f}".format(name=key, **metadata[key])
+            for query, funcname in queries[group]:
+
+                log.info(
+                    "Running query:",
+                    "{}/{}/{}/{}".format(group, query, funcname, WITH_FINE_GRAINED_AUTHORIZATION),
                 )
-            log.success("Throughput: {:02f} QPS".format(ret["throughput"]))
+                func = getattr(dataset, funcname)
+
+                query_statistics = tail_latency(vendor, client, func)
+
+                config_key = [
+                    dataset.NAME,
+                    dataset.get_variant(),
+                    args.vendor_name,
+                    group,
+                    query,
+                ]
+                count = get_query_cache_count(vendor, client, func, config_key)
+
+                vendor.start_benchmark("authorization")
+                if args.warmup_run:
+                    warmup(client)
+                ret = client.execute(
+                    queries=get_queries(func, count),
+                    num_workers=args.num_workers_for_benchmark,
+                )[0]
+                usage = vendor.stop("authorization")
+                ret["database"] = usage
+                ret["query_statistics"] = query_statistics
+
+                # Output summary.
+                print()
+                print(
+                    "Executed",
+                    ret["count"],
+                    "queries in",
+                    ret["duration"],
+                    "seconds.",
+                )
+                print("Queries have been retried", ret["retries"], "times.")
+                print("Database used {:.3f} seconds of CPU time.".format(usage["cpu"]))
+                print("Database peaked at {:.3f} MiB of memory.".format(usage["memory"] / 1024.0 / 1024.0))
+                print("{:<31} {:>20} {:>20} {:>20}".format("Metadata:", "min", "avg", "max"))
+                metadata = ret["metadata"]
+                for key in sorted(metadata.keys()):
+                    print(
+                        "{name:>30}: {minimum:>20.06f} {average:>20.06f} "
+                        "{maximum:>20.06f}".format(name=key, **metadata[key])
+                    )
+                log.success("Throughput: {:02f} QPS".format(ret["throughput"]))
+                # Save results.
+                results_key = [
+                    dataset.NAME,
+                    dataset.get_variant(),
+                    group,
+                    query,
+                    WITH_FINE_GRAINED_AUTHORIZATION,
+                ]
+                results.set_value(*results_key, value=ret)
+
+            # Clean up database from any roles and users job
+            vendor.start_benchmark("authorizations")
+            ret = client.execute(
+                queries=[
+                    ("REVOKE LABELS * FROM user;", {}),
+                    ("REVOKE EDGE_TYPES * FROM user;", {}),
+                    ("DROP USER user;", {}),
+                ]
+            )
+            vendor.stop("authorization")
 
-            # Save results.
-            results_key = [dataset.NAME, dataset.get_variant(), group, test]
-            results.set_value(*results_key, value=ret)
 
 # Save configuration.
 if not args.no_save_query_counts:

tests/mgbench/compare_results.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2021 Memgraph Ltd.
+# Copyright 2022 Memgraph Ltd.
 #
 # Use of this software is governed by the Business Source License
 # included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,48 +15,6 @@ import argparse
 import json
 
 
-FIELDS = [
-    {
-        "name": "throughput",
-        "positive_diff_better": True,
-        "scaling": 1,
-        "unit": "QPS",
-        "diff_treshold": 0.05,  # 5%
-    },
-    {
-        "name": "duration",
-        "positive_diff_better": False,
-        "scaling": 1,
-        "unit": "s",
-    },
-    {
-        "name": "parsing_time",
-        "positive_diff_better": False,
-        "scaling": 1000,
-        "unit": "ms",
-    },
-    {
-        "name": "planning_time",
-        "positive_diff_better": False,
-        "scaling": 1000,
-        "unit": "ms",
-    },
-    {
-        "name": "plan_execution_time",
-        "positive_diff_better": False,
-        "scaling": 1000,
-        "unit": "ms",
-    },
-    {
-        "name": "memory",
-        "positive_diff_better": False,
-        "scaling": 1 / 1024 / 1024,
-        "unit": "MiB",
-        "diff_treshold": 0.02,  # 2%
-    },
-]
-
-
 def load_results(fname):
     with open(fname) as f:
         return json.load(f)
@@ -77,47 +35,54 @@ def recursive_get(data, *args, value=None):
     return data
 
 
-def compare_results(results_from, results_to, fields):
+def compare_results(results_from, results_to, fields, ignored, different_vendors):
     ret = {}
     for dataset, variants in results_to.items():
+        if dataset == "__run_configuration__":
+            continue
         for variant, groups in variants.items():
             for group, scenarios in groups.items():
                 if group == "__import__":
                     continue
                 for scenario, summary_to in scenarios.items():
-                    summary_from = recursive_get(
-                        results_from, dataset, variant, group, scenario,
-                        value={})
-                    if len(summary_from) > 0 and \
-                            summary_to["count"] != summary_from["count"] or \
-                            summary_to["num_workers"] != \
-                            summary_from["num_workers"]:
+                    if scenario in ignored:
+                        continue
+
+                    summary_from = recursive_get(results_from, dataset, variant, group, scenario, value={})
+                    summary_from = summary_from["without_fine_grained_authorization"]
+                    summary_to = summary_to["without_fine_grained_authorization"]
+                    if (
+                        len(summary_from) > 0
+                        and (summary_to["count"] != summary_from["count"] and not different_vendors)
+                        or summary_to["num_workers"] != summary_from["num_workers"]
+                    ):
                         raise Exception("Incompatible results!")
-                    testcode = "/".join([dataset, variant, group, scenario,
-                                         "{:02d}".format(
-                                             summary_to["num_workers"])])
+                    testcode = "/".join([dataset, variant, group, scenario, "{:02d}".format(summary_to["num_workers"])])
                     row = {}
                     performance_changed = False
                     for field in fields:
                         key = field["name"]
                         if key in summary_to:
-                            row[key] = compute_diff(
-                                summary_from.get(key, None),
-                                summary_to[key])
+                            row[key] = compute_diff(summary_from.get(key, None), summary_to[key])
                         elif key in summary_to["database"]:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "database", key,
-                                              value=None),
-                                summary_to["database"][key])
-                        else:
+                                recursive_get(summary_from, "database", key, value=None),
+                                summary_to["database"][key],
+                            )
+                        elif summary_to.get("query_statistics") != None and key in summary_to["query_statistics"]:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "metadata", key,
-                                              "average", value=None),
-                                summary_to["metadata"][key]["average"])
-                        if "diff" not in row[key] or \
-                                ("diff_treshold" in field and
-                                 abs(row[key]["diff"]) >=
-                                 field["diff_treshold"]):
+                                recursive_get(summary_from, "query_statistics", key, value=None),
+                                summary_to["query_statistics"][key],
+                            )
+                        elif not different_vendors:
+                            row[key] = compute_diff(
+                                recursive_get(summary_from, "metadata", key, "average", value=None),
+                                summary_to["metadata"][key]["average"],
+                            )
+                        if row.get(key) != None and (
+                            "diff" not in row[key]
+                            or ("diff_treshold" in field and abs(row[key]["diff"]) >= field["diff_treshold"])
+                        ):
                             performance_changed = True
                     if performance_changed:
                         ret[testcode] = row
@@ -130,29 +95,30 @@ def generate_remarkup(fields, data):
         ret += "<table>\n"
         ret += "  <tr>\n"
         ret += "    <th>Testcode</th>\n"
-        ret += "\n".join(map(lambda x: "    <th>{}</th>".format(
-            x["name"].replace("_", " ").capitalize()), fields)) + "\n"
+        ret += (
+            "\n".join(map(lambda x: "    <th>{}</th>".format(x["name"].replace("_", " ").capitalize()), fields)) + "\n"
+        )
         ret += "  </tr>\n"
         for testcode in sorted(data.keys()):
             ret += "  <tr>\n"
             ret += "    <td>{}</td>\n".format(testcode)
             for field in fields:
-                result = data[testcode][field["name"]]
-                value = result["value"] * field["scaling"]
-                if "diff" in result:
-                    diff = result["diff"]
-                    arrow = "arrow-up" if diff >= 0 else "arrow-down"
-                    if not (field["positive_diff_better"] ^ (diff >= 0)):
-                        color = "green"
+                result = data[testcode].get(field["name"])
+                if result != None:
+                    value = result["value"] * field["scaling"]
+                    if "diff" in result:
+                        diff = result["diff"]
+                        arrow = "arrow-up" if diff >= 0 else "arrow-down"
+                        if not (field["positive_diff_better"] ^ (diff >= 0)):
+                            color = "green"
+                        else:
+                            color = "red"
+                        sign = "{{icon {} color={}}}".format(arrow, color)
+                        ret += '    <td bgcolor="{}">{:.3f}{} ({:+.2%})</td>\n'.format(
+                            color, value, field["unit"], diff
+                        )
                     else:
-                        color = "red"
-                    sign = "{{icon {} color={}}}".format(arrow, color)
-                    ret += "    <td>{:.3f}{} //({:+.2%})// {}</td>\n".format(
-                        value, field["unit"], diff, sign)
-                else:
-                    ret += "    <td>{:.3f}{} //(new)// " \
-                           "{{icon plus color=blue}}</td>\n".format(
-                               value, field["unit"])
+                        ret += '<td bgcolor="blue">{:.3f}{} //(new)// </td>\n'.format(value, field["unit"])
             ret += "  </tr>\n"
         ret += "</table>\n"
     else:
@@ -161,24 +127,139 @@ def generate_remarkup(fields, data):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Compare results of multiple benchmark runs.")
-    parser.add_argument("--compare", action="append", nargs=2,
-                        metavar=("from", "to"),
-                        help="compare results between `from` and `to` files")
+    parser = argparse.ArgumentParser(description="Compare results of multiple benchmark runs.")
+    parser.add_argument(
+        "--compare",
+        action="append",
+        nargs=2,
+        metavar=("from", "to"),
+        help="compare results between `from` and `to` files",
+    )
     parser.add_argument("--output", default="", help="output file name")
+    # file is read line by line, each representing one test name
+    parser.add_argument("--exclude_tests_file", help="file listing test names to be excluded")
+
+    parser.add_argument(
+        "--different-vendors",
+        action="store_true",
+        default=False,
+        help="Comparing different vendors, there is no need for metadata, duration, count check.",
+    )
+    parser.add_argument(
+        "--difference-threshold", type=float, help="Difference threshold for memory and throughput, 0.02 = 2% "
+    )
+
     args = parser.parse_args()
 
+    fields = [
+        {
+            "name": "throughput",
+            "positive_diff_better": True,
+            "scaling": 1,
+            "unit": "QPS",
+            "diff_treshold": 0.05,  # 5%
+        },
+        {
+            "name": "duration",
+            "positive_diff_better": False,
+            "scaling": 1,
+            "unit": "s",
+        },
+        {
+            "name": "parsing_time",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "planning_time",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "plan_execution_time",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "memory",
+            "positive_diff_better": False,
+            "scaling": 1 / 1024 / 1024,
+            "unit": "MiB",
+            "diff_treshold": 0.02,  # 2%
+        },
+        {
+            "name": "max",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "p99",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "p90",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "p75",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "p50",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+        {
+            "name": "mean",
+            "positive_diff_better": False,
+            "scaling": 1000,
+            "unit": "ms",
+        },
+    ]
+
     if args.compare is None or len(args.compare) == 0:
         raise Exception("You must specify at least one pair of files!")
 
+    if args.exclude_tests_file:
+        with open(args.exclude_tests_file, "r") as f:
+            ignored = [line.rstrip("\n") for line in f]
+    else:
+        ignored = []
+
+    cleaned = []
+    if args.different_vendors:
+        ignore_on_different_vendors = {"duration", "parsing_time", "planning_time", "plan_execution_time"}
+        for field in fields:
+            key = field["name"]
+            if key in ignore_on_different_vendors:
+                continue
+            else:
+                cleaned.append(field)
+    fields = cleaned
+
+    if args.difference_threshold > 0.01:
+        for field in fields:
+            if "diff_treshold" in field.keys():
+                field["diff_treshold"] = args.difference_threshold
+
     data = {}
     for file_from, file_to in args.compare:
         results_from = load_results(file_from)
         results_to = load_results(file_to)
-        data.update(compare_results(results_from, results_to, FIELDS))
+        data.update(compare_results(results_from, results_to, fields, ignored, args.different_vendors))
 
-    remarkup = generate_remarkup(FIELDS, data)
+    remarkup = generate_remarkup(fields, data)
     if args.output:
         with open(args.output, "w") as f:
             f.write(remarkup)

tests/mgbench/datasets.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Memgraph Ltd.
+# Copyright 2022 Memgraph Ltd.
 #
 # Use of this software is governed by the Business Source License
 # included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,9 +24,12 @@ class Dataset:
     # One of the available variants that should be used as the default variant.
     DEFAULT_VARIANT = "default"
     # List of query files that should be used to import the dataset.
+    DEFAULT_VENDOR = "memgraph"
     FILES = {
         "default": "/foo/bar",
     }
+    INDEX = None
+    INDEX_FILES = {"default": ""}
     # List of query file URLs that should be used to import the dataset.
     URLS = None
     # Number of vertices/edges for each variant.
@@ -36,7 +39,7 @@ class Dataset:
     # Indicates whether the dataset has properties on edges.
     PROPERTIES_ON_EDGES = False
 
-    def __init__(self, variant=None):
+    def __init__(self, variant=None, vendor=None):
         """
         Accepts a `variant` variable that indicates which variant
         of the dataset should be executed.
@@ -49,7 +52,12 @@ class Dataset:
             raise ValueError("The variant doesn't have a defined URL or " "file path!")
         if variant not in self.SIZES:
             raise ValueError("The variant doesn't have a defined dataset " "size!")
+        if vendor is None:
+            vendor = self.DEFAULT_VENDOR
+        if vendor not in self.INDEX_FILES:
+            raise ValueError("Vendor does not have INDEX for dataset!")
         self._variant = variant
+        self._vendor = vendor
         if self.FILES is not None:
             self._file = self.FILES.get(variant, None)
         else:
@@ -58,6 +66,12 @@ class Dataset:
             self._url = self.URLS.get(variant, None)
         else:
             self._url = None
+
+        if self.INDEX_FILES is not None:
+            self._index = self.INDEX_FILES.get(vendor, None)
+        else:
+            self._index = None
+
         self._size = self.SIZES[variant]
         if "vertices" not in self._size or "edges" not in self._size:
             raise ValueError("The size defined for this variant doesn't " "have the number of vertices and/or edges!")
@@ -67,21 +81,34 @@ class Dataset:
     def prepare(self, directory):
         if self._file is not None:
             print("Using dataset file:", self._file)
-            return
-        # TODO: add support for JSON datasets
-        cached_input, exists = directory.get_file("dataset.cypher")
+        else:
+            # TODO: add support for JSON datasets
+            cached_input, exists = directory.get_file("dataset.cypher")
+            if not exists:
+                print("Downloading dataset file:", self._url)
+                downloaded_file = helpers.download_file(self._url, directory.get_path())
+                print("Unpacking and caching file:", downloaded_file)
+                helpers.unpack_and_move_file(downloaded_file, cached_input)
+            print("Using cached dataset file:", cached_input)
+            self._file = cached_input
+
+        cached_index, exists = directory.get_file(self._vendor + ".cypher")
         if not exists:
-            print("Downloading dataset file:", self._url)
-            downloaded_file = helpers.download_file(self._url, directory.get_path())
+            print("Downloading index file:", self._index)
+            downloaded_file = helpers.download_file(self._index, directory.get_path())
             print("Unpacking and caching file:", downloaded_file)
-            helpers.unpack_and_move_file(downloaded_file, cached_input)
-        print("Using cached dataset file:", cached_input)
-        self._file = cached_input
+            helpers.unpack_and_move_file(downloaded_file, cached_index)
+        print("Using cached index file:", cached_index)
+        self._index = cached_index
 
     def get_variant(self):
         """Returns the current variant of the dataset."""
         return self._variant
 
+    def get_index(self):
+        """Get index file, defined by vendor"""
+        return self._index
+
     def get_file(self):
         """
         Returns path to the file that contains dataset creation queries.
@@ -103,16 +130,26 @@ class Pokec(Dataset):
     VARIANTS = ["small", "medium", "large"]
     DEFAULT_VARIANT = "small"
     FILES = None
+
     URLS = {
         "small": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/pokec_small.setup.cypher",
         "medium": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/pokec_medium.setup.cypher",
         "large": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/pokec_large.setup.cypher.gz",
+        "small": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_small_import.cypher",
+        "medium": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_medium_import.cypher",
+        "large": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/pokec_large.setup.cypher.gz",
     }
     SIZES = {
         "small": {"vertices": 10000, "edges": 121716},
         "medium": {"vertices": 100000, "edges": 1768515},
         "large": {"vertices": 1632803, "edges": 30622564},
     }
+    INDEX = None
+    INDEX_FILES = {
+        "memgraph": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/memgraph.cypher",
+        "neo4j": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/pokec/benchmark/neo4j.cypher",
+    }
+
     PROPERTIES_ON_EDGES = False
 
     # Helpers used to generate the queries
@@ -135,7 +172,10 @@ class Pokec(Dataset):
         return ("MATCH (n:User {id : $id}) RETURN n", {"id": self._get_random_vertex()})
 
     def benchmark__arango__single_vertex_write(self):
-        return ("CREATE (n:UserTemp {id : $id}) RETURN n", {"id": random.randint(1, self._num_vertices * 10)})
+        return (
+            "CREATE (n:UserTemp {id : $id}) RETURN n",
+            {"id": random.randint(1, self._num_vertices * 10)},
+        )
 
     def benchmark__arango__single_edge_write(self):
         vertex_from, vertex_to = self._get_random_from_to()
@@ -151,7 +191,10 @@ class Pokec(Dataset):
         return ("MATCH (n:User) WHERE n.age >= 18 RETURN n.age, COUNT(*)", {})
 
     def benchmark__arango__expansion_1(self):
-        return ("MATCH (s:User {id: $id})-->(n:User) " "RETURN n.id", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (s:User {id: $id})-->(n:User) " "RETURN n.id",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__arango__expansion_1_with_filter(self):
         return (
@@ -160,7 +203,10 @@ class Pokec(Dataset):
         )
 
     def benchmark__arango__expansion_2(self):
-        return ("MATCH (s:User {id: $id})-->()-->(n:User) " "RETURN DISTINCT n.id", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (s:User {id: $id})-->()-->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__arango__expansion_2_with_filter(self):
         return (
@@ -193,7 +239,10 @@ class Pokec(Dataset):
         )
 
     def benchmark__arango__neighbours_2(self):
-        return ("MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__arango__neighbours_2_with_filter(self):
         return (
@@ -261,7 +310,10 @@ class Pokec(Dataset):
         return ("MATCH (n) RETURN min(n.age), max(n.age), avg(n.age)", {})
 
     def benchmark__match__pattern_cycle(self):
-        return ("MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) " "RETURN e1, m, e2", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) " "RETURN e1, m, e2",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__match__pattern_long(self):
         return (
@@ -270,10 +322,16 @@ class Pokec(Dataset):
         )
 
     def benchmark__match__pattern_short(self):
-        return ("MATCH (n:User {id: $id})-[e]->(m) " "RETURN m LIMIT 1", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (n:User {id: $id})-[e]->(m) " "RETURN m LIMIT 1",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__match__vertex_on_label_property(self):
-        return ("MATCH (n:User) WITH n WHERE n.id = $id RETURN n", {"id": self._get_random_vertex()})
+        return (
+            "MATCH (n:User) WITH n WHERE n.id = $id RETURN n",
+            {"id": self._get_random_vertex()},
+        )
 
     def benchmark__match__vertex_on_label_property_index(self):
         return ("MATCH (n:User {id: $id}) RETURN n", {"id": self._get_random_vertex()})
@@ -281,6 +339,132 @@ class Pokec(Dataset):
     def benchmark__match__vertex_on_property(self):
         return ("MATCH (n {id: $id}) RETURN n", {"id": self._get_random_vertex()})
 
+    # Basic benchmark queries
+
+    def benchmark__basic__single_vertex_read_read(self):
+        return ("MATCH (n:User {id : $id}) RETURN n", {"id": self._get_random_vertex()})
+
+    def benchmark__basic__single_vertex_write_write(self):
+        return (
+            "CREATE (n:UserTemp {id : $id}) RETURN n",
+            {"id": random.randint(1, self._num_vertices * 10)},
+        )
+
+    def benchmark__basic__single_vertex_property_update_update(self):
+        return (
+            "MATCH (n {id: $id}) SET n.property = -1",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__single_edge_write_write(self):
+        vertex_from, vertex_to = self._get_random_from_to()
+        return (
+            "MATCH (n:User {id: $from}), (m:User {id: $to}) WITH n, m " "CREATE (n)-[e:Temp]->(m) RETURN e",
+            {"from": vertex_from, "to": vertex_to},
+        )
+
+    def benchmark__basic__aggregate_aggregate(self):
+        return ("MATCH (n:User) RETURN n.age, COUNT(*)", {})
+
+    def benchmark__basic__aggregate_count_aggregate(self):
+        return ("MATCH (n) RETURN count(n), count(n.age)", {})
+
+    def benchmark__basic__aggregate_with_filter_aggregate(self):
+        return ("MATCH (n:User) WHERE n.age >= 18 RETURN n.age, COUNT(*)", {})
+
+    def benchmark__basic__min_max_avg_aggregate(self):
+        return ("MATCH (n) RETURN min(n.age), max(n.age), avg(n.age)", {})
+
+    def benchmark__basic__expansion_1_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->(n:User) " "RETURN n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_1_with_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->(n:User) " "WHERE n.age >= 18 " "RETURN n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_2_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_2_with_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_3_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_3_with_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_4_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__expansion_4_with_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-->()-->()-->()-->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__neighbours_2_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__neighbours_2_with_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__neighbours_2_with_data_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "RETURN DISTINCT n.id, n",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__neighbours_2_with_data_and_filter_analytical(self):
+        return (
+            "MATCH (s:User {id: $id})-[*1..2]->(n:User) " "WHERE n.age >= 18 " "RETURN DISTINCT n.id, n",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__pattern_cycle_analytical(self):
+        return (
+            "MATCH (n:User {id: $id})-[e1]->(m)-[e2]->(n) " "RETURN e1, m, e2",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__pattern_long_analytical(self):
+        return (
+            "MATCH (n1:User {id: $id})-[e1]->(n2)-[e2]->" "(n3)-[e3]->(n4)<-[e4]-(n5) " "RETURN n5 LIMIT 1",
+            {"id": self._get_random_vertex()},
+        )
+
+    def benchmark__basic__pattern_short_analytical(self):
+        return (
+            "MATCH (n:User {id: $id})-[e]->(m) " "RETURN m LIMIT 1",
+            {"id": self._get_random_vertex()},
+        )
+
 
 class AccessControl(Dataset):
 
@@ -299,6 +483,11 @@ class AccessControl(Dataset):
         "medium": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/accesscontrol/accesscontrol_medium.setup.cypher.gz",
         "large": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/accesscontrol/accesscontrol_large.setup.cypher.gz",
     }
+
+    INDEX_FILES = {
+        "memgraph": "https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/accesscontrol/accesscontrol_empty_only_index.setup.cypher.gz",
+    }
+
     SIZES = {
         "empty_only_index": {
             "vertices": 0,
@@ -347,7 +536,7 @@ class AccessControl(Dataset):
         random_value = random.randint(first_uuid, last_uuid)
         return random_value
 
-    def __init__(self, variant=None):
+    def __init__(self, variant=None, vendor=None):
         super().__init__(variant)
         self.next_value_idx = self.get_size()["vertices"] + 1
 
@@ -380,5 +569,30 @@ class AccessControl(Dataset):
 
     def benchmark__match__match_all_vertices_with_edges(self):
         self.next_value_idx += 1
-        query = ("MATCH (permission:Permission)-[e:IS_FOR_FILE]->(file:File) RETURN *", {})
+        query = (
+            "MATCH (permission:Permission)-[e:IS_FOR_FILE]->(file:File) RETURN *",
+            {},
+        )
         return query
+
+
+class TestDataset(Dataset):
+    NAME = "test"
+    VARIANTS = ["small"]
+    DEFAULT_VARIANT = "small"
+    # TODO(gitbuda): ~/Downloads/ doesn't work for some reason, fix! :)
+    # TODO(gitbuda): If ~ is wrong, client fails just with SIGABRT -> hard to figure out -> enable more details from client.
+    FILES = {
+        "small": "/home/buda/Downloads/mgbench/test_dataset_small.cypherl",
+    }
+    SIZES = {
+        "small": {"vertices": 10, "edges": 9},
+    }
+    # TODO(gitbuda): What's the purpose of the INDEX inside the Dataset?
+    # TODO(gitbuda): It's not possible to also inject local index file, it's much easier to add your local file compare to uploading it somewhere or running local HTTP server (easy with e.g. python, but...)
+    INDEX_FILES = {
+        "memgraph": "http://localhost:8080/test_dataset_index.cypherl",
+    }
+
+    def benchmark__basic__test(self):
+        return ("MATCH (n:Label {id: 0})-[*]->(m) RETURN n, m;", {})

tests/mgbench/graph_bench.py

@@ -0,0 +1,212 @@
+import argparse
+import json
+import subprocess
+from pathlib import Path
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        description="Run graph database benchmarks on supported databases(Memgraph and Neo4j)",
+    )
+    parser.add_argument(
+        "--vendor",
+        nargs=2,
+        action="append",
+        metavar=("vendor_name", "vendor_binary"),
+        help="Forward name and paths to vendors binary"
+        "Example: --vendor memgraph /path/to/binary --vendor neo4j /path/to/binary",
+    )
+    parser.add_argument(
+        "--dataset-size",
+        default="small",
+        choices=["small", "medium", "large"],
+        help="Pick a dataset size (small, medium, large)",
+    )
+
+    parser.add_argument("--dataset-group", default="basic", help="Select a group of queries")
+
+    parser.add_argument(
+        "--realistic",
+        nargs=5,
+        action="append",
+        metavar=("num_of_queries", "write", "read", "update", "analytical"),
+        help="Forward config for group run",
+    )
+
+    parser.add_argument(
+        "--mixed",
+        nargs=6,
+        action="append",
+        metavar=(
+            "num_of_queries",
+            "write",
+            "read",
+            "update",
+            "analytical",
+            "query_percentage",
+        ),
+        help="Forward config for query",
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+def run_full_benchmarks(vendor, binary, dataset_size, dataset_group, realistic, mixed):
+
+    configurations = [
+        # Basic full group test cold
+        [
+            "--export-results",
+            vendor + "_" + dataset_size + "_cold_isolated.json",
+        ],
+        # Basic full group test hot
+        [
+            "--export-results",
+            vendor + "_" + dataset_size + "_hot_isolated.json",
+            "--warmup-run",
+        ],
+    ]
+
+    # Configurations for full workload
+    for count, write, read, update, analytical in realistic:
+        cold = [
+            "--export-results",
+            vendor
+            + "_"
+            + dataset_size
+            + "_cold_realistic_{}_{}_{}_{}_{}.json".format(count, write, read, update, analytical),
+            "--mixed-workload",
+            count,
+            write,
+            read,
+            update,
+            analytical,
+        ]
+
+        hot = [
+            "--export-results",
+            vendor
+            + "_"
+            + dataset_size
+            + "_hot_realistic_{}_{}_{}_{}_{}.json".format(count, write, read, update, analytical),
+            "--warmup-run",
+            "--mixed-workload",
+            count,
+            write,
+            read,
+            update,
+            analytical,
+        ]
+        configurations.append(cold)
+        configurations.append(hot)
+
+    # Configurations for workload per query
+    for count, write, read, update, analytical, query in mixed:
+        cold = [
+            "--export-results",
+            vendor
+            + "_"
+            + dataset_size
+            + "_cold_mixed_{}_{}_{}_{}_{}_{}.json".format(count, write, read, update, analytical, query),
+            "--mixed-workload",
+            count,
+            write,
+            read,
+            update,
+            analytical,
+            query,
+        ]
+        hot = [
+            "--export-results",
+            vendor
+            + "_"
+            + dataset_size
+            + "_hot_mixed_{}_{}_{}_{}_{}_{}.json".format(count, write, read, update, analytical, query),
+            "--warmup-run",
+            "--mixed-workload",
+            count,
+            write,
+            read,
+            update,
+            analytical,
+            query,
+        ]
+        configurations.append(cold)
+        configurations.append(hot)
+
+    default_args = [
+        "python3",
+        "benchmark.py",
+        "--vendor-binary",
+        binary,
+        "--vendor-name",
+        vendor,
+        "--num-workers-for-benchmark",
+        "12",
+        "--no-authorization",
+        "pokec/" + dataset_size + "/" + dataset_group + "/*",
+        "--tail-latency",
+        "100",
+    ]
+
+    for config in configurations:
+        full_config = default_args + config
+        print(full_config)
+        subprocess.run(args=full_config, check=True)
+
+
+def collect_all_results(vendor_name, dataset_size, dataset_group):
+    working_directory = Path().absolute()
+    print(working_directory)
+    results = sorted(working_directory.glob(vendor_name + "_" + dataset_size + "_*.json"))
+    summary = {"pokec": {dataset_size: {dataset_group: {}}}}
+
+    for file in results:
+        if "summary" in file.name:
+            continue
+        f = file.open()
+        data = json.loads(f.read())
+        if data["__run_configuration__"]["condition"] == "hot":
+            for key, value in data["pokec"][dataset_size][dataset_group].items():
+                key_condition = key + "_hot"
+                summary["pokec"][dataset_size][dataset_group][key_condition] = value
+        elif data["__run_configuration__"]["condition"] == "cold":
+            for key, value in data["pokec"][dataset_size][dataset_group].items():
+                key_condition = key + "_cold"
+                summary["pokec"][dataset_size][dataset_group][key_condition] = value
+
+    print(summary)
+
+    json_object = json.dumps(summary, indent=4)
+    print(json_object)
+    with open(vendor_name + "_" + dataset_size + "_summary.json", "w") as f:
+        json.dump(summary, f)
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+
+    realistic = args.realistic
+    mixed = args.mixed
+
+    vendor_names = {"memgraph", "neo4j"}
+    for vendor_name, vendor_binary in args.vendor:
+        path = Path(vendor_binary)
+        if vendor_name.lower() in vendor_names and (path.is_file() or path.is_dir()):
+            run_full_benchmarks(
+                vendor_name,
+                vendor_binary,
+                args.dataset_size,
+                args.dataset_group,
+                realistic,
+                mixed,
+            )
+            collect_all_results(vendor_name, args.dataset_size, args.dataset_group)
+        else:
+            raise Exception(
+                "Check that vendor: {} is supported and you are passing right path: {} to binary.".format(
+                    vendor_name, path
+                )
+            )

tests/mgbench/helpers.py

@@ -27,7 +27,7 @@ def get_binary_path(path, base=""):
 
 
 def download_file(url, path):
-    if "https://" in url:
+    if "http://" in url or "https://" in url:
         ret = subprocess.run(
             ["wget", "-nv", "--content-disposition", url], stderr=subprocess.PIPE, cwd=path, check=True
         )

tests/mgbench/runners.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Memgraph Ltd.
+# Copyright 2022 Memgraph Ltd.
 #
 # Use of this software is governed by the Business Source License
 # included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,9 @@ import os
 import re
 import subprocess
 import tempfile
+import threading
 import time
+from pathlib import Path
 
 
 def wait_for_server(port, delay=0.1):
@@ -50,12 +52,28 @@ def _get_usage(pid):
     return {"cpu": total_cpu, "memory": peak_rss}
 
 
+def _get_current_usage(pid):
+    rss = 0
+    with open("/proc/{}/status".format(pid)) as f:
+        for row in f:
+            tmp = row.split()
+            if tmp[0] == "VmRSS:":
+                rss = int(tmp[1])
+    return rss / 1024
+
+
 class Memgraph:
-    def __init__(self, memgraph_binary, temporary_dir, properties_on_edges, extra_args):
+    def __init__(
+        self, memgraph_binary, temporary_dir, properties_on_edges, bolt_port, performance_tracking, extra_args
+    ):
         self._memgraph_binary = memgraph_binary
         self._directory = tempfile.TemporaryDirectory(dir=temporary_dir)
         self._properties_on_edges = properties_on_edges
         self._proc_mg = None
+        self._bolt_port = bolt_port
+        self.performance_tracking = performance_tracking
+        self._stop_event = threading.Event()
+        self._rss = []
         self._extra_args = extra_args
         atexit.register(self._cleanup)
 
@@ -97,6 +115,8 @@ class Memgraph:
             self._proc_mg = None
             raise Exception("The database process died prematurely!")
         wait_for_server(7687)
+        # TODO(gitbuda): Add better logging
+        print("Memgraph is running...")
         ret = self._proc_mg.poll()
         assert ret is None, "The database process died prematurely " "({})!".format(ret)
 
@@ -109,29 +129,256 @@ class Memgraph:
         self._proc_mg = None
         return ret, usage
 
-    def start_preparation(self):
-        if self._memgraph_version >= (0, 50, 0):
-            self._start(storage_snapshot_on_exit=True)
-        else:
-            self._start(snapshot_on_exit=True)
+    def start_preparation(self, workload):
+        if self.performance_tracking:
+            p = threading.Thread(target=self.res_background_tracking, args=(self._rss, self._stop_event))
+            self._stop_event.clear()
+            self._rss.clear()
+            p.start()
+        self._start(storage_snapshot_on_exit=True)
 
-    def start_benchmark(self):
-        # TODO: support custom benchmarking config files!
-        if self._memgraph_version >= (0, 50, 0):
-            self._start(storage_recover_on_startup=True)
-        else:
-            self._start(db_recover_on_startup=True)
+    def start_benchmark(self, workload):
+        if self.performance_tracking:
+            p = threading.Thread(target=self.res_background_tracking, args=(self._rss, self._stop_event))
+            self._stop_event.clear()
+            self._rss.clear()
+            p.start()
+        self._start(storage_recover_on_startup=True)
 
-    def stop(self):
+    def res_background_tracking(self, res, stop_event):
+        print("Started rss tracking.")
+        while not stop_event.is_set():
+            if self._proc_mg != None:
+                self._rss.append(_get_current_usage(self._proc_mg.pid))
+            time.sleep(0.05)
+        print("Stopped rss tracking. ")
+
+    def dump_rss(self, workload):
+        file_name = workload + "_rss"
+        Path.mkdir(Path().cwd() / "memgraph_memory", exist_ok=True)
+        file = Path(Path().cwd() / "memgraph_memory" / file_name)
+        file.touch()
+        with file.open("r+") as f:
+            for rss in self._rss:
+                f.write(str(rss))
+                f.write("\n")
+            f.close()
+
+    def stop(self, workload):
+        if self.performance_tracking:
+            self._stop_event.set()
+            self.dump_rss(workload)
         ret, usage = self._cleanup()
         assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret)
         return usage
 
 
+class Neo4j:
+    def __init__(self, neo4j_path, temporary_dir, bolt_port, performance_tracking):
+        self._neo4j_path = Path(neo4j_path)
+        self._neo4j_binary = Path(neo4j_path) / "bin" / "neo4j"
+        self._neo4j_config = Path(neo4j_path) / "conf" / "neo4j.conf"
+        self._neo4j_pid = Path(neo4j_path) / "run" / "neo4j.pid"
+        self._neo4j_admin = Path(neo4j_path) / "bin" / "neo4j-admin"
+        self.performance_tracking = performance_tracking
+        self._stop_event = threading.Event()
+        self._rss = []
+
+        if not self._neo4j_binary.is_file():
+            raise Exception("Wrong path to binary!")
+        self._directory = tempfile.TemporaryDirectory(dir=temporary_dir)
+        self._bolt_port = bolt_port
+        atexit.register(self._cleanup)
+        configs = []
+        memory_flag = "server.jvm.additional=-XX:NativeMemoryTracking=detail"
+        auth_flag = "dbms.security.auth_enabled=false"
+
+        if self.performance_tracking:
+            configs.append(memory_flag)
+        else:
+            lines = []
+            with self._neo4j_config.open("r") as file:
+                lines = file.readlines()
+                file.close()
+
+            for i in range(0, len(lines)):
+                if lines[i].strip("\n") == memory_flag:
+                    print("Clear up config flag:  " + memory_flag)
+                    lines[i] = "\n"
+                    print(lines[i])
+
+            with self._neo4j_config.open("w") as file:
+                file.writelines(lines)
+                file.close()
+
+        configs.append(auth_flag)
+        print("Check neo4j config flags:")
+        for conf in configs:
+            with self._neo4j_config.open("r+") as file:
+                lines = file.readlines()
+                line_exist = False
+                for line in lines:
+                    if conf == line.rstrip():
+                        line_exist = True
+                        print("Config line exist at line: " + str(lines.index(line)))
+                        print("Line content: " + line)
+                        file.close()
+                        break
+                if not line_exist:
+                    print("Setting config line: " + conf)
+                    file.write(conf)
+                    file.write("\n")
+                    file.close()
+
+    def __del__(self):
+        self._cleanup()
+        atexit.unregister(self._cleanup)
+
+    def _start(self, **kwargs):
+        if self._neo4j_pid.exists():
+            raise Exception("The database process is already running!")
+        args = _convert_args_to_flags(self._neo4j_binary, "start", **kwargs)
+        start_proc = subprocess.run(args, check=True)
+        time.sleep(5)
+        if self._neo4j_pid.exists():
+            print("Neo4j started!")
+        else:
+            raise Exception("The database process died prematurely!")
+        print("Run server check:")
+        wait_for_server(self._bolt_port)
+
+    def _cleanup(self):
+        if self._neo4j_pid.exists():
+            pid = self._neo4j_pid.read_text()
+            print("Clean up: " + pid)
+            usage = _get_usage(pid)
+
+            exit_proc = subprocess.run(args=[self._neo4j_binary, "stop"], capture_output=True, check=True)
+            return exit_proc.returncode, usage
+        else:
+            return 0
+
+    def start_preparation(self, workload):
+        if self.performance_tracking:
+            p = threading.Thread(target=self.res_background_tracking, args=(self._rss, self._stop_event))
+            self._stop_event.clear()
+            self._rss.clear()
+            p.start()
+
+        # Start DB
+        self._start()
+
+        if self.performance_tracking:
+            self.get_memory_usage("start_" + workload)
+
+    def start_benchmark(self, workload):
+        if self.performance_tracking:
+            p = threading.Thread(target=self.res_background_tracking, args=(self._rss, self._stop_event))
+            self._stop_event.clear()
+            self._rss.clear()
+            p.start()
+        # Start DB
+        self._start()
+
+        if self.performance_tracking:
+            self.get_memory_usage("start_" + workload)
+
+    def dump_db(self, path):
+        print("Dumping the neo4j database...")
+        if self._neo4j_pid.exists():
+            raise Exception("Cannot dump DB because it is running.")
+        else:
+            subprocess.run(
+                args=[
+                    self._neo4j_admin,
+                    "database",
+                    "dump",
+                    "--overwrite-destination=false",
+                    "--to-path",
+                    path,
+                    "neo4j",
+                ],
+                check=True,
+            )
+
+    def load_db_from_dump(self, path):
+        print("Loading the neo4j database from dump...")
+        if self._neo4j_pid.exists():
+            raise Exception("Cannot dump DB because it is running.")
+        else:
+            subprocess.run(
+                args=[
+                    self._neo4j_admin,
+                    "database",
+                    "load",
+                    "--from-path=" + path,
+                    "--overwrite-destination=true",
+                    "neo4j",
+                ],
+                check=True,
+            )
+
+    def res_background_tracking(self, res, stop_event):
+        print("Started rss tracking.")
+        while not stop_event.is_set():
+            if self._neo4j_pid.exists():
+                pid = self._neo4j_pid.read_text()
+                self._rss.append(_get_current_usage(pid))
+            time.sleep(0.05)
+        print("Stopped rss tracking. ")
+
+    def is_stopped(self):
+        pid_file = self._neo4j_path / "run" / "neo4j.pid"
+        if pid_file.exists():
+
+            return False
+        else:
+            return True
+
+    def stop(self, workload):
+        if self.performance_tracking:
+            self._stop_event.set()
+            self.get_memory_usage("stop_" + workload)
+            self.dump_rss(workload)
+        ret, usage = self._cleanup()
+        assert ret == 0, "The database process exited with a non-zero " "status ({})!".format(ret)
+        return usage
+
+    def dump_rss(self, workload):
+        file_name = workload + "_rss"
+        Path.mkdir(Path().cwd() / "neo4j_memory", exist_ok=True)
+        file = Path(Path().cwd() / "neo4j_memory" / file_name)
+        file.touch()
+        with file.open("r+") as f:
+            for rss in self._rss:
+                f.write(str(rss))
+                f.write("\n")
+            f.close()
+
+    def get_memory_usage(self, workload):
+        Path.mkdir(Path().cwd() / "neo4j_memory", exist_ok=True)
+
+        pid = self._neo4j_pid.read_text()
+        memory_usage = subprocess.run(args=["jcmd", pid, "VM.native_memory"], capture_output=True, text=True)
+        file = Path(Path().cwd() / "neo4j_memory" / workload)
+        if file.exists():
+            with file.open("r+") as f:
+                f.write(memory_usage.stdout)
+                f.close()
+        else:
+            file.touch()
+            with file.open("r+") as f:
+                f.write(memory_usage.stdout)
+                f.close()
+
+
 class Client:
-    def __init__(self, client_binary, temporary_directory):
+    def __init__(self, client_binary, temporary_directory, bolt_port: int, username: str = "", password: str = ""):
         self._client_binary = client_binary
         self._directory = tempfile.TemporaryDirectory(dir=temporary_directory)
+        self._username = username
+        self._password = password
+        self._bolt_port = bolt_port
 
     def _get_args(self, **kwargs):
         return _convert_args_to_flags(self._client_binary, **kwargs)
@@ -152,8 +399,15 @@ class Client:
                     json.dump(query, f)
                     f.write("\n")
 
-        args = self._get_args(input=file_path, num_workers=num_workers, queries_json=queries_json)
-        ret = subprocess.run(args, stdout=subprocess.PIPE, check=True)
+        args = self._get_args(
+            input=file_path,
+            num_workers=num_workers,
+            queries_json=queries_json,
+            username=self._username,
+            password=self._password,
+            port=self._bolt_port,
+        )
+        ret = subprocess.run(args, capture_output=True, check=True)
         data = ret.stdout.decode("utf-8").strip().split("\n")
         data = [x for x in data if not x.startswith("[")]
         return list(map(json.loads, data))

tests/simulation/CMakeLists.txt

@@ -32,3 +32,4 @@ add_simulation_test(trial_query_storage/query_storage_test.cpp)
 add_simulation_test(sharded_map.cpp)
 add_simulation_test(shard_rsm.cpp)
 add_simulation_test(cluster_property_test.cpp)
+add_simulation_test(request_router.cpp)

tests/simulation/common.hpp

@@ -76,14 +76,10 @@ class MockedShardRsm {
   using WriteRequests = msgs::WriteRequests;
   using WriteResponses = msgs::WriteResponses;
 
-  //  ExpandOneResponse Read(ExpandOneRequest rqst);
-  //  GetPropertiesResponse Read(GetPropertiesRequest rqst);
   msgs::ScanVerticesResponse ReadImpl(msgs::ScanVerticesRequest rqst) {
     msgs::ScanVerticesResponse ret;
     auto as_prop_val = storage::conversions::ConvertPropertyVector(rqst.start_id.second);
-    if (!IsKeyInRange(as_prop_val)) {
-      ret.success = false;
-    } else if (as_prop_val == ShardRsmKey{PropertyValue(0), PropertyValue(0)}) {
+    if (as_prop_val == ShardRsmKey{PropertyValue(0), PropertyValue(0)}) {
       msgs::Value val(int64_t(0));
       ret.next_start_id = std::make_optional<msgs::VertexId>();
       ret.next_start_id->second =
@@ -91,37 +87,46 @@ class MockedShardRsm {
       msgs::ScanResultRow result;
       result.props.push_back(std::make_pair(msgs::PropertyId::FromUint(0), val));
       ret.results.push_back(std::move(result));
-      ret.success = true;
     } else if (as_prop_val == ShardRsmKey{PropertyValue(1), PropertyValue(0)}) {
       msgs::ScanResultRow result;
       msgs::Value val(int64_t(1));
       result.props.push_back(std::make_pair(msgs::PropertyId::FromUint(0), val));
       ret.results.push_back(std::move(result));
-      ret.success = true;
     } else if (as_prop_val == ShardRsmKey{PropertyValue(12), PropertyValue(13)}) {
       msgs::ScanResultRow result;
       msgs::Value val(int64_t(444));
       result.props.push_back(std::make_pair(msgs::PropertyId::FromUint(0), val));
       ret.results.push_back(std::move(result));
-      ret.success = true;
-    } else {
-      ret.success = false;
     }
     return ret;
   }
 
   msgs::ExpandOneResponse ReadImpl(msgs::ExpandOneRequest rqst) { return {}; }
-  msgs::ExpandOneResponse ReadImpl(msgs::GetPropertiesRequest rqst) { return {}; }
+  msgs::GetPropertiesResponse ReadImpl(msgs::GetPropertiesRequest rqst) {
+    msgs::GetPropertiesResponse resp;
+    auto &vertices = rqst.vertex_ids;
+    for (auto &vertex : vertices) {
+      auto as_prop_val = storage::conversions::ConvertPropertyVector(vertex.second);
+      if (as_prop_val == ShardRsmKey{PropertyValue(0), PropertyValue(0)}) {
+        resp.result_row.push_back(msgs::GetPropertiesResultRow{.vertex = std::move(vertex)});
+      } else if (as_prop_val == ShardRsmKey{PropertyValue(1), PropertyValue(0)}) {
+        resp.result_row.push_back(msgs::GetPropertiesResultRow{.vertex = std::move(vertex)});
+      } else if (as_prop_val == ShardRsmKey{PropertyValue(13), PropertyValue(13)}) {
+        resp.result_row.push_back(msgs::GetPropertiesResultRow{.vertex = std::move(vertex)});
+      }
+    }
+    return resp;
+  }
 
   ReadResponses Read(ReadRequests read_requests) {
     return {std::visit([this]<typename T>(T &&request) { return ReadResponses{ReadImpl(std::forward<T>(request))}; },
                        std::move(read_requests))};
   }
 
-  msgs::CreateVerticesResponse ApplyImpl(msgs::CreateVerticesRequest rqst) { return {.success = true}; }
+  msgs::CreateVerticesResponse ApplyImpl(msgs::CreateVerticesRequest rqst) { return {}; }
   msgs::DeleteVerticesResponse ApplyImpl(msgs::DeleteVerticesRequest rqst) { return {}; }
   msgs::UpdateVerticesResponse ApplyImpl(msgs::UpdateVerticesRequest rqst) { return {}; }
-  msgs::CreateExpandResponse ApplyImpl(msgs::CreateExpandRequest rqst) { return {.success = true}; }
+  msgs::CreateExpandResponse ApplyImpl(msgs::CreateExpandRequest rqst) { return {}; }
   msgs::DeleteEdgesResponse ApplyImpl(msgs::DeleteEdgesRequest rqst) { return {}; }
   msgs::UpdateEdgesResponse ApplyImpl(msgs::UpdateEdgesRequest rqst) { return {}; }
   msgs::CommitResponse ApplyImpl(msgs::CommitRequest rqst) { return {}; }

tests/simulation/request_router.cpp

@@ -18,6 +18,8 @@
 #include <thread>
 #include <vector>
 
+#include <spdlog/cfg/env.h>
+
 #include "common.hpp"
 #include "common/types.hpp"
 #include "coordinator/coordinator_client.hpp"
@@ -44,8 +46,8 @@ using coordinator::CoordinatorClient;
 using coordinator::CoordinatorRsm;
 using coordinator::HlcRequest;
 using coordinator::HlcResponse;
-using coordinator::Shard;
 using coordinator::ShardMap;
+using coordinator::ShardMetadata;
 using coordinator::Shards;
 using coordinator::Status;
 using io::Address;
@@ -111,7 +113,7 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
   AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
   AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  Shard shard1 = {aas1_1, aas1_2, aas1_3};
+  ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
 
   auto key1 = storage::v3::PropertyValue(0);
   auto key2 = storage::v3::PropertyValue(0);
@@ -123,7 +125,7 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
   AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
   AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  Shard shard2 = {aas2_1, aas2_2, aas2_3};
+  ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
 
   auto key3 = storage::v3::PropertyValue(12);
   auto key4 = storage::v3::PropertyValue(13);
@@ -152,9 +154,7 @@ void RunStorageRaft(Raft<IoImpl, MockedShardRsm, WriteRequests, WriteResponses,
 }
 
 void TestScanVertices(query::v2::RequestRouterInterface &request_router) {
-  msgs::ExecutionState<ScanVerticesRequest> state{.label = "test_label"};
-
-  auto result = request_router.Request(state);
+  auto result = request_router.ScanVertices("test_label");
   MG_ASSERT(result.size() == 2);
   {
     auto prop = result[0].GetProperty(msgs::PropertyId::FromUint(0));
@@ -162,18 +162,10 @@ void TestScanVertices(query::v2::RequestRouterInterface &request_router) {
     prop = result[1].GetProperty(msgs::PropertyId::FromUint(0));
     MG_ASSERT(prop.int_v == 444);
   }
-
-  result = request_router.Request(state);
-  {
-    MG_ASSERT(result.size() == 1);
-    auto prop = result[0].GetProperty(msgs::PropertyId::FromUint(0));
-    MG_ASSERT(prop.int_v == 1);
-  }
 }
 
 void TestCreateVertices(query::v2::RequestRouterInterface &request_router) {
   using PropVal = msgs::Value;
-  msgs::ExecutionState<CreateVerticesRequest> state;
   std::vector<msgs::NewVertex> new_vertices;
   auto label_id = request_router.NameToLabel("test_label");
   msgs::NewVertex a1{.primary_key = {PropVal(int64_t(1)), PropVal(int64_t(0))}};
@@ -183,13 +175,13 @@ void TestCreateVertices(query::v2::RequestRouterInterface &request_router) {
   new_vertices.push_back(std::move(a1));
   new_vertices.push_back(std::move(a2));
 
-  auto result = request_router.Request(state, std::move(new_vertices));
+  auto result = request_router.CreateVertices(std::move(new_vertices));
   MG_ASSERT(result.size() == 2);
 }
 
 void TestCreateExpand(query::v2::RequestRouterInterface &request_router) {
   using PropVal = msgs::Value;
-  msgs::ExecutionState<msgs::CreateExpandRequest> state;
+  msgs::CreateExpandRequest state;
   std::vector<msgs::NewExpand> new_expands;
 
   const auto edge_type_id = request_router.NameToEdgeType("edge_type");
@@ -203,24 +195,42 @@ void TestCreateExpand(query::v2::RequestRouterInterface &request_router) {
   new_expands.push_back(std::move(expand_1));
   new_expands.push_back(std::move(expand_2));
 
-  auto responses = request_router.Request(state, std::move(new_expands));
+  auto responses = request_router.CreateExpand(std::move(new_expands));
   MG_ASSERT(responses.size() == 2);
-  MG_ASSERT(responses[0].success);
-  MG_ASSERT(responses[1].success);
+  MG_ASSERT(!responses[0].error);
+  MG_ASSERT(!responses[1].error);
 }
 
 void TestExpandOne(query::v2::RequestRouterInterface &request_router) {
-  msgs::ExecutionState<msgs::ExpandOneRequest> state{};
+  msgs::ExpandOneRequest state{};
   msgs::ExpandOneRequest request;
   const auto edge_type_id = request_router.NameToEdgeType("edge_type");
   const auto label = msgs::Label{request_router.NameToLabel("test_label")};
   request.src_vertices.push_back(msgs::VertexId{label, {msgs::Value(int64_t(0)), msgs::Value(int64_t(0))}});
   request.edge_types.push_back(msgs::EdgeType{edge_type_id});
   request.direction = msgs::EdgeDirection::BOTH;
-  auto result_rows = request_router.Request(state, std::move(request));
+  auto result_rows = request_router.ExpandOne(std::move(request));
   MG_ASSERT(result_rows.size() == 2);
 }
 
+void TestGetProperties(query::v2::RequestRouterInterface &request_router) {
+  using PropVal = msgs::Value;
+
+  auto label_id = request_router.NameToLabel("test_label");
+  msgs::VertexId v0{{label_id}, {PropVal(int64_t(0)), PropVal(int64_t(0))}};
+  msgs::VertexId v1{{label_id}, {PropVal(int64_t(1)), PropVal(int64_t(0))}};
+  msgs::VertexId v2{{label_id}, {PropVal(int64_t(13)), PropVal(int64_t(13))}};
+
+  msgs::GetPropertiesRequest request;
+
+  request.vertex_ids.push_back({v0});
+  request.vertex_ids.push_back({v1});
+  request.vertex_ids.push_back({v2});
+
+  auto result = request_router.GetProperties(std::move(request));
+  MG_ASSERT(result.size() == 3);
+}
+
 template <typename RequestRouter>
 void TestAggregate(RequestRouter &request_router) {}
 
@@ -338,11 +348,14 @@ void DoTest() {
   CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, c_addrs[0], c_addrs);
 
   query::v2::RequestRouter<SimulatorTransport> request_router(std::move(coordinator_client), std::move(cli_io));
+  std::function<bool()> tick_simulator = simulator.GetSimulatorTickClosure();
+  request_router.InstallSimulatorTicker(tick_simulator);
 
   request_router.StartTransaction();
   TestScanVertices(request_router);
   TestCreateVertices(request_router);
   TestCreateExpand(request_router);
+  TestGetProperties(request_router);
 
   simulator.ShutDown();
 
@@ -359,4 +372,7 @@ void DoTest() {
 }
 }  // namespace memgraph::query::v2::tests
 
-int main() { memgraph::query::v2::tests::DoTest(); }
+int main() {
+  spdlog::cfg::load_env_levels();
+  memgraph::query::v2::tests::DoTest();
+}

tests/simulation/shard_rsm.cpp

@@ -480,6 +480,65 @@ std::tuple<size_t, std::optional<msgs::VertexId>> AttemptToScanAllWithExpression
   }
 }
 
+msgs::GetPropertiesResponse AttemptToGetProperties(
+    ShardClient &client, std::optional<std::vector<PropertyId>> properties, std::vector<msgs::VertexId> vertices,
+    std::vector<msgs::EdgeId> edges, std::optional<size_t> limit = std::nullopt,
+    std::optional<uint64_t> filter_prop = std::nullopt, bool edge = false,
+    std::optional<std::string> order_by = std::nullopt) {
+  msgs::GetPropertiesRequest req{};
+  req.transaction_id.logical_id = GetTransactionId();
+  req.property_ids = std::move(properties);
+
+  if (filter_prop) {
+    std::string filter_expr = (!edge) ? "MG_SYMBOL_NODE.prop1 >= " : "MG_SYMBOL_EDGE.e_prop = ";
+    filter_expr += std::to_string(*filter_prop);
+    req.filter = std::make_optional(std::move(filter_expr));
+  }
+  if (order_by) {
+    std::string filter_expr = (!edge) ? "MG_SYMBOL_NODE." : "MG_SYMBOL_EDGE.";
+    filter_expr += *order_by;
+    msgs::OrderBy order_by{.expression = {std::move(filter_expr)}, .direction = msgs::OrderingDirection::DESCENDING};
+    std::vector<msgs::OrderBy> request_order_by;
+    request_order_by.push_back(std::move(order_by));
+    req.order_by = std::move(request_order_by);
+  }
+  if (limit) {
+    req.limit = limit;
+  }
+  req.expressions = {std::string("5 = 5")};
+  std::vector<msgs::VertexId> req_v;
+  std::vector<msgs::EdgeId> req_e;
+  for (auto &v : vertices) {
+    req_v.push_back(std::move(v));
+  }
+  for (auto &e : edges) {
+    req_e.push_back(std::move(e));
+  }
+
+  if (!edges.empty()) {
+    MG_ASSERT(edges.size() == vertices.size());
+    size_t id = 0;
+    req.vertices_and_edges.reserve(req_v.size());
+    for (auto &v : req_v) {
+      req.vertices_and_edges.push_back({std::move(v), std::move(req_e[id++])});
+    }
+  } else {
+    req.vertex_ids = std::move(req_v);
+  }
+
+  while (true) {
+    auto read_res = client.SendReadRequest(req);
+    if (read_res.HasError()) {
+      continue;
+    }
+
+    auto write_response_result = read_res.GetValue();
+    auto write_response = std::get<msgs::GetPropertiesResponse>(write_response_result);
+
+    return write_response;
+  }
+}
+
 void AttemptToScanAllWithOrderByOnPrimaryProperty(ShardClient &client, msgs::VertexId start_id, uint64_t batch_limit) {
   msgs::ScanVerticesRequest scan_req;
   scan_req.batch_limit = batch_limit;
@@ -1204,6 +1263,205 @@ void TestExpandOneGraphTwo(ShardClient &client) {
   }
 }
 
+void TestGetProperties(ShardClient &client) {
+  const auto unique_prop_val_1 = GetUniqueInteger();
+  const auto unique_prop_val_2 = GetUniqueInteger();
+  const auto unique_prop_val_3 = GetUniqueInteger();
+  const auto unique_prop_val_4 = GetUniqueInteger();
+  const auto unique_prop_val_5 = GetUniqueInteger();
+
+  MG_ASSERT(AttemptToCreateVertex(client, unique_prop_val_1));
+  MG_ASSERT(AttemptToCreateVertex(client, unique_prop_val_2));
+  MG_ASSERT(AttemptToCreateVertex(client, unique_prop_val_3));
+  MG_ASSERT(AttemptToCreateVertex(client, unique_prop_val_4));
+  MG_ASSERT(AttemptToCreateVertex(client, unique_prop_val_5));
+
+  const msgs::Label prim_label = {.id = get_primary_label()};
+  const msgs::PrimaryKey prim_key = {msgs::Value(static_cast<int64_t>(unique_prop_val_1))};
+  const msgs::VertexId v_id = {prim_label, prim_key};
+  const msgs::PrimaryKey prim_key_2 = {msgs::Value(static_cast<int64_t>(unique_prop_val_2))};
+  const msgs::VertexId v_id_2 = {prim_label, prim_key_2};
+  const msgs::PrimaryKey prim_key_3 = {msgs::Value(static_cast<int64_t>(unique_prop_val_3))};
+  const msgs::VertexId v_id_3 = {prim_label, prim_key_3};
+  const msgs::PrimaryKey prim_key_4 = {msgs::Value(static_cast<int64_t>(unique_prop_val_4))};
+  const msgs::VertexId v_id_4 = {prim_label, prim_key_4};
+  const msgs::PrimaryKey prim_key_5 = {msgs::Value(static_cast<int64_t>(unique_prop_val_5))};
+  const msgs::VertexId v_id_5 = {prim_label, prim_key_5};
+  const auto prop_id_2 = PropertyId::FromUint(2);
+  const auto prop_id_4 = PropertyId::FromUint(4);
+  const auto prop_id_5 = PropertyId::FromUint(5);
+  // No properties
+  {
+    const auto result = AttemptToGetProperties(client, {{}}, {v_id, v_id_2}, {});
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+    for (const auto &elem : result.result_row) {
+      MG_ASSERT(elem.props.size() == 0);
+    }
+  }
+  // All properties
+  {
+    const auto result = AttemptToGetProperties(client, std::nullopt, {v_id, v_id_2}, {});
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+    for (const auto &elem : result.result_row) {
+      MG_ASSERT(elem.props.size() == 3);
+    }
+  }
+  {
+    // Specific properties
+    const auto result =
+        AttemptToGetProperties(client, std::vector{prop_id_2, prop_id_4, prop_id_5}, {v_id, v_id_2, v_id_3}, {});
+    MG_ASSERT(!result.error);
+    MG_ASSERT(!result.result_row.empty());
+    MG_ASSERT(result.result_row.size() == 3);
+    for (const auto &elem : result.result_row) {
+      MG_ASSERT(elem.props.size() == 3);
+    }
+  }
+  {
+    // Two properties from two vertices with a filter on unique_prop_5
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2, prop_id_4}, {v_id, v_id_2, v_id_5}, {},
+                                               std::nullopt, unique_prop_val_5);
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 1);
+  }
+  {
+    // One property from three vertices.
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2}, {v_id, v_id_2, v_id_3}, {});
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 3);
+    MG_ASSERT(result.result_row[0].props.size() == 1);
+    MG_ASSERT(result.result_row[1].props.size() == 1);
+    MG_ASSERT(result.result_row[2].props.size() == 1);
+  }
+  {
+    // Same as before but with limit of 1 row
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2}, {v_id, v_id_2, v_id_3}, {},
+                                               std::make_optional<size_t>(1));
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 1);
+  }
+  {
+    // Same as before but with a limit greater than the elements returned
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2}, std::vector{v_id, v_id_2, v_id_3}, {},
+                                               std::make_optional<size_t>(5));
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 3);
+  }
+  {
+    // Order by on `prop1` (descending)
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2}, {v_id, v_id_2, v_id_3}, {}, std::nullopt,
+                                               std::nullopt, false, "prop1");
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 3);
+    MG_ASSERT(result.result_row[0].vertex == v_id_3);
+    MG_ASSERT(result.result_row[1].vertex == v_id_2);
+    MG_ASSERT(result.result_row[2].vertex == v_id);
+  }
+  {
+    // Order by and filter on >= unique_prop_val_3 && assert result row data members
+    const auto result = AttemptToGetProperties(client, std::vector{prop_id_2}, {v_id, v_id_2, v_id_3, v_id_4, v_id_5},
+                                               {}, std::nullopt, unique_prop_val_3, false, "prop1");
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 3);
+    MG_ASSERT(result.result_row[0].vertex == v_id_5);
+    MG_ASSERT(result.result_row[0].props.size() == 1);
+    MG_ASSERT(result.result_row[0].props.front().second == prim_key_5.front());
+    MG_ASSERT(result.result_row[0].props.size() == 1);
+    MG_ASSERT(result.result_row[0].props.front().first == prop_id_2);
+    MG_ASSERT(result.result_row[0].evaluated_expressions.size() == 1);
+    MG_ASSERT(result.result_row[0].evaluated_expressions.front() == msgs::Value(true));
+
+    MG_ASSERT(result.result_row[1].vertex == v_id_4);
+    MG_ASSERT(result.result_row[1].props.size() == 1);
+    MG_ASSERT(result.result_row[1].props.front().second == prim_key_4.front());
+    MG_ASSERT(result.result_row[1].props.size() == 1);
+    MG_ASSERT(result.result_row[1].props.front().first == prop_id_2);
+    MG_ASSERT(result.result_row[1].evaluated_expressions.size() == 1);
+    MG_ASSERT(result.result_row[1].evaluated_expressions.front() == msgs::Value(true));
+
+    MG_ASSERT(result.result_row[2].vertex == v_id_3);
+    MG_ASSERT(result.result_row[2].props.size() == 1);
+    MG_ASSERT(result.result_row[2].props.front().second == prim_key_3.front());
+    MG_ASSERT(result.result_row[2].props.size() == 1);
+    MG_ASSERT(result.result_row[2].props.front().first == prop_id_2);
+    MG_ASSERT(result.result_row[2].evaluated_expressions.size() == 1);
+    MG_ASSERT(result.result_row[2].evaluated_expressions.front() == msgs::Value(true));
+  }
+
+  // Edges
+  const auto edge_gid = GetUniqueInteger();
+  const auto edge_type_id = EdgeTypeId::FromUint(GetUniqueInteger());
+  const auto unique_edge_prop_id = 7;
+  const auto edge_prop_val = GetUniqueInteger();
+  MG_ASSERT(AttemptToAddEdgeWithProperties(client, unique_prop_val_1, unique_prop_val_2, edge_gid, unique_edge_prop_id,
+                                           edge_prop_val, {edge_type_id}));
+  const auto edge_gid_2 = GetUniqueInteger();
+  const auto edge_prop_val_2 = GetUniqueInteger();
+  MG_ASSERT(AttemptToAddEdgeWithProperties(client, unique_prop_val_3, unique_prop_val_4, edge_gid_2,
+                                           unique_edge_prop_id, edge_prop_val_2, {edge_type_id}));
+  const auto edge_prop_id = PropertyId::FromUint(unique_edge_prop_id);
+  std::vector<msgs::EdgeId> edge_ids = {{edge_gid}, {edge_gid_2}};
+  // No properties
+  {
+    const auto result = AttemptToGetProperties(client, {{}}, {v_id_2, v_id_3}, edge_ids);
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+    for (const auto &elem : result.result_row) {
+      MG_ASSERT(elem.props.size() == 0);
+    }
+  }
+  // All properties
+  {
+    const auto result = AttemptToGetProperties(client, std::nullopt, {v_id_2, v_id_3}, edge_ids);
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+    for (const auto &elem : result.result_row) {
+      MG_ASSERT(elem.props.size() == 1);
+    }
+  }
+  // Properties for two vertices
+  {
+    const auto result = AttemptToGetProperties(client, std::vector{edge_prop_id}, {v_id_2, v_id_3}, edge_ids);
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+  }
+  // Filter
+  {
+    const auto result = AttemptToGetProperties(client, std::vector{edge_prop_id}, {v_id_2, v_id_3}, edge_ids, {},
+                                               {edge_prop_val}, true);
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 1);
+    MG_ASSERT(result.result_row.front().edge);
+    MG_ASSERT(result.result_row.front().edge.value().gid == edge_gid);
+    MG_ASSERT(result.result_row.front().props.size() == 1);
+    MG_ASSERT(result.result_row.front().props.front().second == msgs::Value(static_cast<int64_t>(edge_prop_val)));
+  }
+  // Order by
+  {
+    const auto result =
+        AttemptToGetProperties(client, std::vector{edge_prop_id}, {v_id_2, v_id_3}, edge_ids, {}, {}, true, "e_prop");
+    MG_ASSERT(!result.error);
+    MG_ASSERT(result.result_row.size() == 2);
+    MG_ASSERT(result.result_row[0].vertex == v_id_3);
+    MG_ASSERT(result.result_row[0].edge);
+    MG_ASSERT(result.result_row[0].edge.value().gid == edge_gid_2);
+    MG_ASSERT(result.result_row[0].props.size() == 1);
+    MG_ASSERT(result.result_row[0].props.front().second == msgs::Value(static_cast<int64_t>(edge_prop_val_2)));
+    MG_ASSERT(result.result_row[0].evaluated_expressions.size() == 1);
+    MG_ASSERT(result.result_row[0].evaluated_expressions.front() == msgs::Value(true));
+
+    MG_ASSERT(result.result_row[1].vertex == v_id_2);
+    MG_ASSERT(result.result_row[1].edge);
+    MG_ASSERT(result.result_row[1].edge.value().gid == edge_gid);
+    MG_ASSERT(result.result_row[1].props.size() == 1);
+    MG_ASSERT(result.result_row[1].props.front().second == msgs::Value(static_cast<int64_t>(edge_prop_val)));
+    MG_ASSERT(result.result_row[1].evaluated_expressions.size() == 1);
+    MG_ASSERT(result.result_row[1].evaluated_expressions.front() == msgs::Value(true));
+  }
+}
+
 }  // namespace
 
 int TestMessages() {
@@ -1242,9 +1500,12 @@ int TestMessages() {
   auto shard_ptr2 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
   auto shard_ptr3 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
 
-  shard_ptr1->StoreMapping({{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}});
-  shard_ptr2->StoreMapping({{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}});
-  shard_ptr3->StoreMapping({{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}});
+  shard_ptr1->StoreMapping(
+      {{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}, {7, "e_prop"}});
+  shard_ptr2->StoreMapping(
+      {{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}, {7, "e_prop"}});
+  shard_ptr3->StoreMapping(
+      {{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}, {7, "e_prop"}});
 
   std::vector<Address> address_for_1{shard_server_2_address, shard_server_3_address};
   std::vector<Address> address_for_2{shard_server_1_address, shard_server_3_address};
@@ -1286,6 +1547,8 @@ int TestMessages() {
   TestExpandOneGraphOne(client);
   TestExpandOneGraphTwo(client);
 
+  // GetProperties tests
+  TestGetProperties(client);
   simulator.ShutDown();
 
   SimulatorStats stats = simulator.Stats();

tests/simulation/sharded_map.cpp

@@ -40,8 +40,8 @@ using memgraph::coordinator::CoordinatorRsm;
 using memgraph::coordinator::HlcRequest;
 using memgraph::coordinator::HlcResponse;
 using memgraph::coordinator::PrimaryKey;
-using memgraph::coordinator::Shard;
 using memgraph::coordinator::ShardMap;
+using memgraph::coordinator::ShardMetadata;
 using memgraph::coordinator::Shards;
 using memgraph::coordinator::Status;
 using memgraph::io::Address;
@@ -109,7 +109,7 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
   AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
   AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  Shard shard1 = {aas1_1, aas1_2, aas1_3};
+  ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
 
   const auto key1 = PropertyValue(0);
   const auto key2 = PropertyValue(0);
@@ -121,7 +121,7 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
   AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
   AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
 
-  Shard shard2 = {aas2_1, aas2_2, aas2_3};
+  ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
 
   auto key3 = PropertyValue(12);
   auto key4 = PropertyValue(13);
@@ -131,10 +131,10 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
   return sm;
 }
 
-std::optional<ShardClient *> DetermineShardLocation(const Shard &target_shard, const std::vector<Address> &a_addrs,
-                                                    ShardClient &a_client, const std::vector<Address> &b_addrs,
-                                                    ShardClient &b_client) {
-  for (const auto &addr : target_shard) {
+std::optional<ShardClient *> DetermineShardLocation(const ShardMetadata &target_shard,
+                                                    const std::vector<Address> &a_addrs, ShardClient &a_client,
+                                                    const std::vector<Address> &b_addrs, ShardClient &b_client) {
+  for (const auto &addr : target_shard.peers) {
     if (addr.address == b_addrs[0]) {
       return &b_client;
     }
@@ -275,7 +275,7 @@ int main() {
 
     const PrimaryKey compound_key = {cm_key_1, cm_key_2};
 
-    // Look for Shard
+    // Look for ShardMetadata
     BasicResult<TimedOut, memgraph::coordinator::CoordinatorWriteResponses> read_res =
         coordinator_client.SendWriteRequest(req);
 

tests/simulation/test_cluster.hpp

@@ -47,8 +47,8 @@ using coordinator::GetShardMapRequest;
 using coordinator::GetShardMapResponse;
 using coordinator::Hlc;
 using coordinator::HlcResponse;
-using coordinator::Shard;
 using coordinator::ShardMap;
+using coordinator::ShardMetadata;
 using io::Address;
 using io::Io;
 using io::rsm::RsmClient;
@@ -244,6 +244,8 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
   WaitForShardsToInitialize(coordinator_client);
 
   query::v2::RequestRouter<SimulatorTransport> request_router(std::move(coordinator_client), std::move(cli_io));
+  std::function<bool()> tick_simulator = simulator.GetSimulatorTickClosure();
+  request_router.InstallSimulatorTicker(tick_simulator);
 
   request_router.StartTransaction();
 

tests/unit/CMakeLists.txt

@@ -328,9 +328,6 @@ target_link_libraries(${test_prefix}query_v2_dummy_test mg-query-v2)
 add_unit_test(storage_v3_property_store.cpp)
 target_link_libraries(${test_prefix}storage_v3_property_store mg-storage-v3 fmt)
 
-add_unit_test(storage_v3_key_store.cpp)
-target_link_libraries(${test_prefix}storage_v3_key_store mg-storage-v3 rapidcheck rapidcheck_gtest)
-
 add_unit_test(storage_v3_indices.cpp)
 target_link_libraries(${test_prefix}storage_v3_indices mg-storage-v3)
 

tests/unit/future.cpp

@@ -28,13 +28,19 @@ void Wait(Future<std::string> future_1, Promise<std::string> promise_2) {
 
 TEST(Future, BasicLifecycle) {
   std::atomic_bool waiting = false;
+  std::atomic_bool filled = false;
 
-  std::function<bool()> notifier = [&] {
+  std::function<bool()> wait_notifier = [&] {
     waiting.store(true, std::memory_order_seq_cst);
     return false;
   };
 
-  auto [future_1, promise_1] = FuturePromisePairWithNotifier<std::string>(notifier);
+  std::function<bool()> fill_notifier = [&] {
+    filled.store(true, std::memory_order_seq_cst);
+    return false;
+  };
+
+  auto [future_1, promise_1] = FuturePromisePairWithNotifications<std::string>(wait_notifier, fill_notifier);
   auto [future_2, promise_2] = FuturePromisePair<std::string>();
 
   std::jthread t1(Wait, std::move(future_1), std::move(promise_2));
@@ -50,6 +56,8 @@ TEST(Future, BasicLifecycle) {
   t1.join();
   t2.join();
 
+  EXPECT_TRUE(filled.load(std::memory_order_acquire));
+
   std::string result_2 = std::move(future_2).Wait();
   EXPECT_TRUE(result_2 == "it worked");
 }

tests/unit/high_density_shard_create_scan.cpp

@@ -44,8 +44,8 @@ using coordinator::GetShardMapRequest;
 using coordinator::GetShardMapResponse;
 using coordinator::Hlc;
 using coordinator::HlcResponse;
-using coordinator::Shard;
 using coordinator::ShardMap;
+using coordinator::ShardMetadata;
 using io::Address;
 using io::Io;
 using io::local_transport::LocalSystem;
@@ -194,7 +194,8 @@ void ExecuteOp(query::v2::RequestRouter<LocalTransport> &request_router, std::se
                ScanAll scan_all) {
   auto results = request_router.ScanVertices("test_label");
 
-  MG_ASSERT(results.size() == correctness_model.size());
+  spdlog::error("got {} results, model size is {}", results.size(), correctness_model.size());
+  EXPECT_EQ(results.size(), correctness_model.size());
 
   for (const auto &vertex_accessor : results) {
     const auto properties = vertex_accessor.Properties();

tests/unit/machine_manager.cpp

@@ -45,8 +45,8 @@ using memgraph::coordinator::CoordinatorWriteRequests;
 using memgraph::coordinator::CoordinatorWriteResponses;
 using memgraph::coordinator::Hlc;
 using memgraph::coordinator::HlcResponse;
-using memgraph::coordinator::Shard;
 using memgraph::coordinator::ShardMap;
+using memgraph::coordinator::ShardMetadata;
 using memgraph::io::Io;
 using memgraph::io::local_transport::LocalSystem;
 using memgraph::io::local_transport::LocalTransport;

tests/unit/query_v2_expression_evaluator.cpp

@@ -51,6 +51,8 @@ using memgraph::msgs::CreateVerticesResponse;
 using memgraph::msgs::ExpandOneRequest;
 using memgraph::msgs::ExpandOneResponse;
 using memgraph::msgs::ExpandOneResultRow;
+using memgraph::msgs::GetPropertiesRequest;
+using memgraph::msgs::GetPropertiesResultRow;
 using memgraph::msgs::NewExpand;
 using memgraph::msgs::NewVertex;
 using memgraph::msgs::ScanVerticesRequest;
@@ -84,13 +86,16 @@ class MockedRequestRouter : public RequestRouterInterface {
   void Commit() override {}
   std::vector<VertexAccessor> ScanVertices(std::optional<std::string> /* label */) override { return {}; }
 
-  std::vector<CreateVerticesResponse> CreateVertices(std::vector<memgraph::msgs::NewVertex> new_vertices) override {
+  std::vector<CreateVerticesResponse> CreateVertices(
+      std::vector<memgraph::msgs::NewVertex> /* new_vertices */) override {
     return {};
   }
 
-  std::vector<ExpandOneResultRow> ExpandOne(ExpandOneRequest request) override { return {}; }
+  std::vector<ExpandOneResultRow> ExpandOne(ExpandOneRequest /* request */) override { return {}; }
 
-  std::vector<CreateExpandResponse> CreateExpand(std::vector<NewExpand> new_edges) override { return {}; }
+  std::vector<CreateExpandResponse> CreateExpand(std::vector<NewExpand> /* new_edges */) override { return {}; }
+
+  std::vector<GetPropertiesResultRow> GetProperties(GetPropertiesRequest rqst) override { return {}; }
 
   const std::string &PropertyToName(memgraph::storage::v3::PropertyId id) const override {
     return properties_.IdToName(id.AsUint());

tests/unit/storage_v3_key_store.cpp

@@ -1,46 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <algorithm>
-#include <string>
-#include <vector>
-
-/**
- * gtest/gtest.h must be included before rapidcheck/gtest.h!
- */
-#include <gtest/gtest.h>
-#include <rapidcheck.h>
-#include <rapidcheck/gtest.h>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/property_value.hpp"
-
-namespace memgraph::storage::v3::test {
-
-RC_GTEST_PROP(KeyStore, KeyStore, (std::vector<std::string> values)) {
-  RC_PRE(!values.empty());
-
-  std::vector<PropertyValue> property_values;
-  property_values.reserve(values.size());
-  std::transform(values.begin(), values.end(), std::back_inserter(property_values),
-                 [](std::string &value) { return PropertyValue{std::move(value)}; });
-
-  KeyStore key_store{property_values};
-
-  const auto keys = key_store.Keys();
-  RC_ASSERT(keys.size() == property_values.size());
-  for (int i = 0; i < keys.size(); ++i) {
-    RC_ASSERT(keys[i] == property_values[i]);
-    RC_ASSERT(key_store.GetKey(i) == property_values[i]);
-  }
-}
-}  // namespace memgraph::storage::v3::test

tools/plot/benchmark_datastructures.py

@@ -0,0 +1,185 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+####################################
+#  Benchmark datastructures analyzer
+####################################
+# This scripts uses the output from dataset benchmark tests to plot charts
+# comparing the results of different datastructures on the same operation.
+#
+# Note: Naming the tests is very important in order for this script to recognize
+# which operation is being performed and on which DS, so it should come in this
+# form: BM_Benchmark<Operation><Datastructure>/<RunArgument>
+# where run_argument will be added automatically by google benchmark framework
+
+import argparse
+import json
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import matplotlib.pyplot as plt
+
+
+class Operation(Enum):
+    CONTAINS = "contains"
+    FIND = "find"
+    INSERT = "insert"
+    RANDOM = "random"
+    REMOVE = "remove"
+
+    @classmethod
+    def to_list(cls) -> List[str]:
+        return list(map(lambda c: c.value, cls))
+
+    @staticmethod
+    def get(s: str) -> Optional["Operation"]:
+        try:
+            return Operation[s.upper()]
+        except ValueError:
+            return None
+
+    def __str__(self):
+        return str(self.value)
+
+
+@dataclass(frozen=True)
+class BenchmarkRow:
+    name: str
+    datastructure: str
+    operation: Operation
+    real_time: int
+    cpu_time: int
+    iterations: int
+    time_unit: str
+    run_arg: Optional[Any]
+
+
+class GoogleBenchmarkResult:
+    def __init__(self):
+        self._operation = None
+        self._datastructures: Dict[str, List[BenchmarkRow]] = dict()
+
+    def add_result(self, row: BenchmarkRow) -> None:
+        if self._operation is None:
+            self._operation = row.operation
+        assert self._operation is row.operation
+        if row.datastructure not in self._datastructures:
+            self._datastructures[row.datastructure] = [row]
+        else:
+            self._datastructures[row.datastructure].append(row)
+
+    @property
+    def operation(self) -> Optional[Operation]:
+        return self._operation
+
+    @property
+    def datastructures(self) -> Dict[str, List[BenchmarkRow]]:
+        return self._datastructures
+
+
+def get_operation(s: str) -> Operation:
+    for op in Operation.to_list():
+        if op.lower() in s.lower():
+            operation_enum = Operation.get(op)
+            if operation_enum is not None:
+                return operation_enum
+            else:
+                print("Operation not found!")
+                sys.exit(1)
+    print("Operation not found!")
+    sys.exit(1)
+
+
+def get_row_data(line: Dict[str, Any]) -> BenchmarkRow:
+    """
+    Naming is very important, first must come an Operation name, and then a data
+    structure to test.
+    """
+    full_name = line["name"].split("BM_Benchmark")[1]
+    name_with_run_arg = full_name.split("/")
+    operation = get_operation(name_with_run_arg[0])
+    datastructure = name_with_run_arg[0].split(operation.value.capitalize())[1]
+
+    run_arg = None
+    if len(name_with_run_arg) > 1:
+        run_arg = name_with_run_arg[1]
+
+    return BenchmarkRow(
+        name_with_run_arg[0],
+        datastructure,
+        operation,
+        line["real_time"],
+        line["cpu_time"],
+        line["iterations"],
+        line["time_unit"],
+        run_arg,
+    )
+
+
+def get_benchmark_res(args) -> Optional[GoogleBenchmarkResult]:
+    file_path = Path(args.log_file)
+    if not file_path.exists():
+        print("Error file {file_path} not found!")
+        return None
+    with file_path.open("r") as file:
+        data = json.load(file)
+        res = GoogleBenchmarkResult()
+        assert "benchmarks" in data, "There must be a benchmark list inside"
+        for benchmark in data["benchmarks"]:
+            res.add_result(get_row_data(benchmark))
+        return res
+
+
+def plot_operation(results: GoogleBenchmarkResult, save: bool) -> None:
+    colors = ["red", "green", "blue", "yellow", "purple", "brown"]
+    assert results.operation is not None
+    fig = plt.figure()
+    for ds, benchmarks in results.datastructures.items():
+        if benchmarks:
+            # Print line chart
+            x_axis = [elem.real_time for elem in benchmarks]
+            y_axis = [elem.run_arg for elem in benchmarks]
+            plt.plot(x_axis, y_axis, marker="", color=colors.pop(0), linewidth="2", label=f"{ds}")
+            plt.title(f"Benchmark results for operation {results.operation.value}")
+            plt.xlabel(f"Time [{benchmarks[0].time_unit}]")
+            plt.grid(True)
+            plt.legend()
+            plt.draw()
+        else:
+            print(f"Nothing to do for {ds}...")
+    if save:
+        plt.savefig(f"{results.operation.value}.png")
+        plt.close(fig)
+    else:
+        plt.show()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Process benchmark results.")
+    parser.add_argument("--log_file", type=str)
+    parser.add_argument("--save", type=bool, default=True)
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    res = get_benchmark_res(args)
+    if res is None:
+        print("Failed to get results from log file!")
+        sys.exit(1)
+    plot_operation(res, args.save)
+
+
+if __name__ == "__main__":
+    main()

tools/plot/benchmark_datastructures.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -euox pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+WORKSPACE_DIR=${SCRIPT_DIR}/../../
+CPUS=$(grep -c processor < /proc/cpuinfo)
+
+# Get all benchmark files
+BENCHMARK_FILES=$(find ${WORKSPACE_DIR}/tests/benchmark -type f -iname "data_structures_*")
+
+function test_all() {
+    for BENCH_FILE in ${BENCHMARK_FILES[@]}; do
+        local BASE_NAME=$(basename $BENCH_FILE)
+        local NAME=${BASE_NAME%%.*}
+        echo "Running $NAME"
+        local TEST_FILE=${WORKSPACE_DIR}/build/tests/benchmark/${NAME}
+        if [[ -f "${TEST_FILE}" ]]; then
+            pushd ${WORKSPACE_DIR}/build
+                make -j${CPUS} memgraph__benchmark__${NAME}
+            popd
+            local JSON_OUTPUT=${NAME}_output.json
+            # Run benchmakr test
+            ${WORKSPACE_DIR}/build/tests/benchmark/${NAME} --benchmark_format=json --benchmark_out=${JSON_OUTPUT}
+            # Run analyze script for benchmark test
+            python3 ${WORKSPACE_DIR}/tools/plot/benchmark_datastructures.py --log_file=${JSON_OUTPUT}
+        else
+            echo "File ${TEST_FILE} does not exist!"
+        fi
+    done
+}
+
+function test_memory() {
+    ## We are testing only insert
+    local DATA_STRUCTURES=(SkipList StdMap StdSet BppTree)
+    for DATA_STRUCTURE in ${DATA_STRUCTURES[@]}; do
+        valgrind --tool=massif --massif-out-file=${DATA_STRUCTURE}.massif.out ${WORKSPACE_DIR}/build/tests/benchmark/data_structures_insert --benchmark_filter=BM_BenchmarkInsert${DATA_STRUCTURE}/10000 --benchmark_format=json --benchmark_out=${DATA_STRUCTURE}.json
+    done
+}
+
+ARG_1=${1:-"all"}
+case ${ARG_1} in
+  all)
+    test_all
+    ;;
+  memory)
+    test_memory
+    ;;
+  *)
+    echo "Select either `all` or `memory` benchmark!"
+    ;;
+esac