Merge branch 'project-pineapples' into T1151-MG-distributed-shard-split-logic

- Implement shard split on storage side

.github/workflows/diff.yaml

@@ -99,7 +99,7 @@ jobs:
             echo ${file}
             if [[ ${file} == *.py ]]; then
               python3 -m black --check --diff ${file}
-              python3 -m isort --check-only --diff ${file}
+              python3 -m isort --check-only --profile "black" --diff ${file}
             fi
           done
 
@@ -271,3 +271,30 @@ jobs:
           source ve3/bin/activate
           cd e2e
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
+
+      - name: Run query performance tests
+        run: |
+          cd tests/manual
+          ./query_performance_runner.py
+
+      - name: Get branch name (merge)
+        if: github.event_name != 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
+
+      - name: Get branch name (pull request)
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+
+      - name: Upload macro benchmark results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "query_performance" \
+                    --benchmark-results-path "../../build/tests/manual/query_performance_benchmark/summary.json" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"

.pre-commit-config.yaml

@@ -14,6 +14,7 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
+        args: ["--profile", "black"]
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v13.0.0
     hooks:

CMakeLists.txt

@@ -4,6 +4,7 @@ cmake_minimum_required(VERSION 3.8)
 
 # !! IMPORTANT !! run ./project_root/init.sh before cmake command
 # to download dependencies
+
 if(NOT UNIX)
   message(FATAL_ERROR "Unsupported operating system.")
 endif()
@@ -18,7 +19,6 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
 find_program(CCACHE_FOUND ccache)
 option(USE_CCACHE "ccache:" ON)
 message(STATUS "CCache: ${USE_CCACHE}")
-
 if(CCACHE_FOUND AND USE_CCACHE)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
   set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
@@ -28,8 +28,7 @@ endif(CCACHE_FOUND AND USE_CCACHE)
 # NOTE: must be choosen before use of project() or enable_language()
 find_program(CLANG_FOUND clang)
 find_program(CLANGXX_FOUND clang++)
-
+if (CLANG_FOUND AND CLANGXX_FOUND)
-if(CLANG_FOUND AND CLANGXX_FOUND)
   set(CMAKE_C_COMPILER ${CLANG_FOUND})
   set(CMAKE_CXX_COMPILER ${CLANGXX_FOUND})
 else()
@@ -37,11 +36,12 @@ else()
 endif()
 
 # -----------------------------------------------------------------------------
+
 project(memgraph)
 
 # Install licenses.
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
-  DESTINATION share/doc/memgraph)
+        DESTINATION share/doc/memgraph)
 
 # For more information about how to release a new version of Memgraph, see
 # `release/README.md`.
@@ -61,65 +61,61 @@ set(MEMGRAPH_OVERRIDE_VERSION "")
 set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
 
 # Variables used to generate the versions.
-if(MG_ENTERPRISE)
+if (MG_ENTERPRISE)
   set(get_version_offering "")
 else()
   set(get_version_offering "--open-source")
 endif()
-
 set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
 
 # Get version that should be used in the binary.
 execute_process(
-  OUTPUT_VARIABLE MEMGRAPH_VERSION
+    OUTPUT_VARIABLE MEMGRAPH_VERSION
-  RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
+    RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
-  COMMAND "${get_version_script}" ${get_version_offering}
+    COMMAND "${get_version_script}" ${get_version_offering}
-  "${MEMGRAPH_OVERRIDE_VERSION}"
+                "${MEMGRAPH_OVERRIDE_VERSION}"
-  "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-  "--memgraph-root-dir"
+                "--memgraph-root-dir"
-  "${CMAKE_CURRENT_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
-
 if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
-  message(FATAL_ERROR "Unable to get Memgraph version.")
+    message(FATAL_ERROR "Unable to get Memgraph version.")
 else()
-  MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
+    MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
 endif()
 
 # Get version that should be used in the DEB package.
 execute_process(
-  OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
+    OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
-  RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
+    RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
-  COMMAND "${get_version_script}" ${get_version_offering}
+    COMMAND "${get_version_script}" ${get_version_offering}
-  --variant deb
+                --variant deb
-  "${MEMGRAPH_OVERRIDE_VERSION}"
+                "${MEMGRAPH_OVERRIDE_VERSION}"
-  "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-  "--memgraph-root-dir"
+                "--memgraph-root-dir"
-  "${CMAKE_CURRENT_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
-
 if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
-  message(FATAL_ERROR "Unable to get Memgraph DEB version.")
+    message(FATAL_ERROR "Unable to get Memgraph DEB version.")
 else()
-  MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
+    MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
 endif()
 
 # Get version that should be used in the RPM package.
 execute_process(
-  OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
+    OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
-  RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
+    RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
-  COMMAND "${get_version_script}" ${get_version_offering}
+    COMMAND "${get_version_script}" ${get_version_offering}
-  --variant rpm
+                --variant rpm
-  "${MEMGRAPH_OVERRIDE_VERSION}"
+                "${MEMGRAPH_OVERRIDE_VERSION}"
-  "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-  "--memgraph-root-dir"
+                "--memgraph-root-dir"
-  "${CMAKE_CURRENT_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
-
 if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
-  message(FATAL_ERROR "Unable to get Memgraph RPM version.")
+    message(FATAL_ERROR "Unable to get Memgraph RPM version.")
 else()
-  MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
+    MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
 endif()
 
 # We want the above variables to be updated each time something is committed to
@@ -139,24 +135,22 @@ endif()
 # unnecessary recalculations of the release version. The release version only
 # changes on every `git commit` or `git checkout`. That is why we watch the
 # following files for changes:
-# - `.git/HEAD` -> changes each time a `git checkout` is issued
+#   - `.git/HEAD` -> changes each time a `git checkout` is issued
-# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
+#   - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
-# you are on a branch) and you have to monitor the file of the specific
+#     you are on a branch) and you have to monitor the file of the specific
-# branch to detect when a `git commit` was issued
+#     branch to detect when a `git commit` was issued
 # More details about the contents of the `.git` directory and the specific
 # files used can be seen here:
 # https://git-scm.com/book/en/v2/Git-Internals-Git-References
 set(git_directory "${CMAKE_SOURCE_DIR}/.git")
-
+if (EXISTS "${git_directory}")
-if(EXISTS "${git_directory}")
   set_property(DIRECTORY APPEND PROPERTY
-    CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
+      CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
   file(STRINGS "${git_directory}/HEAD" git_head_data)
-
+  if (git_head_data MATCHES "^ref: ")
-  if(git_head_data MATCHES "^ref: ")
     string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref)
     set_property(DIRECTORY APPEND PROPERTY
-      CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
+        CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
   endif()
 endif()
 
@@ -165,19 +159,16 @@ endif()
 # setup CMake module path, defines path for include() and find_package()
 # https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
-
 # custom function definitions
 include(functions)
-
 # -----------------------------------------------------------------------------
 
 # We want out of source builds, so that cmake generated files don't get mixed
 # with source files. This allows for easier clean up.
 disallow_in_source_build()
 add_custom_target(clean_all
-  COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
+                  COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
-  COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
+                  COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
-
 # -----------------------------------------------------------------------------
 
 # build flags -----------------------------------------------------------------
@@ -188,19 +179,18 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
-
 # c99-designator is disabled because of required mixture of designated and
 # non-designated initializers in Python Query Module code (`py_module.cpp`).
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-    -Werror=switch -Werror=switch-bool -Werror=return-type \
+    -Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
+    -Werror=return-type \
     -Werror=return-stack-address \
     -Wno-c99-designator \
-    -DBOOST_STACKTRACE_USE_ADDR2LINE \
     -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
 
 # Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
-  "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
+    "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
 
 # Statically link libgcc and libstdc++, the GCC allows this according to:
 # https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
@@ -218,47 +208,42 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
 
 SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
 
-# debug flags
+#debug flags
 set(PREFERRED_DEBUGGER "gdb" CACHE STRING
-  "Tunes the debug output for your preferred debugger (gdb or lldb).")
+    "Tunes the debug output for your preferred debugger (gdb or lldb).")
-
+if ("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
-if("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
+    "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
-  "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
+    set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
-  set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
+elseif ("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
-elseif("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
+        "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-  "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
-  set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
 else()
-  message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
+    message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
-    "'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
+            "'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
-  set(CMAKE_CXX_FLAGS_DEBUG "-g")
+    set(CMAKE_CXX_FLAGS_DEBUG "-g")
 endif()
 
 # -----------------------------------------------------------------------------
 
 # default build type is debug
-if(NOT CMAKE_BUILD_TYPE)
+if (NOT CMAKE_BUILD_TYPE)
-  set(CMAKE_BUILD_TYPE "Debug")
+    set(CMAKE_BUILD_TYPE "Debug")
 endif()
-
 message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
-
 # -----------------------------------------------------------------------------
+
 set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
 
 # setup external dependencies -------------------------------------------------
 
 # threading
 find_package(Threads REQUIRED)
-
 # optional readline
 option(USE_READLINE "Use GNU Readline library if available (default ON). \
 Set this to OFF to prevent linking with Readline even if it is available." ON)
-
+if (USE_READLINE)
-if(USE_READLINE)
   find_package(Readline)
-
+  if (READLINE_FOUND)
-  if(READLINE_FOUND)
     add_definitions(-DHAS_READLINE)
   endif()
 endif()
@@ -274,27 +259,24 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
 option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
 option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
 
-if(TEST_COVERAGE)
+if (TEST_COVERAGE)
   string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
-
+  if (NOT lower_build_type STREQUAL "debug")
-  if(NOT lower_build_type STREQUAL "debug")
     message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'")
   endif()
-
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
 endif()
 
-if(MG_ENTERPRISE)
+if (MG_ENTERPRISE)
   add_definitions(-DMG_ENTERPRISE)
 endif()
 
 set(ENABLE_JEMALLOC ON)
 
-if(ASAN)
+if (ASAN)
   message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
   set(ENABLE_JEMALLOC OFF)
-
   # Enable Addres sanitizer and get nicer stack traces in error messages.
   # NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
   # distribution to symbolize the stack traces (note that ideally the
@@ -303,20 +285,19 @@ if(ASAN)
   # provide it in separate ASAN_SYMBOLIZER_PATH environment variable.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
-
   # To detect Stack-use-after-return bugs set run-time flag:
-  # ASAN_OPTIONS=detect_stack_use_after_return=1
+  #   ASAN_OPTIONS=detect_stack_use_after_return=1
   # To check initialization order bugs set run-time flag:
-  # ASAN_OPTIONS=check_initialization_order=true
+  #   ASAN_OPTIONS=check_initialization_order=true
-  # This mode reports an error if initializer for a global variable accesses
+  #     This mode reports an error if initializer for a global variable accesses
-  # dynamically initialized global from another translation unit, which is
+  #     dynamically initialized global from another translation unit, which is
-  # not yet initialized
+  #     not yet initialized
-  # ASAN_OPTIONS=strict_init_order=true
+  #   ASAN_OPTIONS=strict_init_order=true
-  # This mode reports an error if initializer for a global variable accesses
+  #     This mode reports an error if initializer for a global variable accesses
-  # any dynamically initialized global from another translation unit.
+  #     any dynamically initialized global from another translation unit.
 endif()
 
-if(TSAN)
+if (TSAN)
   # ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
   # If some code (e.g. dynamic libraries) is not compiled with the flag, it can
   # lead to false positive race reports, false negative race reports and/or
@@ -329,21 +310,19 @@ if(TSAN)
   # (unwinding stack on each memory access is too expensive).
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
-
   # By default ThreadSanitizer uses addr2line utility to symbolize reports.
   # llvm-symbolizer is faster, consumes less memory and produces much better
   # reports. To use it set runtime flag:
-  # TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
+  #   TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
   # For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
 endif()
 
-if(UBSAN)
+if (UBSAN)
   # Compile with UBSAN but disable vptr check. This is disabled because it
   # requires linking with clang++ to make sure C++ specific parts of the
   # runtime library and c++ standard libraries are present.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
-
   # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
   # Make sure llvm-symbolizer binary is in path.
   # To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
@@ -362,7 +341,7 @@ add_subdirectory(release)
 option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
 message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
 
-if(MG_ENABLE_TESTING)
+if (MG_ENABLE_TESTING)
   enable_testing()
   add_subdirectory(tests)
 endif()

libs/setup.sh

@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
 repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
 
 # google test
-googletest_tag="release-1.8.0"
+googletest_tag="release-1.12.1"
 repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
 
 # libbcrypt

src/CMakeLists.txt

@@ -21,6 +21,7 @@ add_subdirectory(auth)
 add_subdirectory(parser)
 add_subdirectory(expr)
 add_subdirectory(coordinator)
+add_subdirectory(functions)
 
 if (MG_ENTERPRISE)
   add_subdirectory(audit)

src/common/errors.hpp

@@ -0,0 +1,68 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <cstdint>
+#include <string_view>
+
+namespace memgraph::common {
+
+enum class ErrorCode : uint8_t {
+  SERIALIZATION_ERROR,
+  NONEXISTENT_OBJECT,
+  DELETED_OBJECT,
+  VERTEX_HAS_EDGES,
+  PROPERTIES_DISABLED,
+  VERTEX_ALREADY_INSERTED,
+  // Schema Violations
+  SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
+  SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
+  SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
+  SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
+  SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
+  SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
+
+  OBJECT_NOT_FOUND,
+};
+
+constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
+  switch (code) {
+    case ErrorCode::SERIALIZATION_ERROR:
+      return "SERIALIZATION_ERROR";
+    case ErrorCode::NONEXISTENT_OBJECT:
+      return "NONEXISTENT_OBJECT";
+    case ErrorCode::DELETED_OBJECT:
+      return "DELETED_OBJECT";
+    case ErrorCode::VERTEX_HAS_EDGES:
+      return "VERTEX_HAS_EDGES";
+    case ErrorCode::PROPERTIES_DISABLED:
+      return "PROPERTIES_DISABLED";
+    case ErrorCode::VERTEX_ALREADY_INSERTED:
+      return "VERTEX_ALREADY_INSERTED";
+    case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
+      return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
+    case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
+      return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
+    case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
+      return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
+    case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
+      return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
+    case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
+      return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
+    case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
+      return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
+    case ErrorCode::OBJECT_NOT_FOUND:
+      return "OBJECT_NOT_FOUND";
+  }
+}
+
+}  // namespace memgraph::common

src/communication/bolt/v1/states/executing.hpp

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
     }
     case Signature::Route: {
       if constexpr (bolt_minor >= 3) {
-        if (signature == Signature::Route) return HandleRoute<TSession>(session);
+        return HandleRoute<TSession>(session);
       } else {
         spdlog::trace("Supported only in bolt v4.3");
         return State::Close;

src/coordinator/coordinator.cpp

@@ -18,12 +18,7 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_r
 
   // add this storage engine to any under-replicated shards that it is not already a part of
 
-  auto initializing_rsms_for_shard_manager =
+  return shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
-      shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
-
-  return HeartbeatResponse{
-      .shards_to_initialize = initializing_rsms_for_shard_manager,
-  };
 }
 
 CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
@@ -67,12 +62,8 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
 CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
   SplitShardResponse res{};
 
-  if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
+  res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
-    res.success = false;
+                                      split_shard_request.split_key);
-  } else {
-    res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
-                                        split_shard_request.split_key);
-  }
 
   return res;
 }

src/coordinator/coordinator.hpp

@@ -121,15 +121,6 @@ struct InitializeLabelResponse {
   std::optional<ShardMap> fresher_shard_map;
 };
 
-struct HeartbeatRequest {
-  Address from_storage_manager;
-  std::set<boost::uuids::uuid> initialized_rsms;
-};
-
-struct HeartbeatResponse {
-  std::vector<ShardToInitialize> shards_to_initialize;
-};
-
 using CoordinatorWriteRequests =
     std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
                  DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;

src/coordinator/coordinator_worker.hpp

@@ -71,6 +71,9 @@ struct QueueInner {
   // starvation by sometimes randomizing priorities, rather than following a strict
   // prioritization.
   std::deque<Message> queue;
+
+  uint64_t submitted = 0;
+  uint64_t calls_to_pop = 0;
 };
 
 /// There are two reasons to implement our own Queue instead of using
@@ -86,6 +89,8 @@ class Queue {
       MG_ASSERT(inner_.use_count() > 0);
       std::unique_lock<std::mutex> lock(inner_->mu);
 
+      inner_->submitted++;
+
       inner_->queue.emplace_back(std::move(message));
     }  // lock dropped before notifying condition variable
 
@@ -96,6 +101,9 @@ class Queue {
     MG_ASSERT(inner_.use_count() > 0);
     std::unique_lock<std::mutex> lock(inner_->mu);
 
+    inner_->calls_to_pop++;
+    inner_->cv.notify_all();
+
     while (inner_->queue.empty()) {
       inner_->cv.wait(lock);
     }
@@ -105,6 +113,15 @@ class Queue {
 
     return message;
   }
+
+  void BlockOnQuiescence() const {
+    MG_ASSERT(inner_.use_count() > 0);
+    std::unique_lock<std::mutex> lock(inner_->mu);
+
+    while (inner_->calls_to_pop <= inner_->submitted) {
+      inner_->cv.wait(lock);
+    }
+  }
 };
 
 /// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
@@ -129,9 +146,7 @@ class CoordinatorWorker {
 
  public:
   CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
-      : io_(std::move(io)),
+      : io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
-        queue_(std::move(queue)),
-        coordinator_{std::move(io_.ForkLocal()), {}, std::move(coordinator)} {}
 
   CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
   CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
@@ -140,15 +155,12 @@ class CoordinatorWorker {
   ~CoordinatorWorker() = default;
 
   void Run() {
-    while (true) {
+    bool should_continue = true;
+    while (should_continue) {
       Message message = queue_.Pop();
 
-      const bool should_continue = std::visit(
+      should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
-          [this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
+                                   std::move(message));
-
-      if (!should_continue) {
-        return;
-      }
     }
   }
 };

src/coordinator/hybrid_logical_clock.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,17 +23,17 @@ namespace memgraph::coordinator {
 using Time = memgraph::io::Time;
 
 /// Hybrid-logical clock
-struct Hlc {
+struct Hlc final {
-  uint64_t logical_id = 0;
+  uint64_t logical_id{0};
   Time coordinator_wall_clock = Time::min();
 
-  auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
+  auto operator<=>(const Hlc &other) const noexcept { return logical_id <=> other.logical_id; }
 
-  bool operator==(const Hlc &other) const = default;
+  bool operator==(const Hlc &other) const noexcept = default;
-  bool operator<(const Hlc &other) const = default;
+  bool operator<(const Hlc &other) const noexcept = default;
-  bool operator==(const uint64_t other) const { return logical_id == other; }
+  bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
-  bool operator<(const uint64_t other) const { return logical_id < other; }
+  bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
-  bool operator>=(const uint64_t other) const { return logical_id >= other; }
+  bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
 
   friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
     auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);

src/coordinator/shard_map.cpp

@@ -9,6 +9,7 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
+#include <map>
 #include <optional>
 #include <unordered_map>
 #include <vector>
@@ -228,7 +229,7 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept {
   return shard_map_version;
 }
 
-// TODO(antaljanosbenjamin) use a single map for all name id 
+// TODO(antaljanosbenjamin) use a single map for all name id
 // mapping and a single counter to maintain the next id
 std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
   std::unordered_map<uint64_t, std::string> id_to_names;
@@ -248,9 +249,27 @@ std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
 
 Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
 
-std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
+boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
-                                                      std::set<boost::uuids::uuid> initialized) {
+  return boost::uuids::uuid{0,
-  std::vector<ShardToInitialize> ret{};
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            static_cast<unsigned char>(shard_id >> 56U),
+                            static_cast<unsigned char>(shard_id >> 48U),
+                            static_cast<unsigned char>(shard_id >> 40U),
+                            static_cast<unsigned char>(shard_id >> 32U),
+                            static_cast<unsigned char>(shard_id >> 24U),
+                            static_cast<unsigned char>(shard_id >> 16U),
+                            static_cast<unsigned char>(shard_id >> 8U),
+                            static_cast<unsigned char>(shard_id)};
+}
+
+HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
+  HeartbeatResponse ret{};
 
   bool mutated = false;
 
@@ -262,44 +281,74 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
         high_key = next_it->first;
       }
       // TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
-      bool machine_contains_shard = false;
+      bool shard_assigned_to_machine = false;
 
-      for (auto &aas : shard) {
+      for (auto &peer_metadata : shard.peers) {
-        if (initialized.contains(aas.address.unique_id)) {
+        const bool same_machine = peer_metadata.address.last_known_ip == storage_manager.last_known_ip &&
-          machine_contains_shard = true;
+                                  peer_metadata.address.last_known_port == storage_manager.last_known_port;
-          if (aas.status != Status::CONSENSUS_PARTICIPANT) {
+
-            spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
+        if (initialized.contains(peer_metadata.address.unique_id)) {
-            aas.status = Status::CONSENSUS_PARTICIPANT;
+          shard_assigned_to_machine = true;
+
+          if (!same_machine) {
+            // set the last known ip and port to the storage manager that has heartbeated it just now
+            mutated = true;
+            peer_metadata.address.last_known_ip = storage_manager.last_known_ip;
+            peer_metadata.address.last_known_port = storage_manager.last_known_port;
           }
+
+          if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
+            mutated = true;
+            spdlog::info("marking shard as full consensus participant: {}", peer_metadata.address.unique_id);
+            peer_metadata.status = Status::CONSENSUS_PARTICIPANT;
+          }
+        } else if (same_machine && peer_metadata.status == Status::INITIALIZING) {
+          // we are expecting this shard to be initialized (rather than split) on this machine,
+          // so send it an initialization request
+
+          shard_assigned_to_machine = true;
+          spdlog::info("reminding shard manager that they should begin participating in shard");
+
+          ret.shards_to_initialize.push_back(ShardToInitialize{
+              .uuid = peer_metadata.address.unique_id,
+              .label_id = label_id,
+              .min_key = low_key,
+              .max_key = high_key,
+              .schema = schemas[label_id],
+              .config = Config{},
+              .id_to_names = IdToNames(),
+          });
+        } else if (same_machine && peer_metadata.status == Status::PENDING_SPLIT) {
+          // we are expecting this shard to be split, so send it a split request
+
+          ret.shards_to_split.push_back(ShardToSplit{
+              .shard_to_split_uuid = peer_metadata.split_from,
+              .new_right_side_uuid = peer_metadata.address.unique_id,
+              .split_requested_at = shard.version,
+              .label_id = label_id,
+              .split_key = low_key,
+              .schema = schemas[label_id],
+              .config = Config{},
+              .id_to_names = IdToNames(),
+          });
         } else {
-          const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
+          MG_ASSERT(
-                                    aas.address.last_known_port == storage_manager.last_known_port;
+              !same_machine,
-          if (same_machine) {
+              "failed to properly handle a new Status type in the heartbeat management and shard assignment code");
-            machine_contains_shard = true;
-            spdlog::info("reminding shard manager that they should begin participating in shard");
-
-            ret.push_back(ShardToInitialize{
-                .uuid = aas.address.unique_id,
-                .label_id = label_id,
-                .min_key = low_key,
-                .max_key = high_key,
-                .schema = schemas[label_id],
-                .config = Config{},
-                .id_to_names = IdToNames(),
-            });
-          }
         }
       }
 
-      if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
+      if (!shard_assigned_to_machine && shard.peers.size() < label_space.replication_factor) {
         Address address = storage_manager;
 
-        // TODO(tyler) use deterministic UUID so that coordinators don't diverge here
+        // NB: increment version for each new uuid for deterministic creation
-        address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
+        IncrementShardMapVersion();
+
+        address.unique_id = NewShardUuid(shard_map_version.logical_id);
 
         spdlog::info("assigning shard manager to shard");
 
-        ret.push_back(ShardToInitialize{
+        ret.shards_to_initialize.push_back(ShardToInitialize{
             .uuid = address.unique_id,
             .label_id = label_id,
             .min_key = low_key,
@@ -309,12 +358,12 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
             .id_to_names = IdToNames(),
         });
 
-        AddressAndStatus aas = {
+        PeerMetadata peer_metadata = {
             .address = address,
             .status = Status::INITIALIZING,
         };
 
-        shard.emplace_back(aas);
+        shard.peers.emplace_back(peer_metadata);
       }
     }
   }
@@ -322,11 +371,12 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
   if (mutated) {
     IncrementShardMapVersion();
   }
+
   return ret;
 }
 
 bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
-  if (previous_shard_map_version != shard_map_version) {
+  if (previous_shard_map_version != shard_map_version || !label_spaces.contains(label_id)) {
     return false;
   }
 
@@ -335,11 +385,26 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons
 
   MG_ASSERT(!shards_in_map.empty());
   MG_ASSERT(!shards_in_map.contains(key));
-  MG_ASSERT(label_spaces.contains(label_id));
 
-  // Finding the Shard that the new PrimaryKey should map to.
+  // Finding the ShardMetadata that the new PrimaryKey should map to.
-  auto prev = std::prev(shards_in_map.upper_bound(key));
+  ShardMetadata duplicated_shard = GetShardForKey(label_id, key);
-  Shard duplicated_shard = prev->second;
+
+  std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
+
+  for (auto &peer_metadata : duplicated_shard.peers) {
+    peer_metadata.status = Status::PENDING_SPLIT;
+    peer_metadata.split_from = peer_metadata.address.unique_id;
+
+    // NB: increment version for each new uuid for deterministic creation
+    IncrementShardMapVersion();
+
+    auto new_uuid = NewShardUuid(shard_map_version.logical_id);
+
+    // store new uuid for the right side of each shard
+    split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
+
+    peer_metadata.address.unique_id = new_uuid;
+  }
 
   // Apply the split
   shards_in_map[key] = duplicated_shard;
@@ -360,7 +425,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
   labels.emplace(std::move(label_name), label_id);
 
   PrimaryKey initial_key = SchemaToMinKey(schema);
-  Shard empty_shard = {};
+  ShardMetadata empty_shard = {};
 
   Shards shards = {
       {initial_key, empty_shard},
@@ -383,6 +448,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
 void ShardMap::AddServer(Address server_address) {
   // Find a random place for the server to plug in
 }
+
 std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
   if (const auto it = labels.find(label); it != labels.end()) {
     return it->second;
@@ -455,7 +521,7 @@ Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey
   return shards;
 }
 
-Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
+ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
   MG_ASSERT(labels.contains(label_name));
 
   LabelId label_id = labels.at(label_name);
@@ -468,7 +534,7 @@ Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &ke
   return std::prev(label_space.shards.upper_bound(key))->second;
 }
 
-Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
+ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
   MG_ASSERT(label_spaces.contains(label_id));
 
   const auto &label_space = label_spaces.at(label_id);
@@ -532,13 +598,13 @@ EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new
 bool ShardMap::ClusterInitialized() const {
   for (const auto &[label_id, label_space] : label_spaces) {
     for (const auto &[low_key, shard] : label_space.shards) {
-      if (shard.size() < label_space.replication_factor) {
+      if (shard.peers.size() < label_space.replication_factor) {
         spdlog::info("label_space below desired replication factor");
         return false;
       }
 
-      for (const auto &aas : shard) {
+      for (const auto &peer_metadata : shard.peers) {
-        if (aas.status != Status::CONSENSUS_PARTICIPANT) {
+        if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
           spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
           return false;
         }

src/coordinator/shard_map.hpp

@@ -47,37 +47,71 @@ using memgraph::storage::v3::SchemaProperty;
 enum class Status : uint8_t {
   CONSENSUS_PARTICIPANT,
   INITIALIZING,
+  PENDING_SPLIT,
   // TODO(tyler) this will possibly have more states,
   // depending on the reconfiguration protocol that we
   // implement.
 };
 
-struct AddressAndStatus {
+struct PeerMetadata {
   memgraph::io::Address address;
   Status status;
+  boost::uuids::uuid split_from;
 
-  friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
+  friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
 
-  friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
+  friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
-    in << "AddressAndStatus { address: ";
+    in << "PeerMetadata { address: ";
-    in << address_and_status.address;
+    in << peer_metadata.address;
-    if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
+
-      in << ", status: CONSENSUS_PARTICIPANT }";
+    if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
+      in << ", status: CONSENSUS_PARTICIPANT";
+    } else if (peer_metadata.status == Status::INITIALIZING) {
+      in << ", status: INITIALIZING";
+    } else if (peer_metadata.status == Status::PENDING_SPLIT) {
+      in << ", status: PENDING_SPLIT";
     } else {
-      in << ", status: INITIALIZING }";
+      MG_ASSERT(false, "failed to update the operator<< implementation for Status");
     }
 
+    in << ", split_from: " << peer_metadata.split_from << " }";
+
     return in;
   }
 
-  friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
+  friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
-    return lhs.address == rhs.address;
-  }
 };
 
 using PrimaryKey = std::vector<PropertyValue>;
-using Shard = std::vector<AddressAndStatus>;
+
-using Shards = std::map<PrimaryKey, Shard>;
+struct ShardMetadata {
+  std::vector<PeerMetadata> peers;
+  uint64_t version;
+
+  friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
+    using utils::print_helpers::operator<<;
+
+    in << "ShardMetadata { peers: ";
+    in << shard.peers;
+    in << " version: ";
+    in << shard.version;
+    in << " }";
+
+    return in;
+  }
+
+  friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
+
+  friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
+    if (lhs.peers != rhs.peers) {
+      return lhs.peers < rhs.peers;
+    }
+
+    return lhs.version < rhs.version;
+  }
+};
+
+using Shards = std::map<PrimaryKey, ShardMetadata>;
 using LabelName = std::string;
 using PropertyName = std::string;
 using EdgeTypeName = std::string;
@@ -94,12 +128,33 @@ struct ShardToInitialize {
   std::unordered_map<uint64_t, std::string> id_to_names;
 };
 
+struct ShardToSplit {
+  boost::uuids::uuid shard_to_split_uuid;
+  boost::uuids::uuid new_right_side_uuid;
+  Hlc split_requested_at;
+  LabelId label_id;
+  PrimaryKey split_key;
+  std::vector<SchemaProperty> schema;
+  Config config;
+  std::unordered_map<uint64_t, std::string> id_to_names;
+};
+
+struct HeartbeatRequest {
+  Address from_storage_manager;
+  std::set<boost::uuids::uuid> initialized_rsms;
+};
+
+struct HeartbeatResponse {
+  std::vector<ShardToInitialize> shards_to_initialize;
+  std::vector<ShardToSplit> shards_to_split;
+};
+
 PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
 
 struct LabelSpace {
   std::vector<SchemaProperty> schema;
   // Maps between the smallest primary key stored in the shard and the shard
-  std::map<PrimaryKey, Shard> shards;
+  std::map<PrimaryKey, ShardMetadata> shards;
   size_t replication_factor;
 
   friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
@@ -141,7 +196,7 @@ struct ShardMap {
   std::unordered_map<uint64_t, std::string> IdToNames();
 
   // Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
-  std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
+  HeartbeatResponse AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
 
   bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
 
@@ -160,9 +215,9 @@ struct ShardMap {
 
   Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
 
-  Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
+  ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
 
-  Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
+  ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
 
   PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
 

src/expr/CMakeLists.txt

@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
-target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)
+target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)

src/expr/ast/cypher_main_visitor.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ constexpr char kId[] = "ID";
 
 namespace MG_INJECTED_NAMESPACE_NAME {
 namespace detail {
-using antlropencypher::MemgraphCypher;
+using antlropencypher::v2::MemgraphCypher;
 
 template <typename TVisitor>
 std::optional<std::pair<Expression *, size_t>> VisitMemoryLimit(MemgraphCypher::MemoryLimitContext *memory_limit_ctx,
@@ -211,13 +211,13 @@ inline std::string_view ToString(const PulsarConfigKey key) {
 }
 }  // namespace detail
 
-using antlropencypher::MemgraphCypher;
+using antlropencypher::v2::MemgraphCypher;
 
 struct ParsingContext {
   bool is_query_cached = false;
 };
 
-class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
+class CypherMainVisitor : public antlropencypher::v2::MemgraphCypherBaseVisitor {
  public:
   explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {}
 

src/expr/interpret/eval.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,6 +24,7 @@
 #include "expr/exceptions.hpp"
 #include "expr/interpret/frame.hpp"
 #include "expr/semantic/symbol_table.hpp"
+#include "functions/awesome_memgraph_functions.hpp"
 #include "utils/exceptions.hpp"
 
 namespace memgraph::expr {
@@ -35,8 +36,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
           typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
 class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
  public:
-  ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
+  ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
-                      DbAccessor *dba, StorageView view)
+                      StorageView view)
       : frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
 
   using ExpressionVisitor<TypedValue>::Visit;
@@ -46,12 +47,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   TypedValue Visit(NamedExpression &named_expression) override {
     const auto &symbol = symbol_table_->at(named_expression);
     auto value = named_expression.expression_->Accept(*this);
-    frame_->at(symbol) = value;
+    frame_->At(symbol) = value;
     return value;
   }
 
   TypedValue Visit(Identifier &ident) override {
-    return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
+    return TypedValue(frame_->At(symbol_table_->at(ident)), ctx_->memory);
   }
 
   // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
@@ -100,6 +101,28 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
 #undef BINARY_OPERATOR_VISITOR
 #undef UNARY_OPERATOR_VISITOR
 
+  void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
+    switch (shard_error) {
+      case Error::DELETED_OBJECT:
+        throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
+      case Error::NONEXISTENT_OBJECT:
+        throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
+      case Error::SERIALIZATION_ERROR:
+      case Error::VERTEX_HAS_EDGES:
+      case Error::PROPERTIES_DISABLED:
+      case Error::VERTEX_ALREADY_INSERTED:
+      case Error::OBJECT_NOT_FOUND:
+        throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
+      case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
+      case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
+      case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
+      case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
+      case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
+      case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
+        throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
+    }
+  }
+
   TypedValue Visit(AndOperator &op) override {
     auto value1 = op.expression1_->Accept(*this);
     if (value1.IsBool() && !value1.ValueBool()) {
@@ -396,17 +419,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
     }
     if (has_label.HasError()) {
-      switch (has_label.GetError()) {
+      HandleObjectAccessError(has_label.GetError().code, "labels");
-        case Error::DELETED_OBJECT:
-          throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
-        case Error::NONEXISTENT_OBJECT:
-          throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
-        case Error::SERIALIZATION_ERROR:
-        case Error::VERTEX_HAS_EDGES:
-        case Error::PROPERTIES_DISABLED:
-        case Error::VERTEX_ALREADY_INSERTED:
-          throw ExpressionRuntimeException("Unexpected error when accessing labels.");
-      }
     }
     return *has_label;
   }
@@ -415,8 +428,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
             typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
   TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
     auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
-    auto has_label = vertex.HasLabel(label);
+    return vertex.HasLabel(label);
-    return !has_label;
   }
 
   TypedValue Visit(LabelsTest &labels_test) override {
@@ -458,7 +470,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   }
 
   TypedValue Visit(Aggregation &aggregation) override {
-    return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
+    return TypedValue(frame_->At(symbol_table_->at(aggregation)), ctx_->memory);
   }
 
   TypedValue Visit(Coalesce &coalesce) override {
@@ -479,7 +491,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   }
 
   TypedValue Visit(Function &function) override {
-    FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
+    functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
     // Stack allocate evaluated arguments when there's a small number of them.
     if (function.arguments_.size() <= 8) {
       TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
@@ -516,8 +528,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
     auto accumulator = reduce.initializer_->Accept(*this);
     for (const auto &element : list) {
-      frame_->at(accumulator_symbol) = accumulator;
+      frame_->At(accumulator_symbol) = accumulator;
-      frame_->at(element_symbol) = element;
+      frame_->At(element_symbol) = element;
       accumulator = reduce.expression_->Accept(*this);
     }
     return accumulator;
@@ -539,7 +551,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       if (element.IsNull()) {
         result.emplace_back();
       } else {
-        frame_->at(element_symbol) = element;
+        frame_->At(element_symbol) = element;
         result.emplace_back(extract.expression_->Accept(*this));
       }
     }
@@ -559,7 +571,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     bool has_null_elements = false;
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = all.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
@@ -596,7 +608,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     bool has_value = false;
     bool predicate_satisfied = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = single.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
@@ -633,7 +645,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &symbol = symbol_table_->at(*any.identifier_);
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = any.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
@@ -665,7 +677,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &symbol = symbol_table_->at(*none.identifier_);
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = none.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
@@ -744,17 +756,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
     }
     if (maybe_prop.HasError()) {
-      switch (maybe_prop.GetError()) {
+      HandleObjectAccessError(maybe_prop.GetError().code, "property");
-        case Error::DELETED_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
-        case Error::NONEXISTENT_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
-        case Error::SERIALIZATION_ERROR:
-        case Error::VERTEX_HAS_EDGES:
-        case Error::PROPERTIES_DISABLED:
-        case Error::VERTEX_ALREADY_INSERTED:
-          throw ExpressionRuntimeException("Unexpected error when getting a property.");
-      }
     }
     return conv_(*maybe_prop, ctx_->memory);
   }
@@ -773,24 +775,14 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
     }
     if (maybe_prop.HasError()) {
-      switch (maybe_prop.GetError()) {
+      HandleObjectAccessError(maybe_prop.GetError().code, "property");
-        case Error::DELETED_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
-        case Error::NONEXISTENT_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
-        case Error::SERIALIZATION_ERROR:
-        case Error::VERTEX_HAS_EDGES:
-        case Error::PROPERTIES_DISABLED:
-        case Error::VERTEX_ALREADY_INSERTED:
-          throw ExpressionRuntimeException("Unexpected error when getting a property.");
-      }
     }
     return conv_(*maybe_prop, ctx_->memory);
   }
 
   LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
 
-  Frame<TypedValue> *frame_;
+  Frame *frame_;
   const SymbolTable *symbol_table_;
   const EvaluationContext *ctx_;
   DbAccessor *dba_;

src/expr/interpret/frame.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,26 +20,43 @@
 
 namespace memgraph::expr {
 
-template <typename TypedValue>
 class Frame {
  public:
   /// Create a Frame of given size backed by a utils::NewDeleteResource()
-  explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
+  explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
 
-  Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
+  Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
 
   TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
   const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
 
-  TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
+  TypedValue &At(const Symbol &symbol) { return elems_.at(symbol.position()); }
-  const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
+  const TypedValue &At(const Symbol &symbol) const { return elems_.at(symbol.position()); }
 
-  auto &elems() { return elems_; }
+  uint64_t Id() const { return id_; }
+  void SetId(const uint64_t id) { id_ = id; }
+
+  const utils::pmr::vector<TypedValue> &Elems() const { return elems_; }
 
   utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
 
  private:
+  uint64_t id_{0U};
   utils::pmr::vector<TypedValue> elems_;
 };
 
+class FrameWithValidity final : public Frame {
+ public:
+  explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
+
+  FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
+
+  bool IsValid() const noexcept { return is_valid_; }
+  void MakeValid() noexcept { is_valid_ = true; }
+  void MakeInvalid() noexcept { is_valid_ = false; }
+
+ private:
+  bool is_valid_;
+};
+
 }  // namespace memgraph::expr

src/functions/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(mg-functions INTERFACE)

src/glue/v2/communication.cpp

@@ -15,13 +15,13 @@
 #include <string>
 #include <vector>
 
+#include "common/errors.hpp"
 #include "coordinator/shard_map.hpp"
 #include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
 #include "query/v2/requests.hpp"
-#include "query/v2/shard_request_manager.hpp"
 #include "storage/v3/edge_accessor.hpp"
 #include "storage/v3/id_types.hpp"
-#include "storage/v3/result.hpp"
 #include "storage/v3/shard.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "storage/v3/view.hpp"
@@ -71,106 +71,101 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
   }
 }
 
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
+communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
-    const query::v2::accessors::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                         const query::v2::RequestRouterInterface *request_router,
-    storage::v3::View /*view*/) {
+                                         storage::v3::View /*view*/) {
   auto id = communication::bolt::Id::FromUint(0);
 
   auto labels = vertex.Labels();
   std::vector<std::string> new_labels;
   new_labels.reserve(labels.size());
   for (const auto &label : labels) {
-    new_labels.push_back(shard_request_manager->LabelToName(label.id));
+    new_labels.push_back(request_router->LabelToName(label.id));
   }
 
   auto properties = vertex.Properties();
   std::map<std::string, Value> new_properties;
   for (const auto &[prop, property_value] : properties) {
-    new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
+    new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
   }
   return communication::bolt::Vertex{id, new_labels, new_properties};
 }
 
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
+communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
-    const query::v2::accessors::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                     const query::v2::RequestRouterInterface *request_router,
-    storage::v3::View /*view*/) {
+                                     storage::v3::View /*view*/) {
   // TODO(jbajic) Fix bolt communication
   auto id = communication::bolt::Id::FromUint(0);
   auto from = communication::bolt::Id::FromUint(0);
   auto to = communication::bolt::Id::FromUint(0);
-  const auto &type = shard_request_manager->EdgeTypeToName(edge.EdgeType());
+  const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
 
   auto properties = edge.Properties();
   std::map<std::string, Value> new_properties;
   for (const auto &[prop, property_value] : properties) {
-    new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
+    new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
   }
   return communication::bolt::Edge{id, from, to, type, new_properties};
 }
 
-storage::v3::Result<communication::bolt::Path> ToBoltPath(
+communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
-    const query::v2::accessors::Path & /*edge*/, const msgs::ShardRequestManagerInterface * /*shard_request_manager*/,
+                                     const query::v2::RequestRouterInterface * /*request_router*/,
-    storage::v3::View /*view*/) {
+                                     storage::v3::View /*view*/) {
   // TODO(jbajic) Fix bolt communication
-  return {storage::v3::Error::DELETED_OBJECT};
+  MG_ASSERT(false, "Path is unimplemented!");
+  return {};
 }
 
-storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value,
+Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
-                                       const msgs::ShardRequestManagerInterface *shard_request_manager,
+                  storage::v3::View view) {
-                                       storage::v3::View view) {
   switch (value.type()) {
     case query::v2::TypedValue::Type::Null:
-      return Value();
+      return {};
     case query::v2::TypedValue::Type::Bool:
-      return Value(value.ValueBool());
+      return {value.ValueBool()};
     case query::v2::TypedValue::Type::Int:
-      return Value(value.ValueInt());
+      return {value.ValueInt()};
     case query::v2::TypedValue::Type::Double:
-      return Value(value.ValueDouble());
+      return {value.ValueDouble()};
     case query::v2::TypedValue::Type::String:
-      return Value(std::string(value.ValueString()));
+      return {std::string(value.ValueString())};
     case query::v2::TypedValue::Type::List: {
       std::vector<Value> values;
       values.reserve(value.ValueList().size());
       for (const auto &v : value.ValueList()) {
-        auto maybe_value = ToBoltValue(v, shard_request_manager, view);
+        auto value = ToBoltValue(v, request_router, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
+        values.emplace_back(std::move(value));
-        values.emplace_back(std::move(*maybe_value));
       }
-      return Value(std::move(values));
+      return {std::move(values)};
     }
     case query::v2::TypedValue::Type::Map: {
       std::map<std::string, Value> map;
       for (const auto &kv : value.ValueMap()) {
-        auto maybe_value = ToBoltValue(kv.second, shard_request_manager, view);
+        auto value = ToBoltValue(kv.second, request_router, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
+        map.emplace(kv.first, std::move(value));
-        map.emplace(kv.first, std::move(*maybe_value));
       }
-      return Value(std::move(map));
+      return {std::move(map)};
     }
     case query::v2::TypedValue::Type::Vertex: {
-      auto maybe_vertex = ToBoltVertex(value.ValueVertex(), shard_request_manager, view);
+      auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
-      if (maybe_vertex.HasError()) return maybe_vertex.GetError();
+      return {std::move(vertex)};
-      return Value(std::move(*maybe_vertex));
     }
     case query::v2::TypedValue::Type::Edge: {
-      auto maybe_edge = ToBoltEdge(value.ValueEdge(), shard_request_manager, view);
+      auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
-      if (maybe_edge.HasError()) return maybe_edge.GetError();
+      return {std::move(edge)};
-      return Value(std::move(*maybe_edge));
     }
     case query::v2::TypedValue::Type::Path: {
-      auto maybe_path = ToBoltPath(value.ValuePath(), shard_request_manager, view);
+      auto path = ToBoltPath(value.ValuePath(), request_router, view);
-      if (maybe_path.HasError()) return maybe_path.GetError();
+      return {std::move(path)};
-      return Value(std::move(*maybe_path));
     }
     case query::v2::TypedValue::Type::Date:
-      return Value(value.ValueDate());
+      return {value.ValueDate()};
     case query::v2::TypedValue::Type::LocalTime:
-      return Value(value.ValueLocalTime());
+      return {value.ValueLocalTime()};
     case query::v2::TypedValue::Type::LocalDateTime:
-      return Value(value.ValueLocalDateTime());
+      return {value.ValueLocalDateTime()};
     case query::v2::TypedValue::Type::Duration:
-      return Value(value.ValueDuration());
+      return {value.ValueDuration()};
   }
 }
 

src/glue/v2/communication.hpp

@@ -15,11 +15,12 @@
 #include "communication/bolt/v1/value.hpp"
 #include "coordinator/shard_map.hpp"
 #include "query/v2/bindings/typed_value.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
 #include "storage/v3/shard.hpp"
 #include "storage/v3/view.hpp"
+#include "utils/result.hpp"
 
 namespace memgraph::storage::v3 {
 class EdgeAccessor;
@@ -31,40 +32,37 @@ namespace memgraph::glue::v2 {
 
 /// @param storage::v3::VertexAccessor for converting to
 ///        communication::bolt::Vertex.
-/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting label and property names.
+/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
 /// @param storage::v3::View for deciding which vertex attributes are visible.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
+communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
-    const storage::v3::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                         const query::v2::RequestRouterInterface *request_router,
-    storage::v3::View view);
+                                         storage::v3::View view);
 
 /// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
-/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting edge type and property names.
+/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
 /// @param storage::v3::View for deciding which edge attributes are visible.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
+communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
-    const storage::v3::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                     const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
-    storage::v3::View view);
 
 /// @param query::v2::Path for converting to communication::bolt::Path.
-/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
+/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
 /// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Path> ToBoltPath(
+communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
-    const query::v2::accessors::Path &path, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                     const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
-    storage::v3::View view);
 
 /// @param query::v2::TypedValue for converting to communication::bolt::Value.
-/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
+/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
 /// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Value> ToBoltValue(
+communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
-    const query::v2::TypedValue &value, const msgs::ShardRequestManagerInterface *shard_request_manager,
+                                       const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
-    storage::v3::View view);
 
 query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
 
@@ -74,8 +72,7 @@ storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &val
 
 communication::bolt::Value ToBoltValue(msgs::Value value);
 
-communication::bolt::Value ToBoltValue(msgs::Value value,
+communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
-                                       const msgs::ShardRequestManagerInterface *shard_request_manager,
                                        storage::v3::View view);
 
 }  // namespace memgraph::glue::v2

src/io/address.hpp

@@ -20,6 +20,8 @@
 #include <boost/uuid/uuid_generators.hpp>
 #include <boost/uuid/uuid_io.hpp>
 
+#include "utils/logging.hpp"
+
 namespace memgraph::io {
 
 struct PartialAddress {
@@ -58,18 +60,39 @@ struct Address {
   uint16_t last_known_port;
 
   static Address TestAddress(uint16_t port) {
+    MG_ASSERT(port <= 255);
+
     return Address{
-        .unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
+        .unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
         .last_known_port = port,
     };
   }
 
+  // NB: don't use this in test code because it is non-deterministic
   static Address UniqueLocalAddress() {
     return Address{
         .unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
     };
   }
 
+  /// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
+  Address ForkLocalCoordinator() {
+    return Address{
+        .unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+        .last_known_ip = last_known_ip,
+        .last_known_port = last_known_port,
+    };
+  }
+
+  /// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
+  Address ForkLocalShardManager() {
+    return Address{
+        .unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+        .last_known_ip = last_known_ip,
+        .last_known_port = last_known_port,
+    };
+  }
+
   /// Returns a new ID with the same IP and port but a unique UUID.
   Address ForkUniqueAddress() {
     return Address{

src/io/future.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -35,10 +35,13 @@ class Shared {
   std::optional<T> item_;
   bool consumed_ = false;
   bool waiting_ = false;
-  std::function<bool()> simulator_notifier_ = nullptr;
+  bool filled_ = false;
+  std::function<bool()> wait_notifier_ = nullptr;
+  std::function<void()> fill_notifier_ = nullptr;
 
  public:
-  explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
+  explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
+      : wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
   Shared() = default;
   Shared(Shared &&) = delete;
   Shared &operator=(Shared &&) = delete;
@@ -64,8 +67,7 @@ class Shared {
     waiting_ = true;
 
     while (!item_) {
-      bool simulator_progressed = false;
+      if (wait_notifier_) [[unlikely]] {
-      if (simulator_notifier_) [[unlikely]] {
         // We can't hold our own lock while notifying
         // the simulator because notifying the simulator
         // involves acquiring the simulator's mutex
@@ -77,7 +79,7 @@ class Shared {
         // so we have to get out of its way to avoid
         // a cyclical deadlock.
         lock.unlock();
-        simulator_progressed = std::invoke(simulator_notifier_);
+        std::invoke(wait_notifier_);
         lock.lock();
         if (item_) {
           // item may have been filled while we
@@ -85,8 +87,7 @@ class Shared {
           // the simulator of our waiting_ status.
           break;
         }
-      }
+      } else {
-      if (!simulator_progressed) [[likely]] {
         cv_.wait(lock);
       }
       MG_ASSERT(!consumed_, "Future consumed twice!");
@@ -117,11 +118,19 @@ class Shared {
       std::unique_lock<std::mutex> lock(mu_);
 
       MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
-      MG_ASSERT(!item_, "Promise filled twice!");
+      MG_ASSERT(!filled_, "Promise filled twice!");
 
-      item_ = item;
+      item_ = std::move(item);
+      filled_ = true;
     }  // lock released before condition variable notification
 
+    if (fill_notifier_) {
+      spdlog::trace("calling fill notifier");
+      std::invoke(fill_notifier_);
+    } else {
+      spdlog::trace("not calling fill notifier");
+    }
+
     cv_.notify_all();
   }
 
@@ -226,7 +235,7 @@ class Promise {
   // Fill the expected item into the Future.
   void Fill(T item) {
     MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
-    shared_->Fill(item);
+    shared_->Fill(std::move(item));
     filled_or_moved_ = true;
   }
 
@@ -253,8 +262,9 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
 }
 
 template <typename T>
-std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
+std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
-  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
+                                                                    std::function<void()> fill_notifier) {
+  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
 
   Future<T> future = Future<T>(shared);
   Promise<T> promise = Promise<T>(shared);

src/io/local_transport/local_system.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/io/local_transport/local_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,10 +30,11 @@ class LocalTransport {
   explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
       : local_transport_handle_(std::move(local_transport_handle)) {}
 
-  template <Message RequestT, Message ResponseT>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
-    return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address,
+                                    std::function<void()> fill_notifier, Duration timeout) {
-                                                                                std::move(request), timeout);
+    return local_transport_handle_->template SubmitRequest<ResponseT, RequestT>(
+        to_address, from_address, std::move(request), timeout, fill_notifier);
   }
 
   template <Message... Ms>
@@ -42,8 +43,8 @@ class LocalTransport {
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
-    return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
+    return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
   }
 
   Time Now() const { return local_transport_handle_->Now(); }

src/io/local_transport/local_transport_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -104,10 +104,10 @@ class LocalTransportHandle {
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
     auto type_info = TypeInfoFor(message);
 
-    std::any message_any(std::forward<M>(message));
+    std::any message_any(std::move(message));
     OpaqueMessage opaque_message{.to_address = to_address,
                                  .from_address = from_address,
                                  .request_id = request_id,
@@ -138,10 +138,14 @@ class LocalTransportHandle {
     cv_.notify_all();
   }
 
-  template <Message RequestT, Message ResponseT>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
+  ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
-                                          Duration timeout) {
+                                          Duration timeout, std::function<void()> fill_notifier) {
-    auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
+        // set null notifier for when the Future::Wait is called
+        nullptr,
+        // set notifier for when Promise::Fill is called
+        std::move(fill_notifier));
 
     const bool port_matches = to_address.last_known_port == from_address.last_known_port;
     const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
@@ -164,7 +168,7 @@ class LocalTransportHandle {
       promises_.emplace(std::move(promise_key), std::move(dop));
     }  // lock dropped
 
-    Send(to_address, from_address, request_id, std::forward<RequestT>(request));
+    Send<RequestT>(to_address, from_address, request_id, std::move(request));
 
     return std::move(future);
   }

src/io/message_conversion.hpp

@@ -13,6 +13,7 @@
 
 #include <boost/core/demangle.hpp>
 
+#include "io/time.hpp"
 #include "io/transport.hpp"
 #include "utils/type_info_ref.hpp"
 
@@ -38,6 +39,7 @@ struct OpaqueMessage {
   uint64_t request_id;
   std::any message;
   utils::TypeInfoRef type_info;
+  Time deliverable_at;
 
   /// Recursively tries to match a specific type from the outer
   /// variant's parameter pack against the type of the std::any,

src/io/message_histogram_collector.hpp

@@ -13,6 +13,7 @@
 
 #include <chrono>
 #include <cmath>
+#include <compare>
 #include <unordered_map>
 
 #include <boost/core/demangle.hpp>
@@ -39,6 +40,8 @@ struct LatencyHistogramSummary {
   Duration p100;
   Duration sum;
 
+  friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
+
   friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
     in << "{ \"count\": " << histo.count;
     in << ", \"p0\": " << histo.p0.count();
@@ -80,6 +83,8 @@ struct LatencyHistogramSummaries {
     return output;
   }
 
+  friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
+
   friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
     using memgraph::utils::print_helpers::operator<<;
     in << histo.latencies;

src/io/notifier.hpp

@@ -0,0 +1,93 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <optional>
+#include <vector>
+
+namespace memgraph::io {
+
+class ReadinessToken {
+  size_t id_;
+
+ public:
+  explicit ReadinessToken(size_t id) : id_(id) {}
+  size_t GetId() const { return id_; }
+};
+
+class Inner {
+  std::condition_variable cv_;
+  std::mutex mu_;
+  std::vector<ReadinessToken> ready_;
+  std::optional<std::function<bool()>> tick_simulator_;
+
+ public:
+  void Notify(ReadinessToken readiness_token) {
+    {
+      std::unique_lock<std::mutex> lock(mu_);
+      ready_.emplace_back(readiness_token);
+    }  // mutex dropped
+
+    cv_.notify_all();
+  }
+
+  ReadinessToken Await() {
+    std::unique_lock<std::mutex> lock(mu_);
+
+    while (ready_.empty()) {
+      if (tick_simulator_) [[unlikely]] {
+        // This avoids a deadlock in a similar way that
+        // Future::Wait will release its mutex while
+        // interacting with the simulator, due to
+        // the fact that the simulator may cause
+        // notifications that we are interested in.
+        lock.unlock();
+        std::invoke(tick_simulator_.value());
+        lock.lock();
+      } else {
+        cv_.wait(lock);
+      }
+    }
+
+    ReadinessToken ret = ready_.back();
+    ready_.pop_back();
+    return ret;
+  }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
+    std::unique_lock<std::mutex> lock(mu_);
+    tick_simulator_ = tick_simulator;
+  }
+};
+
+class Notifier {
+  std::shared_ptr<Inner> inner_;
+
+ public:
+  Notifier() : inner_(std::make_shared<Inner>()) {}
+  Notifier(const Notifier &) = default;
+  Notifier &operator=(const Notifier &) = default;
+  Notifier(Notifier &&old) = default;
+  Notifier &operator=(Notifier &&old) = default;
+  ~Notifier() = default;
+
+  void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
+
+  ReadinessToken Await() const { return inner_->Await(); }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
+};
+
+}  // namespace memgraph::io

src/io/rsm/raft.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,7 +19,7 @@
 #include <map>
 #include <set>
 #include <thread>
-#include <unordered_map>
+#include <type_traits>
 #include <vector>
 
 #include <boost/core/demangle.hpp>
@@ -92,33 +92,43 @@ struct ReadResponse {
 };
 
 template <class... ReadReturn>
-utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &read_response) {
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
-  return TypeInfoForVariant(read_response.read_return);
+  return TypeInfoForVariant(response.read_return);
 }
 
 template <class ReadReturn>
-utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* read_response */) {
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
   return typeid(ReadReturn);
 }
 
+template <class ReadOperation>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
+  return typeid(ReadOperation);
+}
+
+template <class... ReadOperations>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
+  return TypeInfoForVariant(request.operation);
+}
+
 template <class... WriteReturn>
-utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &write_response) {
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
-  return TypeInfoForVariant(write_response.write_return);
+  return TypeInfoForVariant(response.write_return);
 }
 
 template <class WriteReturn>
-utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* write_response */) {
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
   return typeid(WriteReturn);
 }
 
 template <class WriteOperation>
-utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* write_request */) {
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
   return typeid(WriteOperation);
 }
 
 template <class... WriteOperations>
-utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &write_request) {
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
-  return TypeInfoForVariant(write_request.operation);
+  return TypeInfoForVariant(request.operation);
 }
 
 /// AppendRequest is a raft-level message that the Leader
@@ -182,7 +192,7 @@ struct PendingClientRequest {
 
 struct Leader {
   std::map<Address, FollowerTracker> followers;
-  std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
+  std::map<LogIndex, PendingClientRequest> pending_client_requests;
   Time last_broadcast = Time::min();
 
   std::string static ToString() { return "\tLeader   \t"; }
@@ -237,7 +247,7 @@ to a CAS operation.
 template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
           typename ReadResponseValue>
 concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
-  { state.Read(r) } -> std::same_as<ReadResponseValue>;
+  { state.Read(std::move(r)) } -> std::same_as<ReadResponseValue>;
   { state.Apply(w) } -> std::same_as<WriteResponseValue>;
 };
 
@@ -312,19 +322,12 @@ class Raft {
   }
 
   void Run() {
-    // debug - gvolfing
-    uint64_t tick_count = 0;
-
     while (!io_.ShouldShutDown()) {
       const auto now = io_.Now();
       if (now >= next_cron_) {
         next_cron_ = Cron();
       }
 
-      // debug - gvolfing
-      spdlog::info("Raft::Run() awakened on thread {}, Cron tick: {}", std::this_thread::get_id(), tick_count);
-      tick_count++;
-
       const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
 
       auto request_result =
@@ -400,7 +403,7 @@ class Raft {
         const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
         leader.pending_client_requests.erase(apply_index);
 
-        const WriteResponse<WriteResponseValue> resp{
+        WriteResponse<WriteResponseValue> resp{
             .success = true,
             .write_return = std::move(write_return),
             .raft_index = apply_index,
@@ -552,7 +555,7 @@ class Raft {
       for (const auto &peer : peers_) {
         // request_id not necessary to set because it's not a Future-backed Request.
         static constexpr auto request_id = 0;
-        io_.template Send<VoteRequest>(peer, request_id, request);
+        io_.template Send(peer, request_id, VoteRequest{request});
         outstanding_votes.insert(peer);
       }
 
@@ -622,13 +625,12 @@ class Raft {
       MG_ASSERT(std::max(req.term, state_.term) == req.term);
     }
 
-    const VoteResponse res{
+    io_.Send(from_address, request_id,
-        .term = std::max(req.term, state_.term),
+             VoteResponse{
-        .committed_log_size = state_.committed_log_size,
+                 .term = std::max(req.term, state_.term),
-        .vote_granted = new_leader,
+                 .committed_log_size = state_.committed_log_size,
-    };
+                 .vote_granted = new_leader,
-
+             });
-    io_.Send(from_address, request_id, res);
 
     if (new_leader) {
       // become a follower
@@ -690,7 +692,7 @@ class Raft {
 
       return Leader{
           .followers = std::move(followers),
-          .pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
+          .pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
       };
     }
 
@@ -716,6 +718,10 @@ class Raft {
         .log_size = state_.log.size(),
     };
 
+    static_assert(std::is_trivially_copyable_v<AppendResponse>,
+                  "This function copies this message, therefore it is important to be trivially copyable. Otherwise it "
+                  "should be moved");
+
     if constexpr (std::is_same<ALL, Leader>()) {
       MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
     }
@@ -734,7 +740,7 @@ class Raft {
       // become follower of this leader, reply with our log status
       state_.term = req.term;
 
-      io_.Send(from_address, request_id, res);
+      io_.Send(from_address, request_id, AppendResponse{res});
 
       Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
       return Follower{
@@ -745,7 +751,7 @@ class Raft {
 
     if (req.term < state_.term) {
       // nack this request from an old leader
-      io_.Send(from_address, request_id, res);
+      io_.Send(from_address, request_id, AppendResponse{res});
 
       return std::nullopt;
     }
@@ -806,7 +812,7 @@ class Raft {
 
     Log("returning log_size of ", res.log_size);
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, AppendResponse{res});
 
     return std::nullopt;
   }
@@ -854,18 +860,20 @@ class Raft {
   // Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
   std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
                              Address from_address) {
-    Log("handling ReadOperation");
+    auto type_info = TypeInfoFor(req);
-    ReadOperation read_operation = req.operation;
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    Log("handling ReadOperation<" + demangled_name + ">");
+    ReadOperation &read_operation = req.operation;
 
-    ReadResponseValue read_return = replicated_state_.Read(read_operation);
+    ReadResponseValue read_return = replicated_state_.Read(std::move(read_operation));
 
-    const ReadResponse<ReadResponseValue> resp{
+    ReadResponse<ReadResponseValue> resp{
         .success = true,
         .read_return = std::move(read_return),
         .retry_leader = std::nullopt,
     };
 
-    io_.Send(from_address, request_id, resp);
+    io_.Send(from_address, request_id, std::move(resp));
 
     return std::nullopt;
   }
@@ -874,11 +882,11 @@ class Raft {
   std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
                              RequestId request_id, Address from_address) {
     Log("received ReadOperation - not redirecting because no Leader is known");
-    const ReadResponse<ReadResponseValue> res{
+    ReadResponse<ReadResponseValue> res{
         .success = false,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     Cron();
 
@@ -890,12 +898,12 @@ class Raft {
                              Address from_address) {
     Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
 
-    const ReadResponse<ReadResponseValue> res{
+    ReadResponse<ReadResponseValue> res{
         .success = false,
         .retry_leader = follower.leader_address,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     return std::nullopt;
   }
@@ -909,12 +917,12 @@ class Raft {
                              Address from_address) {
     Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
 
-    const WriteResponse<WriteResponseValue> res{
+    WriteResponse<WriteResponseValue> res{
         .success = false,
         .retry_leader = follower.leader_address,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     return std::nullopt;
   }
@@ -923,11 +931,11 @@ class Raft {
                              RequestId request_id, Address from_address) {
     Log("received WriteRequest - not redirecting because no Leader is known");
 
-    const WriteResponse<WriteResponseValue> res{
+    WriteResponse<WriteResponseValue> res{
         .success = false,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     Cron();
 

src/io/rsm/rsm_client.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,10 +14,12 @@
 #include <iostream>
 #include <optional>
 #include <type_traits>
+#include <unordered_map>
 #include <vector>
 
 #include "io/address.hpp"
 #include "io/errors.hpp"
+#include "io/notifier.hpp"
 #include "io/rsm/raft.hpp"
 #include "utils/result.hpp"
 
@@ -36,6 +38,14 @@ using memgraph::io::rsm::WriteRequest;
 using memgraph::io::rsm::WriteResponse;
 using memgraph::utils::BasicResult;
 
+template <typename RequestT, typename ResponseT>
+struct AsyncRequest {
+  Time start_time;
+  RequestT request;
+  Notifier notifier;
+  ResponseFuture<ResponseT> future;
+};
+
 template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
           typename ReadResponseT>
 class RsmClient {
@@ -47,23 +57,15 @@ class RsmClient {
 
   /// State for single async read/write operations. In the future this could become a map
   /// of async operations that can be accessed via an ID etc...
-  std::optional<Time> async_read_before_;
+  std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
-  std::optional<ResponseFuture<ReadResponse<ReadResponseT>>> async_read_;
+  std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
-  ReadRequestT current_read_request_;
-
-  std::optional<Time> async_write_before_;
-  std::optional<ResponseFuture<WriteResponse<WriteResponseT>>> async_write_;
-  WriteRequestT current_write_request_;
 
   void SelectRandomLeader() {
     std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
     size_t addr_index = io_.Rand(addr_distrib);
     leader_ = server_addrs_[addr_index];
 
-    spdlog::debug(
+    spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
-        "client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
-        "a RSM Candidate) trying a random one at index {} with address {}",
-        addr_index, leader_.ToString());
   }
 
   template <typename ResponseT>
@@ -91,107 +93,76 @@ class RsmClient {
   ~RsmClient() = default;
 
   BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
-    WriteRequest<WriteRequestT> client_req;
+    Notifier notifier;
-    client_req.operation = req;
+    const ReadinessToken readiness_token{0};
-
+    SendAsyncWriteRequest(std::move(req), notifier, readiness_token);
-    const Duration overall_timeout = io_.GetDefaultTimeout();
+    auto poll_result = AwaitAsyncWriteRequest(readiness_token);
-    const Time before = io_.Now();
+    while (!poll_result) {
-
+      poll_result = AwaitAsyncWriteRequest(readiness_token);
-    do {
+    }
-      spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
+    return poll_result.value();
-      ResponseFuture<WriteResponse<WriteResponseT>> response_future =
-          io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
-      ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
-
-      if (response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        return response_result.GetError();
-      }
-
-      ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
-      WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
-
-      if (write_response.success) {
-        return std::move(write_response.write_return);
-      }
-
-      PossiblyRedirectLeader(write_response);
-    } while (io_.Now() < before + overall_timeout);
-
-    return TimedOut{};
   }
 
   BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
-    ReadRequest<ReadRequestT> read_req;
+    Notifier notifier;
-    read_req.operation = req;
+    const ReadinessToken readiness_token{0};
-
+    SendAsyncReadRequest(std::move(req), notifier, readiness_token);
-    const Duration overall_timeout = io_.GetDefaultTimeout();
+    auto poll_result = AwaitAsyncReadRequest(readiness_token);
-    const Time before = io_.Now();
+    while (!poll_result) {
-
+      poll_result = AwaitAsyncReadRequest(readiness_token);
-    do {
+    }
-      spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
+    return poll_result.value();
-
-      ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
-          io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
-
-      // receive response
-      ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
-
-      if (get_response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        return get_response_result.GetError();
-      }
-
-      ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
-      ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
-
-      if (read_get_response.success) {
-        return std::move(read_get_response.read_return);
-      }
-
-      PossiblyRedirectLeader(read_get_response);
-    } while (io_.Now() < before + overall_timeout);
-
-    return TimedOut{};
   }
 
   /// AsyncRead methods
-  void SendAsyncReadRequest(const ReadRequestT &req) {
+  void SendAsyncReadRequest(ReadRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
-    MG_ASSERT(!async_read_);
-
     ReadRequest<ReadRequestT> read_req = {.operation = req};
 
-    if (!async_read_before_) {
+    AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
-      async_read_before_ = io_.Now();
+        .start_time = io_.Now(),
-    }
+        .request = std::move(req),
-    current_read_request_ = std::move(req);
+        .notifier = notifier,
-    async_read_ = io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
+        .future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
+            leader_, std::move(read_req), notifier, readiness_token),
+    };
+
+    async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
   }
 
-  std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest() {
+  void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
-    MG_ASSERT(async_read_);
+    auto &async_request = async_reads_.at(readiness_token.GetId());
 
-    if (!async_read_->IsReady()) {
+    ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
+
+    async_request.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
+        leader_, std::move(read_req), async_request.notifier, readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
+
+    if (!async_request.future.IsReady()) {
       return std::nullopt;
     }
 
-    return AwaitAsyncReadRequest();
+    return AwaitAsyncReadRequest(readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest() {
+  std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
-    ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(*async_read_).Wait();
+    auto &async_request = async_reads_.at(readiness_token.GetId());
-    async_read_.reset();
+    ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
 
     const Duration overall_timeout = io_.GetDefaultTimeout();
-    const bool past_time_out = io_.Now() < *async_read_before_ + overall_timeout;
+    const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
     const bool result_has_error = get_response_result.HasError();
 
     if (result_has_error && past_time_out) {
       // TODO static assert the exact type of error.
       spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-      async_read_before_ = std::nullopt;
+      async_reads_.erase(readiness_token.GetId());
       return TimedOut{};
     }
+
     if (!result_has_error) {
       ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
       ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
@@ -199,54 +170,69 @@ class RsmClient {
       PossiblyRedirectLeader(read_get_response);
 
       if (read_get_response.success) {
-        async_read_before_ = std::nullopt;
+        async_reads_.erase(readiness_token.GetId());
+        spdlog::debug("returning read_return for RSM request");
         return std::move(read_get_response.read_return);
       }
-      SendAsyncReadRequest(current_read_request_);
+    } else {
-    } else if (result_has_error) {
       SelectRandomLeader();
-      SendAsyncReadRequest(current_read_request_);
     }
+
+    ResendAsyncReadRequest(readiness_token);
+
     return std::nullopt;
   }
 
   /// AsyncWrite methods
-  void SendAsyncWriteRequest(const WriteRequestT &req) {
+  void SendAsyncWriteRequest(WriteRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
-    MG_ASSERT(!async_write_);
-
     WriteRequest<WriteRequestT> write_req = {.operation = req};
 
-    if (!async_write_before_) {
+    AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
-      async_write_before_ = io_.Now();
+        .start_time = io_.Now(),
-    }
+        .request = std::move(req),
-    current_write_request_ = std::move(req);
+        .notifier = notifier,
-    async_write_ = io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, write_req);
+        .future = io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
+            leader_, std::move(write_req), notifier, readiness_token),
+    };
+
+    async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
   }
 
-  std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest() {
+  void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
-    MG_ASSERT(async_write_);
+    auto &async_request = async_writes_.at(readiness_token.GetId());
 
-    if (!async_write_->IsReady()) {
+    WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
+
+    async_request.future =
+        io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
+            leader_, std::move(write_req), async_request.notifier, readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
+
+    if (!async_request.future.IsReady()) {
       return std::nullopt;
     }
 
-    return AwaitAsyncWriteRequest();
+    return AwaitAsyncWriteRequest(readiness_token);
   }
 
-  std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest() {
+  std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
-    ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(*async_write_).Wait();
+    auto &async_request = async_writes_.at(readiness_token.GetId());
-    async_write_.reset();
+    ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
 
     const Duration overall_timeout = io_.GetDefaultTimeout();
-    const bool past_time_out = io_.Now() < *async_write_before_ + overall_timeout;
+    const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
     const bool result_has_error = get_response_result.HasError();
 
     if (result_has_error && past_time_out) {
       // TODO static assert the exact type of error.
       spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-      async_write_before_ = std::nullopt;
+      async_writes_.erase(readiness_token.GetId());
       return TimedOut{};
     }
+
     if (!result_has_error) {
       ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
           std::move(get_response_result.GetValue());
@@ -255,14 +241,15 @@ class RsmClient {
       PossiblyRedirectLeader(write_get_response);
 
       if (write_get_response.success) {
-        async_write_before_ = std::nullopt;
+        async_writes_.erase(readiness_token.GetId());
         return std::move(write_get_response.write_return);
       }
-      SendAsyncWriteRequest(current_write_request_);
+    } else {
-    } else if (result_has_error) {
       SelectRandomLeader();
-      SendAsyncWriteRequest(current_write_request_);
     }
+
+    ResendAsyncWriteRequest(readiness_token);
+
     return std::nullopt;
   }
 };

src/io/simulator/simulator.hpp

@@ -41,7 +41,7 @@ class Simulator {
   Io<SimulatorTransport> Register(Address address) {
     std::uniform_int_distribution<uint64_t> seed_distrib;
     uint64_t seed = seed_distrib(rng_);
-    return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
+    return Io{SimulatorTransport(simulator_handle_, address, seed), address};
   }
 
   void IncrementServerCountAndWaitForQuiescentState(Address address) {
@@ -49,5 +49,14 @@ class Simulator {
   }
 
   SimulatorStats Stats() { return simulator_handle_->Stats(); }
+
+  std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
+
+  std::function<bool()> GetSimulatorTickClosure() {
+    std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
+      return handle_copy->MaybeTickSimulator();
+    };
+    return tick_closure;
+  }
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_config.hpp

@@ -26,5 +26,6 @@ struct SimulatorConfig {
   uint64_t rng_seed = 0;
   Time start_time = Time::min();
   Time abort_time = Time::max();
+  Duration message_delay = std::chrono::microseconds(100);
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_handle.cpp

@@ -23,6 +23,12 @@ namespace memgraph::io::simulator {
 void SimulatorHandle::ShutDown() {
   std::unique_lock<std::mutex> lock(mu_);
   should_shut_down_ = true;
+  for (auto it = promises_.begin(); it != promises_.end();) {
+    auto &[promise_key, dop] = *it;
+    std::move(dop).promise.TimeOut();
+    it = promises_.erase(it);
+  }
+  can_receive_.clear();
   cv_.notify_all();
 }
 
@@ -46,92 +52,94 @@ void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address addre
     const bool all_servers_blocked = blocked_servers == server_addresses_.size();
 
     if (all_servers_blocked) {
+      spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
+                    server_addresses_.size());
       return;
     }
 
+    spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
+                  server_addresses_.size());
     cv_.wait(lock);
   }
 }
 
+bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
+  // NB: never sort based on the request ID etc...
+  // This should only be used from std::stable_sort
+  // because by comparing on the from_address alone,
+  // we expect the sender ordering to remain
+  // deterministic.
+  const auto &[addr_1, om_1] = lhs;
+  const auto &[addr_2, om_2] = rhs;
+  return om_1.from_address < om_2.from_address;
+}
+
 bool SimulatorHandle::MaybeTickSimulator() {
   std::unique_lock<std::mutex> lock(mu_);
 
   const size_t blocked_servers = blocked_on_receive_.size();
 
-  // Don't judge, this is a workaround for poc. gvolfing
+  if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
-  server_count_++;
-  if (server_count_ == 3) {
-    is_quiescent_state_achieved_ = true;
-  }
-
-  // if (!is_quiescent_state_achieved_) {
-  if (!is_quiescent_state_achieved_) {
-    // return SimulatorProgression::DidSetUpOneServer;
-    cv_.notify_all();  // -> deadlock
-    // return false;// -> deadlock
-    return true;  // -> deadlock
-  }
-  // if (blocked_servers < server_addresses_.size() && is_quiescent_state_achieved_) {
-  if (blocked_servers < server_addresses_.size()) {
     // we only need to advance the simulator when all
     // servers have reached a quiescent state, blocked
     // on their own futures or receive methods.
     return false;
   }
 
+  // We allow the simulator to progress the state of the system only
+  // after all servers are blocked on receive.
+  spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
   stats_.simulator_ticks++;
-
+  blocked_on_receive_.clear();
   cv_.notify_all();
 
-  TimeoutPromisesPastDeadline();
+  bool timed_anything_out = TimeoutPromisesPastDeadline();
 
-  auto sort_based_on_message_id = [](const std::pair<Address, OpaqueMessage> &lhs,
+  if (timed_anything_out) {
-                                     const std::pair<Address, OpaqueMessage> &rhs) {
+    spdlog::trace("simulator progressing: timed out a request");
-    // return a.second.request_id < b.second.request_id;
+  }
-    if (lhs.second.from_address != rhs.second.from_address) {
-      return lhs.second.from_address < rhs.second.from_address;
-    }
 
-    return lhs.second.request_id < rhs.second.request_id;
+  const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
-  };
+
-  std::sort(in_flight_.begin(), in_flight_.end(), sort_based_on_message_id);
+  // We don't always want to advance the clock with every message that we deliver because
+  // when we advance it for every message, it causes timeouts to occur for any "request path"
+  // over a certain length. Alternatively, we don't want to simply deliver multiple messages
+  // in a single simulator tick because that would reduce the amount of concurrent message
+  // mixing that may naturally occur in production. This approach is to mod the random clock
+  // advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
+  // by only advancing the clock by an even amount etc...) and only advancing the clock close to
+  // half of the time.
+  if (clock_advance.count() % 97 > 49) {
+    spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
+    cluster_wide_time_microseconds_ += clock_advance;
+    stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
+  }
+
+  if (cluster_wide_time_microseconds_ >= config_.abort_time) {
+    spdlog::error(
+        "Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
+        "in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
+        config_.rng_seed);
+    throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
+  }
 
   if (in_flight_.empty()) {
-    // return early here because there are no messages to schedule
-
-    // We tick the clock forward when all servers are blocked but
-    // there are no in-flight messages to schedule delivery of.
-    // const Duration clock_advance = std::chrono::microseconds{Rand(time_distrib_)};
-    // cluster_wide_time_microseconds_ += clock_advance;
-
-    if (cluster_wide_time_microseconds_ >= config_.abort_time) {
-      if (should_shut_down_) {
-        return false;
-      }
-      spdlog::error(
-          "Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
-          "in an expected amount of time.");
-      throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
-    }
-    // spdlog::info(
-    //     "Time increased with {} to {}", clock_advance.count(),
-    //     std::chrono::duration_cast<std::chrono::milliseconds>(cluster_wide_time_microseconds_.time_since_epoch())
-    //         .count());
     return true;
   }
 
-  if (config_.scramble_messages) {
+  std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
+
+  if (config_.scramble_messages && in_flight_.size() > 1) {
     // scramble messages
     std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
-    const size_t swap_index = RandLocked(swap_distrib);
+    const size_t swap_index = swap_distrib(rng_);
     std::swap(in_flight_[swap_index], in_flight_.back());
   }
 
   auto [to_address, opaque_message] = std::move(in_flight_.back());
   in_flight_.pop_back();
-  event_log_.AddAndLog(opaque_message, "Handle ");
 
-  const int drop_threshold = RandLocked(drop_distrib_);
+  const int drop_threshold = drop_distrib_(rng_);
   const bool should_drop = drop_threshold < config_.drop_percent;
 
   if (should_drop) {
@@ -141,7 +149,6 @@ bool SimulatorHandle::MaybeTickSimulator() {
   PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
 
   if (promises_.contains(promise_key)) {
-    event_log_.AddAndLog(opaque_message, "Promise contains");
     // complete waiting promise if it's there
     DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
     promises_.erase(promise_key);
@@ -151,27 +158,25 @@ bool SimulatorHandle::MaybeTickSimulator() {
     if (should_drop || normal_timeout) {
       stats_.timed_out_requests++;
       dop.promise.TimeOut();
+      spdlog::trace("simulator timing out request ");
     } else {
       stats_.total_responses++;
       Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
       auto type_info = opaque_message.type_info;
       dop.promise.Fill(std::move(opaque_message), response_latency);
       histograms_.Measure(type_info, response_latency);
+      spdlog::trace("simulator replying to request");
     }
   } else if (should_drop) {
-    event_log_.AddAndLog(opaque_message, "Dropped");
     // don't add it anywhere, let it drop
+    spdlog::trace("simulator silently dropping request");
   } else {
-    event_log_.AddAndLog(opaque_message, "To can_receive");
     // add to can_receive_ if not
-    // This might be needed I am not sure, it can cause deadlock if you uncomment. Didn't find the issue yet.
+    spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
-    // MG_ASSERT(server_addresses_.contains(to_address));
+                  opaque_message.to_address.last_known_port);
-    // MG_ASSERT(blocked_on_receive_.contains(to_address));
-    // blocked_on_receive_.erase(to_address);
-
     const auto &[om_vec, inserted] =
-        can_receive_.try_emplace(to_address.ToPartialAddress(), std::vector<OpaqueMessage>());
+        can_receive_.try_emplace(to_address.ToPartialAddress(), std::deque<OpaqueMessage>());
-    om_vec->second.emplace_back(std::move(opaque_message));
+    om_vec->second.emplace_front(std::move(opaque_message));
   }
 
   return true;
@@ -182,11 +187,6 @@ Time SimulatorHandle::Now() const {
   return cluster_wide_time_microseconds_;
 }
 
-Time SimulatorHandle::NowLocked() const {
-  // std::unique_lock<std::mutex> lock(mu_);
-  return cluster_wide_time_microseconds_;
-}
-
 SimulatorStats SimulatorHandle::Stats() {
   std::unique_lock<std::mutex> lock(mu_);
   return stats_;

src/io/simulator/simulator_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,20 +17,13 @@
 #include <map>
 #include <memory>
 #include <optional>
-#include <queue>
 #include <set>
-#include <string>
 #include <utility>
 #include <variant>
 #include <vector>
 
-#include <sstream>
+#include <boost/core/demangle.hpp>
 
-#include <boost/lexical_cast.hpp>
-#include <boost/stacktrace.hpp>
-#include <boost/uuid/uuid_io.hpp>
-
-#include "fmt/format.h"
 #include "io/address.hpp"
 #include "io/errors.hpp"
 #include "io/message_conversion.hpp"
@@ -39,50 +32,9 @@
 #include "io/simulator/simulator_stats.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
-#include "spdlog/spdlog.h"
 
 namespace memgraph::io::simulator {
 
-struct EventDescriptor {
-  boost::asio::ip::address from_address_ip;
-  uint16_t from_address_port;
-  boost::asio::ip::address to_address_ip;
-  uint16_t to_address_port;
-  uint64_t request_id;
-  std::string_view caller;
-  utils::TypeInfoRef type_info;
-  uint64_t event_id;
-  Time time;
-};
-
-class SimulatorHandle;
-class EventLog {
-  std::vector<EventDescriptor> event_log_;
-  static constexpr const char *eventlog_signal_string_ = "EVENTLOG_UPDATE";
-
-  void LogEvent(const EventDescriptor &event) const {
-    spdlog::info(
-        fmt::format("{} ({})-> caller: {}, from_address: {}:{}, to_address: {}:{}, request_id: {}, type: {}, event_id: "
-                    "{}, thread_id: {}",
-                    eventlog_signal_string_,
-                    std::chrono::duration_cast<std::chrono::milliseconds>(event.time.time_since_epoch()).count(),
-                    event.caller, event.from_address_ip.to_string(), event.from_address_port,
-                    event.to_address_ip.to_string(), event.to_address_port, event.request_id,
-                    event.type_info.get().name(), event.event_id, std::this_thread::get_id()));
-  }
-
- public:
-  SimulatorHandle *handle{nullptr};
-
-  void AddAndLog(const OpaqueMessage &message, std::string_view caller);
-
-  void LogAllEvents() const {
-    for (const auto &event : event_log_) {
-      LogEvent(event);
-    }
-  }
-};
-
 class SimulatorHandle {
   mutable std::mutex mu_{};
   mutable std::condition_variable cv_;
@@ -94,11 +46,7 @@ class SimulatorHandle {
   std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
 
   // messages that are sent to servers that may later receive them
-  std::map<PartialAddress, std::vector<OpaqueMessage>> can_receive_;
+  std::map<PartialAddress, std::deque<OpaqueMessage>> can_receive_;
-
-  // maybe a boolean here
-  bool is_quiescent_state_achieved_ = false;
-  std::atomic<int> server_count_;
 
   Time cluster_wide_time_microseconds_;
   bool should_shut_down_ = false;
@@ -106,38 +54,34 @@ class SimulatorHandle {
   std::set<Address> blocked_on_receive_;
   std::set<Address> server_addresses_;
   std::mt19937 rng_;
-  std::uniform_int_distribution<int> time_distrib_{5, 50};
+  std::uniform_int_distribution<int> time_distrib_{0, 30000};
   std::uniform_int_distribution<int> drop_distrib_{0, 99};
   SimulatorConfig config_;
   MessageHistogramCollector histograms_;
   RequestId request_id_counter_{0};
-  EventLog event_log_;
 
-  void TimeoutPromisesPastDeadline() {
+  bool TimeoutPromisesPastDeadline() {
+    bool timed_anything_out = false;
     const Time now = cluster_wide_time_microseconds_;
     for (auto it = promises_.begin(); it != promises_.end();) {
       auto &[promise_key, dop] = *it;
-      const bool timed_out = dop.deadline < now;
+      if (dop.deadline < now && config_.perform_timeouts) {
-      if (timed_out && config_.perform_timeouts) {
+        spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
-        // spdlog::info("timing out request from requester {}.", promise_key.requester_address.ToString());
-        spdlog::info("timing out request from requester {}. bool timed_out: {}",
-                     promise_key.requester_address.ToString(), timed_out);
         std::move(dop).promise.TimeOut();
         it = promises_.erase(it);
 
         stats_.timed_out_requests++;
+        timed_anything_out = true;
       } else {
         ++it;
       }
     }
+    return timed_anything_out;
   }
 
  public:
   explicit SimulatorHandle(SimulatorConfig config)
-      : server_count_(0), cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {
+      : cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
-    spdlog::info("SimulatorHandle constructed.");
-    event_log_.handle = this;
-  }
 
   LatencyHistogramSummaries ResponseLatencies();
 
@@ -161,43 +105,51 @@ class SimulatorHandle {
 
   bool ShouldShutDown() const;
 
-  template <Message Request, Message Response>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
+  ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
-                                         std::function<bool()> &&maybe_tick_simulator) {
+                                          Duration timeout, std::function<bool()> &&maybe_tick_simulator,
+                                          std::function<void()> &&fill_notifier) {
     auto type_info = TypeInfoFor(request);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
 
-    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
-        std::forward<std::function<bool()>>(maybe_tick_simulator));
+        // set notifier for when the Future::Wait is called
+        std::move(maybe_tick_simulator),
+        // set notifier for when Promise::Fill is called
+        std::move(fill_notifier));
 
-    std::unique_lock<std::mutex> lock(mu_);
+    {
+      std::unique_lock<std::mutex> lock(mu_);
 
-    RequestId request_id = ++request_id_counter_;
+      RequestId request_id = ++request_id_counter_;
 
-    const Time deadline = cluster_wide_time_microseconds_ + timeout;
+      const Time deadline = cluster_wide_time_microseconds_ + timeout;
 
-    std::any message(request);
+      std::any message(std::move(request));
-    OpaqueMessage om{.to_address = to_address,
+      OpaqueMessage om{.to_address = to_address,
-                     .from_address = from_address,
+                       .from_address = from_address,
-                     .request_id = request_id,
+                       .request_id = request_id,
-                     .message = std::move(message),
+                       .message = std::move(message),
-                     .type_info = type_info};
+                       .type_info = type_info,
-    event_log_.AddAndLog(om, "SubmitRequest");
+                       .deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
-    in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
+      in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
 
-    PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
+      PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
-    OpaquePromise opaque_promise(std::move(promise).ToUnique());
+      OpaquePromise opaque_promise(std::move(promise).ToUnique());
-    DeadlineAndOpaquePromise dop{
+      DeadlineAndOpaquePromise dop{
-        .requested_at = cluster_wide_time_microseconds_,
+          .requested_at = cluster_wide_time_microseconds_,
-        .deadline = deadline,
+          .deadline = deadline,
-        .promise = std::move(opaque_promise),
+          .promise = std::move(opaque_promise),
-    };
+      };
 
-    MG_ASSERT(!promises_.contains(promise_key));
+      MG_ASSERT(!promises_.contains(promise_key));
 
-    promises_.emplace(std::move(promise_key), std::move(dop));
+      promises_.emplace(std::move(promise_key), std::move(dop));
 
-    stats_.total_messages++;
+      stats_.total_messages++;
-    stats_.total_requests++;
+      stats_.total_requests++;
+    }  // lock dropped here
 
     cv_.notify_all();
 
@@ -208,96 +160,79 @@ class SimulatorHandle {
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
     std::unique_lock<std::mutex> lock(mu_);
 
-    blocked_on_receive_.emplace(receiver);
-
     const Time deadline = cluster_wide_time_microseconds_ + timeout;
 
     auto partial_address = receiver.ToPartialAddress();
 
     while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
       if (can_receive_.contains(partial_address)) {
-        std::vector<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
+        std::deque<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
-        if (!can_rx.empty()) {
+
+        bool contains_items = !can_rx.empty();
+        bool can_receive = contains_items && can_rx.back().deliverable_at <= cluster_wide_time_microseconds_;
+
+        if (can_receive) {
           OpaqueMessage message = std::move(can_rx.back());
           can_rx.pop_back();
 
-          event_log_.AddAndLog(message, "Receivexxxxxx");
-
           // TODO(tyler) search for item in can_receive_ that matches the desired types, rather
           // than asserting that the last item in can_rx matches.
           auto m_opt = std::move(message).Take<Ms...>();
           MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
 
-          blocked_on_receive_.erase(receiver);
-
           return std::move(m_opt).value();
         }
+        if (contains_items) {
+          auto count = can_rx.back().deliverable_at.time_since_epoch().count();
+          auto now_count = cluster_wide_time_microseconds_.time_since_epoch().count();
+          spdlog::trace("can't receive message yet due to artificial latency. deliverable_at: {}, now: {}", count,
+                        now_count);
+        }
       }
 
-      lock.unlock();
+      if (!should_shut_down_) {
-      auto simulator_progress = MaybeTickSimulator();
+        if (!blocked_on_receive_.contains(receiver)) {
-      lock.lock();
+          blocked_on_receive_.emplace(receiver);
-      if (!should_shut_down_ && !simulator_progress) {
+          spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
+          cv_.notify_all();
+        }
         cv_.wait(lock);
       }
     }
-
+    spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
-    blocked_on_receive_.erase(receiver);
 
     return TimedOut{};
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M message) {
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
+    spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
     auto type_info = TypeInfoFor(message);
-    std::unique_lock<std::mutex> lock(mu_);
+    {
-    std::any message_any(std::move(message));
+      std::unique_lock<std::mutex> lock(mu_);
-    OpaqueMessage om{.to_address = to_address,
+      std::any message_any(std::move(message));
-                     .from_address = from_address,
+      OpaqueMessage om{.to_address = to_address,
-                     .request_id = request_id,
+                       .from_address = from_address,
-                     .message = std::move(message_any),
+                       .request_id = request_id,
-                     .type_info = type_info};
+                       .message = std::move(message_any),
-    event_log_.AddAndLog(om, "Sendxxxxxxxxx");
+                       .type_info = type_info,
-    in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
+                       .deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
+      in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
 
-    stats_.total_messages++;
+      stats_.total_messages++;
+    }  // lock dropped before cv notification
 
     cv_.notify_all();
   }
 
   Time Now() const;
 
-  Time NowLocked() const;
-
   template <class D = std::poisson_distribution<>, class Return = uint64_t>
   Return Rand(D distrib) {
     std::unique_lock<std::mutex> lock(mu_);
-    return RandLocked<D, Return>(std::forward<D>(distrib));
+    return distrib(rng_);
-  }
-
-  template <class D = std::poisson_distribution<>, class Return = uint64_t>
-  Return RandLocked(D distrib) {
-    Return res = distrib(rng_);
-    spdlog::info("Getting random value from thread {}: {}. Stacktrace:\n{}", std::this_thread::get_id(), res,
-                 boost::stacktrace::stacktrace());
-    return res;
   }
 
   SimulatorStats Stats();
 };
-
-inline void EventLog::AddAndLog(const OpaqueMessage &message, std::string_view caller) {
-  EventDescriptor event{.from_address_ip = message.from_address.last_known_ip,
-                        .from_address_port = message.from_address.last_known_port,
-                        .to_address_ip = message.to_address.last_known_ip,
-                        .to_address_port = message.to_address.last_known_port,
-                        .request_id = message.request_id,
-                        .caller = caller,
-                        .type_info = message.type_info,
-                        .event_id = event_log_.size(),
-                        .time = handle->NowLocked()};
-
-  event_log_.push_back(event);
-  LogEvent(event);
-}
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_stats.hpp

@@ -13,6 +13,10 @@
 
 #include <cstdint>
 
+#include <fmt/format.h>
+
+#include "io/time.hpp"
+
 namespace memgraph::io::simulator {
 struct SimulatorStats {
   uint64_t total_messages = 0;
@@ -21,5 +25,22 @@ struct SimulatorStats {
   uint64_t total_requests = 0;
   uint64_t total_responses = 0;
   uint64_t simulator_ticks = 0;
+  Duration elapsed_time;
+
+  friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
+
+  friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
+    auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
+
+    std::string formated = fmt::format(
+        "SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
+        "total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
+        stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
+        stats.total_responses, stats.simulator_ticks, elapsed_ms);
+
+    in << formated;
+
+    return in;
+  }
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,6 +15,7 @@
 #include <utility>
 
 #include "io/address.hpp"
+#include "io/notifier.hpp"
 #include "io/simulator/simulator_handle.hpp"
 #include "io/time.hpp"
 
@@ -25,19 +26,22 @@ using memgraph::io::Time;
 
 class SimulatorTransport {
   std::shared_ptr<SimulatorHandle> simulator_handle_;
-  const Address address_;
+  Address address_;
   std::mt19937 rng_;
 
  public:
   SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
       : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
 
-  template <Message RequestT, Message ResponseT>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
-    std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
+                                    std::function<void()> notification, Duration timeout) {
+    std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
+      return handle_copy->MaybeTickSimulator();
+    };
 
-    return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address, std::move(request),
+    return simulator_handle_->template SubmitRequest<ResponseT, RequestT>(
-                                                                          timeout, std::move(maybe_tick_simulator));
+        to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
   }
 
   template <Message... Ms>
@@ -46,19 +50,17 @@ class SimulatorTransport {
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
+  void Send(Address to_address, Address from_address, uint64_t request_id, RValueRef<M> message) {
-    return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
+    return simulator_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
   }
 
   Time Now() const { return simulator_handle_->Now(); }
 
   bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
 
-  template <class D = std::poisson_distribution<uint64_t>, class Return = uint64_t>
+  template <class D = std::poisson_distribution<>, class Return = uint64_t>
   Return Rand(D distrib) {
-    // debug - gvolfing
+    return distrib(rng_);
-    auto ret = distrib(rng_);
-    return ret;
   }
 
   LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }

src/io/transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,9 @@
 #include "io/errors.hpp"
 #include "io/future.hpp"
 #include "io/message_histogram_collector.hpp"
+#include "io/notifier.hpp"
 #include "io/time.hpp"
+#include "utils/concepts.hpp"
 #include "utils/result.hpp"
 
 namespace memgraph::io {
@@ -31,7 +33,15 @@ using memgraph::utils::BasicResult;
 // reasonable constraints around message types over time,
 // as we adapt things to use Thrift-generated message types.
 template <typename T>
-concept Message = std::same_as<T, std::decay_t<T>>;
+concept Message = std::movable<T> && std::copyable<T>;
+
+template <utils::Object T>
+struct RValueRefEnforcer {
+  using Type = T &&;
+};
+
+template <typename T>
+using RValueRef = typename RValueRefEnforcer<T>::Type;
 
 using RequestId = uint64_t;
 
@@ -81,19 +91,44 @@ class Io {
   Duration GetDefaultTimeout() { return default_timeout_; }
 
   /// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
-  template <Message RequestT, Message ResponseT>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
+  ResponseFuture<ResponseT> RequestWithTimeout(Address address, RValueRef<RequestT> request, Duration timeout) {
     const Address from_address = address_;
-    return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, timeout);
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<ResponseT, RequestT>(address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
   }
 
   /// Issue a request that times out after the default timeout. This tends
   /// to be used by clients.
-  template <Message RequestT, Message ResponseT>
+  template <Message ResponseT, Message RequestT>
-  ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
+  ResponseFuture<ResponseT> Request(Address to_address, RValueRef<RequestT> request) {
     const Duration timeout = default_timeout_;
     const Address from_address = address_;
-    return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request), timeout);
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RValueRef<RequestT> request, Notifier notifier,
+                                                    ReadinessToken readiness_token) {
+    const Duration timeout = default_timeout_;
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT &&request, Notifier notifier,
+                                                              ReadinessToken readiness_token, Duration timeout) {
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<ResponseT>(to_address, from_address, std::forward<RequestT>(request),
+                                                       fill_notifier, timeout);
   }
 
   /// Wait for an explicit number of microseconds for a request of one of the
@@ -115,9 +150,9 @@ class Io {
   /// responses are not necessarily expected, and for servers to respond to requests.
   /// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
   template <Message M>
-  void Send(Address to_address, RequestId request_id, M message) {
+  void Send(Address to_address, RequestId request_id, M &&message) {
     Address from_address = address_;
-    return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
+    return implementation_.template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
   }
 
   /// The current system time. This time source should be preferred over any other,
@@ -137,7 +172,14 @@ class Io {
   Address GetAddress() { return address_; }
   void SetAddress(Address address) { address_ = address; }
 
-  Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
+  Io<I> ForkLocal(boost::uuids::uuid uuid) {
+    Address new_address{
+        .unique_id = uuid,
+        .last_known_ip = address_.last_known_ip,
+        .last_known_port = address_.last_known_port,
+    };
+    return Io(implementation_, new_address);
+  }
 
   LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
 };

src/machine_manager/machine_config.hpp

@@ -42,6 +42,7 @@ struct MachineConfig {
   boost::asio::ip::address listen_ip;
   uint16_t listen_port;
   size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
+  bool sync_message_handling = false;
 };
 
 }  // namespace memgraph::machine_manager

src/machine_manager/machine_manager.hpp

@@ -78,10 +78,10 @@ class MachineManager {
   MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
       : io_(io),
         config_(config),
-        coordinator_address_(io.GetAddress().ForkUniqueAddress()),
+        coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
-        shard_manager_{io.ForkLocal(), config.shard_worker_threads, coordinator_address_} {
+        shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
-    auto coordinator_io = io.ForkLocal();
+                       coordinator_address_} {
-    coordinator_io.SetAddress(coordinator_address_);
+    auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
     CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
     coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
   }
@@ -101,11 +101,23 @@ class MachineManager {
   Address CoordinatorAddress() { return coordinator_address_; }
 
   void Run() {
-    while (!io_.ShouldShutDown()) {
+    while (true) {
+      MaybeBlockOnSyncHandling();
+
+      if (io_.ShouldShutDown()) {
+        break;
+      }
+
       const auto now = io_.Now();
 
+      uint64_t now_us = now.time_since_epoch().count();
+      uint64_t next_us = next_cron_.time_since_epoch().count();
+
       if (now >= next_cron_) {
+        spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
         next_cron_ = Cron();
+      } else {
+        spdlog::info("now {} < next_cron_ {}", now_us, next_us);
       }
 
       Duration receive_timeout = std::max(next_cron_, now) - now;
@@ -194,10 +206,27 @@ class MachineManager {
   }
 
  private:
+  // This method exists for controlling concurrency
+  // during deterministic simulation testing.
+  void MaybeBlockOnSyncHandling() {
+    if (!config_.sync_message_handling) {
+      return;
+    }
+
+    // block on coordinator
+    coordinator_queue_.BlockOnQuiescence();
+
+    // block on shards
+    shard_manager_.BlockOnQuiescence();
+  }
+
   Time Cron() {
     spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
     coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
-    return shard_manager_.Cron();
+    MaybeBlockOnSyncHandling();
+    Time ret = shard_manager_.Cron();
+    MaybeBlockOnSyncHandling();
+    return ret;
   }
 };
 

src/memgraph.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,6 +33,7 @@
 #include <spdlog/sinks/dist_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 
+#include "common/errors.hpp"
 #include "communication/bolt/v1/constants.hpp"
 #include "communication/websocket/auth.hpp"
 #include "communication/websocket/server.hpp"
@@ -453,7 +454,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
 
   std::map<std::string, memgraph::communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
                                                                    std::optional<int> qid) override {
-    TypedValueResultStream stream(encoder, interpreter_.GetShardRequestManager());
+    TypedValueResultStream stream(encoder, interpreter_.GetRequestRouter());
     return PullResults(stream, n, qid);
   }
 
@@ -480,20 +481,9 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
       const auto &summary = interpreter_.Pull(&stream, n, qid);
       std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
       for (const auto &kv : summary) {
-        auto maybe_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetShardRequestManager(),
+        auto bolt_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetRequestRouter(),
-                                                           memgraph::storage::v3::View::NEW);
+                                                          memgraph::storage::v3::View::NEW);
-        if (maybe_value.HasError()) {
+        decoded_summary.emplace(kv.first, std::move(bolt_value));
-          switch (maybe_value.GetError()) {
-            case memgraph::storage::v3::Error::DELETED_OBJECT:
-            case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
-            case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
-            case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
-            case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
-            case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
-              throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming summary.");
-          }
-        }
-        decoded_summary.emplace(kv.first, std::move(*maybe_value));
       }
       return decoded_summary;
     } catch (const memgraph::query::v2::QueryException &e) {
@@ -507,35 +497,22 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
   /// before forwarding the calls to original TEncoder.
   class TypedValueResultStream {
    public:
-    TypedValueResultStream(TEncoder *encoder, const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager)
+    TypedValueResultStream(TEncoder *encoder, const memgraph::query::v2::RequestRouterInterface *request_router)
-        : encoder_(encoder), shard_request_manager_(shard_request_manager) {}
+        : encoder_(encoder), request_router_(request_router) {}
 
     void Result(const std::vector<memgraph::query::v2::TypedValue> &values) {
       std::vector<memgraph::communication::bolt::Value> decoded_values;
       decoded_values.reserve(values.size());
       for (const auto &v : values) {
-        auto maybe_value = memgraph::glue::v2::ToBoltValue(v, shard_request_manager_, memgraph::storage::v3::View::NEW);
+        auto bolt_value = memgraph::glue::v2::ToBoltValue(v, request_router_, memgraph::storage::v3::View::NEW);
-        if (maybe_value.HasError()) {
+        decoded_values.emplace_back(std::move(bolt_value));
-          switch (maybe_value.GetError()) {
-            case memgraph::storage::v3::Error::DELETED_OBJECT:
-              throw memgraph::communication::bolt::ClientError("Returning a deleted object as a result.");
-            case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
-              throw memgraph::communication::bolt::ClientError("Returning a nonexistent object as a result.");
-            case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
-            case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
-            case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
-            case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
-              throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming results.");
-          }
-        }
-        decoded_values.emplace_back(std::move(*maybe_value));
       }
       encoder_->MessageRecord(decoded_values);
     }
 
    private:
     TEncoder *encoder_;
-    const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager_{nullptr};
+    const memgraph::query::v2::RequestRouterInterface *request_router_{nullptr};
   };
   memgraph::query::v2::Interpreter interpreter_;
   memgraph::communication::v2::ServerEndpoint endpoint_;
@@ -663,6 +640,8 @@ int main(int argc, char **argv) {
   memgraph::machine_manager::MachineManager<memgraph::io::local_transport::LocalTransport> mm{io, config, coordinator};
   std::jthread mm_thread([&mm] { mm.Run(); });
 
+  auto rr_factory = std::make_unique<memgraph::query::v2::LocalRequestRouterFactory>(io);
+
   memgraph::query::v2::InterpreterContext interpreter_context{
       (memgraph::storage::v3::Shard *)(nullptr),
       {.query = {.allow_load_csv = FLAGS_allow_load_csv},
@@ -673,7 +652,7 @@ int main(int argc, char **argv) {
        .stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries,
        .stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)},
       FLAGS_data_directory,
-      std::move(io),
+      std::move(rr_factory),
       mm.CoordinatorAddress()};
 
   SessionData session_data{&interpreter_context};

src/parser/CMakeLists.txt

@@ -23,7 +23,7 @@ add_custom_command(
     COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
     COMMAND
         java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-            -Dlanguage=Cpp -visitor -package antlropencypher
+            -Dlanguage=Cpp -visitor -package antlropencypher::v2
             -o ${opencypher_generated}
             ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"

src/parser/opencypher/parser.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,10 +14,10 @@
 #include <string>
 
 #include "antlr4-runtime.h"
-#include "utils/exceptions.hpp"
 #include "parser/opencypher/generated/MemgraphCypher.h"
 #include "parser/opencypher/generated/MemgraphCypherLexer.h"
 #include "utils/concepts.hpp"
+#include "utils/exceptions.hpp"
 
 namespace memgraph::frontend::opencypher {
 
@@ -32,11 +32,9 @@ class SyntaxException : public utils::BasicException {
  * This thing must me a class since parser.cypher() returns pointer and there is
  * no way for us to get ownership over the object.
  */
-enum class ParserOpTag : uint8_t {
+enum class ParserOpTag : uint8_t { CYPHER, EXPRESSION };
-  CYPHER, EXPRESSION
-};
 
-template<ParserOpTag Tag = ParserOpTag::CYPHER>
+template <ParserOpTag Tag = ParserOpTag::CYPHER>
 class Parser {
  public:
   /**
@@ -46,10 +44,9 @@ class Parser {
   Parser(const std::string query) : query_(std::move(query)) {
     parser_.removeErrorListeners();
     parser_.addErrorListener(&error_listener_);
-    if constexpr(Tag == ParserOpTag::CYPHER) {
+    if constexpr (Tag == ParserOpTag::CYPHER) {
       tree_ = parser_.cypher();
-    }
+    } else {
-    else {
       tree_ = parser_.expression();
     }
     if (parser_.getNumberOfSyntaxErrors()) {
@@ -75,11 +72,11 @@ class Parser {
   FirstMessageErrorListener error_listener_;
   std::string query_;
   antlr4::ANTLRInputStream input_{query_};
-  antlropencypher::MemgraphCypherLexer lexer_{&input_};
+  antlropencypher::v2::MemgraphCypherLexer lexer_{&input_};
   antlr4::CommonTokenStream tokens_{&lexer_};
 
   // generate ast
-  antlropencypher::MemgraphCypher parser_{&tokens_};
+  antlropencypher::v2::MemgraphCypher parser_{&tokens_};
   antlr4::tree::ParseTree *tree_ = nullptr;
 };
 }  // namespace memgraph::frontend::opencypher

src/query/CMakeLists.txt

@@ -48,18 +48,20 @@ add_dependencies(mg-query generate_lcp_query)
 target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(mg-query dl cppitertools Boost::headers)
 target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
+
 if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
     set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
 endif()
+
 if("${MG_PYTHON_VERSION}" STREQUAL "")
     find_package(Python3 3.5 REQUIRED COMPONENTS Development)
 else()
     find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
 endif()
+
 target_link_libraries(mg-query Python3::Python)
 
 # Generate Antlr openCypher parser
-
 set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
 set(opencypher_generated ${opencypher_frontend}/generated)
 set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
@@ -82,15 +84,15 @@ add_custom_command(
     OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
     COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
     COMMAND
-        java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
+    java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-            -Dlanguage=Cpp -visitor -package antlropencypher
+    -Dlanguage=Cpp -visitor -package antlropencypher
-            -o ${opencypher_generated}
+    -o ${opencypher_generated}
-            ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+    ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     DEPENDS
-        ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+    ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
-        ${opencypher_frontend}/grammar/CypherLexer.g4
+    ${opencypher_frontend}/grammar/CypherLexer.g4
-        ${opencypher_frontend}/grammar/Cypher.g4)
+    ${opencypher_frontend}/grammar/Cypher.g4)
 
 add_custom_target(generate_opencypher_parser
     DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})

src/query/v2/CMakeLists.txt

@@ -11,7 +11,6 @@ set(mg_query_v2_sources
     cypher_query_interpreter.cpp
     frontend/semantic/required_privileges.cpp
     frontend/stripped.cpp
-    interpret/awesome_memgraph_functions.cpp
     interpreter.cpp
     metadata.cpp
     plan/operator.cpp
@@ -24,7 +23,8 @@ set(mg_query_v2_sources
     plan/variable_start_planner.cpp
     serialization/property_value.cpp
     bindings/typed_value.cpp
-    accessors.cpp)
+    accessors.cpp
+    multiframe.cpp)
 
 find_package(Boost REQUIRED)
 
@@ -34,7 +34,7 @@ target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
 target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
 target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
-target_link_libraries(mg-query-v2 mg-expr)
+target_link_libraries(mg-query-v2 mg-expr mg-functions)
 
 if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
     set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")

src/query/v2/accessors.cpp

@@ -10,24 +10,25 @@
 // licenses/APL.txt.
 
 #include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
 #include "query/v2/requests.hpp"
-#include "query/v2/shard_request_manager.hpp"
 #include "storage/v3/id_types.hpp"
 
 namespace memgraph::query::v2::accessors {
-EdgeAccessor::EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager)
+EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
-    : edge(std::move(edge)), manager_(manager) {}
+    : edge(std::move(edge)), request_router_(request_router) {}
 
 EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
 
 const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
 
 Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
-  auto prop_id = manager_->NameToProperty(prop_name);
+  auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
-  auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
+  if (!maybe_prop) {
-  if (it == edge.properties.end()) {
     return {};
   }
+  const auto prop_id = *maybe_prop;
+  auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
   return it->second;
 }
 
@@ -35,21 +36,23 @@ const Edge &EdgeAccessor::GetEdge() const { return edge; }
 
 bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
 
+size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
+
 VertexAccessor EdgeAccessor::To() const {
-  return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
+  return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
 }
 
 VertexAccessor EdgeAccessor::From() const {
-  return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
+  return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
 }
 
 VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
-                               const msgs::ShardRequestManagerInterface *manager)
+                               const RequestRouterInterface *request_router)
-    : vertex(std::move(v)), properties(std::move(props)), manager_(manager) {}
+    : vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
 
 VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
-                               const msgs::ShardRequestManagerInterface *manager)
+                               const RequestRouterInterface *request_router)
-    : vertex(std::move(v)), manager_(manager) {
+    : vertex(std::move(v)), request_router_(request_router) {
   properties.reserve(props.size());
   for (auto &[id, value] : props) {
     properties.emplace_back(std::make_pair(id, std::move(value)));
@@ -57,8 +60,8 @@ VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
 }
 
 VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
-                               const msgs::ShardRequestManagerInterface *manager)
+                               const RequestRouterInterface *request_router)
-    : vertex(std::move(v)), manager_(manager) {
+    : vertex(std::move(v)), request_router_(request_router) {
   properties.reserve(props.size());
   for (const auto &[id, value] : props) {
     properties.emplace_back(std::make_pair(id, value));
@@ -88,7 +91,11 @@ Value VertexAccessor::GetProperty(PropertyId prop_id) const {
 
 // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
 Value VertexAccessor::GetProperty(const std::string &prop_name) const {
-  return GetProperty(manager_->NameToProperty(prop_name));
+  auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
+  if (!maybe_prop) {
+    return {};
+  }
+  return GetProperty(*maybe_prop);
 }
 
 msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }

src/query/v2/accessors.hpp

@@ -24,24 +24,24 @@
 #include "utils/memory.hpp"
 #include "utils/memory_tracker.hpp"
 
-namespace memgraph::msgs {
+namespace memgraph::query::v2 {
-class ShardRequestManagerInterface;
+class RequestRouterInterface;
-}  // namespace memgraph::msgs
+}  // namespace memgraph::query::v2
 
 namespace memgraph::query::v2::accessors {
 
-using Value = memgraph::msgs::Value;
+using Value = msgs::Value;
-using Edge = memgraph::msgs::Edge;
+using Edge = msgs::Edge;
-using Vertex = memgraph::msgs::Vertex;
+using Vertex = msgs::Vertex;
-using Label = memgraph::msgs::Label;
+using Label = msgs::Label;
-using PropertyId = memgraph::msgs::PropertyId;
+using PropertyId = msgs::PropertyId;
-using EdgeTypeId = memgraph::msgs::EdgeTypeId;
+using EdgeTypeId = msgs::EdgeTypeId;
 
 class VertexAccessor;
 
 class EdgeAccessor final {
  public:
-  explicit EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager);
+  explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
 
   [[nodiscard]] EdgeTypeId EdgeType() const;
 
@@ -53,12 +53,7 @@ class EdgeAccessor final {
 
   [[nodiscard]] bool IsCycle() const;
 
-  // Dummy function
+  [[nodiscard]] size_t CypherId() const;
-  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  [[nodiscard]] size_t CypherId() const { return 10; }
-
-  //  bool HasSrcAccessor const { return src == nullptr; }
-  //  bool HasDstAccessor const { return dst == nullptr; }
 
   [[nodiscard]] VertexAccessor To() const;
   [[nodiscard]] VertexAccessor From() const;
@@ -69,7 +64,7 @@ class EdgeAccessor final {
 
  private:
   Edge edge;
-  const msgs::ShardRequestManagerInterface *manager_;
+  const RequestRouterInterface *request_router_;
 };
 
 class VertexAccessor final {
@@ -78,10 +73,10 @@ class VertexAccessor final {
   using Label = msgs::Label;
   using VertexId = msgs::VertexId;
   VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
-                 const msgs::ShardRequestManagerInterface *manager);
+                 const RequestRouterInterface *request_router);
 
-  VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const msgs::ShardRequestManagerInterface *manager);
+  VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
-  VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const msgs::ShardRequestManagerInterface *manager);
+  VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
 
   [[nodiscard]] Label PrimaryLabel() const;
 
@@ -98,48 +93,11 @@ class VertexAccessor final {
 
   [[nodiscard]] msgs::Vertex GetVertex() const;
 
-  // Dummy function
   // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  [[nodiscard]] size_t CypherId() const { return 10; }
+  [[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
 
-  //  auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
+  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  //      -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
+  [[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
-  //    auto maybe_edges = impl_.InEdges(view, edge_types);
-  //    if (maybe_edges.HasError()) return maybe_edges.GetError();
-  //    return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
-  //  }
-  //
-  //  auto InEdges(storage::View view) const { return InEdges(view, {}); }
-  //
-  //  auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
-  //  const
-  //      -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
-  //    auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
-  //    if (maybe_edges.HasError()) return maybe_edges.GetError();
-  //    return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
-  //  }
-  //
-  //  auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
-  //      -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-  //    auto maybe_edges = impl_.OutEdges(view, edge_types);
-  //    if (maybe_edges.HasError()) return maybe_edges.GetError();
-  //    return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
-  //  }
-  //
-  //  auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
-  //
-  //  auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
-  //                const VertexAccessor &dest) const
-  //      -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-  //    auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
-  //    if (maybe_edges.HasError()) return maybe_edges.GetError();
-  //    return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
-  //  }
-
-  //  storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
-  //
-  //  storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
-  //
 
   friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
     return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
@@ -150,13 +108,9 @@ class VertexAccessor final {
  private:
   Vertex vertex;
   std::vector<std::pair<PropertyId, Value>> properties;
-  const msgs::ShardRequestManagerInterface *manager_;
+  const RequestRouterInterface *request_router_;
 };
 
-// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
-
-// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
-
 // Highly mocked interface. Won't work if used.
 class Path {
  public:
@@ -197,7 +151,14 @@ class Path {
   friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
   utils::MemoryResource *GetMemoryResource() { return mem; }
 
+  auto &vertices() { return vertices_; }
+  auto &edges() { return edges_; }
+  const auto &vertices() const { return vertices_; }
+  const auto &edges() const { return edges_; }
+
  private:
+  std::vector<VertexAccessor> vertices_;
+  std::vector<EdgeAccessor> edges_;
   utils::MemoryResource *mem = utils::NewDeleteResource();
 };
 }  // namespace memgraph::query::v2::accessors

src/query/v2/bindings/eval.hpp

@@ -21,29 +21,27 @@
 #include "query/v2/requests.hpp"
 #include "storage/v3/conversions.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/result.hpp"
 #include "storage/v3/view.hpp"
 
-namespace memgraph::msgs {
-class ShardRequestManagerInterface;
-} // namespace memgraph::msgs
-
 namespace memgraph::query::v2 {
 
-inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
+class RequestRouterInterface;
+
 namespace detail {
 class Callable {
  public:
-  auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
+  auto operator()(const storage::v3::PropertyValue &val) const {
-    return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
+    return storage::v3::PropertyToTypedValue<TypedValue>(val);
   };
-  auto operator()(const msgs::Value &val, memgraph::msgs::ShardRequestManagerInterface *manager) const {
+  auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
-    return ValueToTypedValue(val, manager);
+    return ValueToTypedValue(val, request_router);
   };
 };
 
 }  // namespace detail
-using ExpressionEvaluator = memgraph::expr::ExpressionEvaluator<
+using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
-    TypedValue, memgraph::query::v2::EvaluationContext, memgraph::msgs::ShardRequestManagerInterface, storage::v3::View,
+                                                      storage::v3::View, storage::v3::LabelId, msgs::Value,
-    storage::v3::LabelId, msgs::Value, detail::Callable, memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
+                                                      detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
 
 }  // namespace memgraph::query::v2

src/query/v2/bindings/frame.hpp

@@ -13,9 +13,10 @@
 
 #include "query/v2/bindings/bindings.hpp"
 
-#include "query/v2/bindings/typed_value.hpp"
 #include "expr/interpret/frame.hpp"
+#include "query/v2/bindings/typed_value.hpp"
 
 namespace memgraph::query::v2 {
-using Frame = memgraph::expr::Frame<TypedValue>;
+using Frame = memgraph::expr::Frame;
-} // namespace memgraph::query::v2
+using FrameWithValidity = memgraph::expr::FrameWithValidity;
+}  // namespace memgraph::query::v2

src/query/v2/common.hpp

@@ -20,7 +20,6 @@
 
 #include "query/v2/bindings/symbol.hpp"
 #include "query/v2/bindings/typed_value.hpp"
-#include "query/v2/db_accessor.hpp"
 #include "query/v2/exceptions.hpp"
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/path.hpp"
@@ -28,7 +27,6 @@
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
-#include "storage/v3/shard_operation_result.hpp"
 #include "storage/v3/view.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
@@ -82,39 +80,5 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
     throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
 }
 
-template <typename T>
-concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
-                                           const storage::v3::PropertyValue new_value) {
-  { accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
-};
-
-template <typename T>
-concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
-                                                      const storage::v3::PropertyValue new_value) {
-  {
-    accessor.SetPropertyAndValidate(key, new_value)
-    } -> std::same_as<storage::v3::ShardOperationResult<storage::v3::PropertyValue>>;
-};
-
-template <typename TRecordAccessor>
-concept RecordAccessor =
-    AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
-
-inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
-  switch (error) {
-    case storage::v3::Error::SERIALIZATION_ERROR:
-      throw TransactionSerializationException();
-    case storage::v3::Error::DELETED_OBJECT:
-      throw QueryRuntimeException("Trying to set properties on a deleted object.");
-    case storage::v3::Error::PROPERTIES_DISABLED:
-      throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
-    case storage::v3::Error::VERTEX_HAS_EDGES:
-    case storage::v3::Error::NONEXISTENT_OBJECT:
-    case storage::v3::Error::VERTEX_ALREADY_INSERTED:
-
-      throw QueryRuntimeException("Unexpected error when setting a property.");
-  }
-}
-
 int64_t QueryTimestamp();
 }  // namespace memgraph::query::v2

src/query/v2/context.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
 #include "query/v2/parameters.hpp"
 #include "query/v2/plan/profile.hpp"
 //#include "query/v2/trigger.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/async_timer.hpp"
 
 namespace memgraph::query::v2 {
@@ -60,27 +60,27 @@ struct EvaluationContext {
   mutable std::unordered_map<std::string, int64_t> counters;
 };
 
-inline std::vector<storage::v3::PropertyId> NamesToProperties(
+inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
-    const std::vector<std::string> &property_names, msgs::ShardRequestManagerInterface *shard_request_manager) {
+                                                              RequestRouterInterface *request_router) {
   std::vector<storage::v3::PropertyId> properties;
   // TODO Fix by using reference
   properties.reserve(property_names.size());
-  if (shard_request_manager != nullptr) {
+  if (request_router != nullptr) {
     for (const auto &name : property_names) {
-      properties.push_back(shard_request_manager->NameToProperty(name));
+      properties.push_back(request_router->NameToProperty(name));
     }
   }
   return properties;
 }
 
 inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
-                                                       msgs::ShardRequestManagerInterface *shard_request_manager) {
+                                                       RequestRouterInterface *request_router) {
   std::vector<storage::v3::LabelId> labels;
   labels.reserve(label_names.size());
   // TODO Fix by using reference
-  if (shard_request_manager != nullptr) {
+  if (request_router != nullptr) {
     for (const auto &name : label_names) {
-      labels.push_back(shard_request_manager->NameToLabel(name));
+      labels.push_back(request_router->NameToLabel(name));
     }
   }
   return labels;
@@ -97,7 +97,7 @@ struct ExecutionContext {
   plan::ProfilingStats *stats_root{nullptr};
   ExecutionStats execution_stats;
   utils::AsyncTimer timer;
-  msgs::ShardRequestManagerInterface *shard_request_manager{nullptr};
+  RequestRouterInterface *request_router{nullptr};
   IdAllocator *edge_ids_alloc;
 };
 

src/query/v2/conversions.hpp

@@ -12,12 +12,12 @@
 #pragma once
 #include "bindings/typed_value.hpp"
 #include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
 #include "query/v2/requests.hpp"
-#include "query/v2/shard_request_manager.hpp"
 
 namespace memgraph::query::v2 {
 
-inline TypedValue ValueToTypedValue(const msgs::Value &value, msgs::ShardRequestManagerInterface *manager) {
+inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
   using Value = msgs::Value;
   switch (value.type) {
     case Value::Type::Null:
@@ -35,7 +35,7 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, msgs::ShardRequest
       std::vector<TypedValue> dst;
       dst.reserve(lst.size());
       for (const auto &elem : lst) {
-        dst.push_back(ValueToTypedValue(elem, manager));
+        dst.push_back(ValueToTypedValue(elem, request_router));
       }
       return TypedValue(std::move(dst));
     }
@@ -43,19 +43,23 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, msgs::ShardRequest
       const auto &value_map = value.map_v;
       std::map<std::string, TypedValue> dst;
       for (const auto &[key, val] : value_map) {
-        dst[key] = ValueToTypedValue(val, manager);
+        dst[key] = ValueToTypedValue(val, request_router);
       }
       return TypedValue(std::move(dst));
     }
     case Value::Type::Vertex:
       return TypedValue(accessors::VertexAccessor(
-          value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, manager));
+          value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
     case Value::Type::Edge:
-      return TypedValue(accessors::EdgeAccessor(value.edge_v, manager));
+      return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
   }
   throw std::runtime_error("Incorrect type in conversion");
 }
 
+inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
+  return ValueToTypedValue(value, request_router);
+};
+
 inline msgs::Value TypedValueToValue(const TypedValue &value) {
   using Value = msgs::Value;
   switch (value.type()) {

src/query/v2/cypher_query_interpreter.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,12 @@
 
 #include "query/v2/cypher_query_interpreter.hpp"
 #include "query/v2/bindings/symbol_generator.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
+DEFINE_HIDDEN_bool(query_v2_cost_planner, true, "Use the cost-estimating query planner.");
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
+DEFINE_VALIDATED_int32(query_v2_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
                        FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
 
 namespace memgraph::query::v2 {
@@ -118,19 +118,19 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
 }
 
 std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
-                                             msgs::ShardRequestManagerInterface *shard_manager,
+                                             RequestRouterInterface *request_router,
                                              const std::vector<Identifier *> &predefined_identifiers) {
-  auto vertex_counts = plan::MakeVertexCountCache(shard_manager);
+  auto vertex_counts = plan::MakeVertexCountCache(request_router);
   auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
   auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
-  auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
+  auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_v2_cost_planner);
   return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
                                                  std::move(symbol_table));
 }
 
 std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
                                               const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
-                                              msgs::ShardRequestManagerInterface *shard_manager,
+                                              RequestRouterInterface *request_router,
                                               const std::vector<Identifier *> &predefined_identifiers) {
   std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
   if (plan_cache) {
@@ -146,7 +146,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
   }
 
   auto plan = std::make_shared<CachedPlan>(
-      MakeLogicalPlan(std::move(ast_storage), query, parameters, shard_manager, predefined_identifiers));
+      MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
   if (plan_cache_access) {
     plan_cache_access->insert({hash, plan});
   }

src/query/v2/cypher_query_interpreter.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,9 +22,9 @@
 #include "utils/timer.hpp"
 
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_bool(query_cost_planner);
+DECLARE_bool(query_v2_cost_planner);
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_int32(query_plan_cache_ttl);
+DECLARE_int32(query_v2_plan_cache_ttl);
 
 namespace memgraph::query::v2 {
 
@@ -58,7 +58,7 @@ class CachedPlan {
 
   bool IsExpired() const {
     // NOLINTNEXTLINE (modernize-use-nullptr)
-    return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
+    return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_v2_plan_cache_ttl);
   };
 
  private:
@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
 };
 
 std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
-                                             msgs::ShardRequestManagerInterface *shard_manager,
+                                             RequestRouterInterface *request_router,
                                              const std::vector<Identifier *> &predefined_identifiers);
 
 /**
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
  */
 std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
                                               const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
-                                              msgs::ShardRequestManagerInterface *shard_manager,
+                                              RequestRouterInterface *request_router,
                                               const std::vector<Identifier *> &predefined_identifiers = {});
 
 }  // namespace memgraph::query::v2

src/query/v2/db_accessor.hpp

@@ -23,7 +23,6 @@
 #include "storage/v3/key_store.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
-#include "storage/v3/shard_operation_result.hpp"
 
 ///////////////////////////////////////////////////////////
 // Our communication layer and query engine don't mix
@@ -37,7 +36,7 @@
 // This cannot be avoided by simple include orderings so we
 // simply undefine those macros as we're sure that libkrb5
 // won't and can't be used anywhere in the query engine.
-#include "storage/v3/storage.hpp"
+#include "storage/v3/shard.hpp"
 #include "utils/logging.hpp"
 #include "utils/result.hpp"
 
@@ -65,24 +64,11 @@ class EdgeAccessor final {
 
   auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
 
-  storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
+  storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
-                                                              storage::v3::PropertyId key) const {
+                                                                   storage::v3::PropertyId key) const {
     return impl_.GetProperty(key, view);
   }
 
-  storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
-                                                              const storage::v3::PropertyValue &value) {
-    return impl_.SetProperty(key, value);
-  }
-
-  storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
-    return SetProperty(key, storage::v3::PropertyValue());
-  }
-
-  storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
-    return impl_.ClearProperties();
-  }
-
   VertexAccessor To() const;
 
   VertexAccessor From() const;
@@ -114,53 +100,19 @@ class VertexAccessor final {
 
   auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
 
-  storage::v3::ShardOperationResult<bool> AddLabel(storage::v3::LabelId label) {
+  storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
-    return impl_.AddLabelAndValidate(label);
-  }
-
-  storage::v3::ShardOperationResult<bool> AddLabelAndValidate(storage::v3::LabelId label) {
-    return impl_.AddLabelAndValidate(label);
-  }
-
-  storage::v3::ShardOperationResult<bool> RemoveLabel(storage::v3::LabelId label) {
-    return impl_.RemoveLabelAndValidate(label);
-  }
-
-  storage::v3::ShardOperationResult<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
-    return impl_.RemoveLabelAndValidate(label);
-  }
-
-  storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
     return impl_.HasLabel(label, view);
   }
 
   auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
 
-  storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
+  storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
-                                                              storage::v3::PropertyId key) const {
+                                                                   storage::v3::PropertyId key) const {
     return impl_.GetProperty(key, view);
   }
 
-  storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
-                                                                            const storage::v3::PropertyValue &value) {
-    return impl_.SetPropertyAndValidate(key, value);
-  }
-
-  storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetPropertyAndValidate(
-      storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
-    return impl_.SetPropertyAndValidate(key, value);
-  }
-
-  storage::v3::ShardOperationResult<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
-    return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
-  }
-
-  storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
-    return impl_.ClearProperties();
-  }
-
   auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
     auto maybe_edges = impl_.InEdges(view, edge_types);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -170,7 +122,7 @@ class VertexAccessor final {
 
   auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
                const VertexAccessor &dest) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
     const auto dest_id = dest.impl_.Id(view).GetValue();
     auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
@@ -178,7 +130,7 @@ class VertexAccessor final {
   }
 
   auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
     auto maybe_edges = impl_.OutEdges(view, edge_types);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -188,16 +140,16 @@ class VertexAccessor final {
 
   auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
                 const VertexAccessor &dest) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
     const auto dest_id = dest.impl_.Id(view).GetValue();
     auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
   }
 
-  storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
+  storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
 
-  storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
+  storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
 
   // TODO(jbajic) Fix Remove Gid
   static int64_t CypherId() { return 1; }

src/query/v2/exceptions.hpp

@@ -224,12 +224,4 @@ class VersionInfoInMulticommandTxException : public QueryException {
       : QueryException("Version info query not allowed in multicommand transactions.") {}
 };
 
-/**
- * An exception for an illegal operation that violates schema
- */
-class SchemaViolationException : public QueryRuntimeException {
- public:
-  using QueryRuntimeException::QueryRuntimeException;
-};
-
 }  // namespace memgraph::query::v2

src/query/v2/frontend/ast/ast.lcp

@@ -20,11 +20,13 @@
 #include "query/v2/bindings/ast_visitor.hpp"
 #include "common/types.hpp"
 #include "query/v2/bindings/symbol.hpp"
-#include "query/v2/interpret/awesome_memgraph_functions.hpp"
+#include "functions/awesome_memgraph_functions.hpp"
 #include "query/v2/bindings/typed_value.hpp"
 #include "query/v2/db_accessor.hpp"
 #include "query/v2/path.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/typeinfo.hpp"
+#include "query/v2/conversions.hpp"
 
 cpp<#
 
@@ -836,13 +838,15 @@ cpp<#
               :slk-load (slk-load-ast-vector "Expression"))
    (function-name "std::string" :scope :public)
    (function "std::function<TypedValue(const TypedValue *, int64_t,
-                                       const FunctionContext &)>"
+                                       const functions::FunctionContext<RequestRouterInterface> &)>"
              :scope :public
              :dont-save t
              :clone :copy
              :slk-load (lambda (member)
                         #>cpp
-                        self->${member} = query::v2::NameToFunction(self->function_name_);
+                        self->${member} = functions::NameToFunction<TypedValue,
+                                            functions::FunctionContext<RequestRouterInterface>,
+                                            functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
                         cpp<#)))
   (:public
     #>cpp
@@ -865,7 +869,8 @@ cpp<#
              const std::vector<Expression *> &arguments)
         : arguments_(arguments),
           function_name_(function_name),
-          function_(NameToFunction(function_name_)) {
+          function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
+                                              functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
       if (!function_) {
         throw SemanticException("Function '{}' doesn't exist.", function_name);
       }

src/query/v2/interpret/awesome_memgraph_functions.hpp

@@ -20,12 +20,10 @@
 #include "storage/v3/view.hpp"
 #include "utils/memory.hpp"
 
-namespace memgraph::msgs {
-class ShardRequestManagerInterface;
-} // namespace memgraph::msgs
-
 namespace memgraph::query::v2 {
 
+class RequestRouterInterface;
+
 namespace {
 const char kStartsWith[] = "STARTSWITH";
 const char kEndsWith[] = "ENDSWITH";
@@ -34,9 +32,9 @@ const char kId[] = "ID";
 }  // namespace
 
 struct FunctionContext {
-  // TODO(kostasrim) consider optional here. ShardRequestManager does not exist on the storage.
+  // TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
   // DbAccessor *db_accessor;
-  msgs::ShardRequestManagerInterface *manager;
+  RequestRouterInterface *request_router;
   utils::MemoryResource *memory;
   int64_t timestamp;
   std::unordered_map<std::string, int64_t> *counters;

src/query/v2/interpreter.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,13 +41,13 @@
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/frontend/semantic/required_privileges.hpp"
 #include "query/v2/metadata.hpp"
+#include "query/v2/multiframe.hpp"
 #include "query/v2/plan/planner.hpp"
 #include "query/v2/plan/profile.hpp"
 #include "query/v2/plan/vertex_count_cache.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/shard.hpp"
-#include "storage/v3/storage.hpp"
 #include "utils/algorithm.hpp"
 #include "utils/csv_parsing.hpp"
 #include "utils/event_counter.hpp"
@@ -64,7 +64,11 @@
 #include "utils/tsc.hpp"
 #include "utils/variant_helpers.hpp"
 
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
+
 namespace EventCounter {
+
 extern Event ReadQuery;
 extern Event WriteQuery;
 extern Event ReadWriteQuery;
@@ -74,6 +78,7 @@ extern const Event LabelPropertyIndexCreated;
 
 extern const Event StreamsCreated;
 extern const Event TriggersCreated;
+
 }  // namespace EventCounter
 
 namespace memgraph::query::v2 {
@@ -144,18 +149,18 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
 /// @throw QueryRuntimeException if an error ocurred.
 
 Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters &parameters,
-                         msgs::ShardRequestManagerInterface *manager) {
+                         RequestRouterInterface *request_router) {
   // Empty frame for evaluation of password expression. This is OK since
   // password should be either null or string literal and it's evaluation
   // should not depend on frame.
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
   // the argument to Callback.
   evaluation_context.timestamp = QueryTimestamp();
   evaluation_context.parameters = parameters;
-  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
 
   std::string username = auth_query->user_;
   std::string rolename = auth_query->role_;
@@ -313,16 +318,16 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
 }
 
 Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters,
-                                InterpreterContext *interpreter_context, msgs::ShardRequestManagerInterface *manager,
+                                InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
                                 std::vector<Notification> *notifications) {
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
   // the argument to Callback.
   evaluation_context.timestamp = QueryTimestamp();
   evaluation_context.parameters = parameters;
-  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
 
   Callback callback;
   switch (repl_query->action_) {
@@ -449,8 +454,8 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
 }
 
 Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &parameters,
-                            msgs::ShardRequestManagerInterface *manager) {
+                            RequestRouterInterface *request_router) {
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   // TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -459,7 +464,7 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &param
       std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
           .count();
   evaluation_context.parameters = parameters;
-  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
 
   Callback callback;
   switch (setting_query->action_) {
@@ -650,17 +655,21 @@ struct PullPlanVector {
 struct PullPlan {
   explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
                     DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
-                    msgs::ShardRequestManagerInterface *shard_request_manager = nullptr,
+                    RequestRouterInterface *request_router = nullptr,
                     //                    TriggerContextCollector *trigger_context_collector = nullptr,
                     std::optional<size_t> memory_limit = {});
   std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
                                                         const std::vector<Symbol> &output_symbols,
                                                         std::map<std::string, TypedValue> *summary);
+  std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
+                                                                const std::vector<Symbol> &output_symbols,
+                                                                std::map<std::string, TypedValue> *summary);
 
  private:
   std::shared_ptr<CachedPlan> plan_ = nullptr;
   plan::UniqueCursorPtr cursor_ = nullptr;
-  expr::Frame<TypedValue> frame_;
+  expr::FrameWithValidity frame_;
+  MultiFrame multi_frame_;
   ExecutionContext ctx_;
   std::optional<size_t> memory_limit_;
 
@@ -680,29 +689,137 @@ struct PullPlan {
 
 PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
                    DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
-                   msgs::ShardRequestManagerInterface *shard_request_manager, const std::optional<size_t> memory_limit)
+                   RequestRouterInterface *request_router, const std::optional<size_t> memory_limit)
     : plan_(plan),
       cursor_(plan->plan().MakeCursor(execution_memory)),
       frame_(plan->symbol_table().max_position(), execution_memory),
+      multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
       memory_limit_(memory_limit) {
   ctx_.db_accessor = dba;
   ctx_.symbol_table = plan->symbol_table();
   ctx_.evaluation_context.timestamp = QueryTimestamp();
   ctx_.evaluation_context.parameters = parameters;
-  ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, shard_request_manager);
+  ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, request_router);
-  ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, shard_request_manager);
+  ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, request_router);
   if (interpreter_context->config.execution_timeout_sec > 0) {
     ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
   }
   ctx_.is_shutting_down = &interpreter_context->is_shutting_down;
   ctx_.is_profile_query = is_profile_query;
-  ctx_.shard_request_manager = shard_request_manager;
+  ctx_.request_router = request_router;
   ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
 }
+std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
+                                                                        const std::vector<Symbol> &output_symbols,
+                                                                        std::map<std::string, TypedValue> *summary) {
+  // Set up temporary memory for a single Pull. Initial memory comes from the
+  // stack. 256 KiB should fit on the stack and should be more than enough for a
+  // single `Pull`.
+  MG_ASSERT(!n.has_value(), "should pull all!");
+  static constexpr size_t stack_size = 256UL * 1024UL;
+  char stack_data[stack_size];
+  utils::ResourceWithOutOfMemoryException resource_with_exception;
+  utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
+  // We can throw on every query because a simple queries for deleting will use only
+  // the stack allocated buffer.
+  // Also, we want to throw only when the query engine requests more memory and not the storage
+  // so we add the exception to the allocator.
+  // TODO (mferencevic): Tune the parameters accordingly.
+  utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
+  std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
+
+  if (memory_limit_) {
+    maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
+    ctx_.evaluation_context.memory = &*maybe_limited_resource;
+  } else {
+    ctx_.evaluation_context.memory = &pool_memory;
+  }
+
+  // Returns true if a result was pulled.
+  const auto pull_result = [&]() -> bool { return cursor_->PullMultiple(multi_frame_, ctx_); };
+
+  const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
+    // TODO: The streamed values should also probably use the above memory.
+    std::vector<TypedValue> values;
+    values.reserve(output_symbols.size());
+
+    for (const auto &symbol : output_symbols) {
+      values.emplace_back(frame[symbol]);
+    }
+
+    stream->Result(values);
+  };
+
+  // Get the execution time of all possible result pulls and streams.
+  utils::Timer timer;
+
+  int i = 0;
+  if (has_unsent_results_ && !output_symbols.empty()) {
+    // stream unsent results from previous pull
+    for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
+      stream_values(frame);
+      frame.MakeInvalid();
+      ++i;
+      if (i == n) {
+        break;
+      }
+    }
+  }
+
+  for (; !n || i < n;) {
+    if (!pull_result()) {
+      break;
+    }
+
+    if (!output_symbols.empty()) {
+      for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
+        stream_values(frame);
+        frame.MakeInvalid();
+        ++i;
+        if (i == n) {
+          break;
+        }
+      }
+    } else {
+      multi_frame_.MakeAllFramesInvalid();
+    }
+  }
+
+  // If we finished because we streamed the requested n results,
+  // we try to pull the next result to see if there is more.
+  // If there is additional result, we leave the pulled result in the frame
+  // and set the flag to true.
+  has_unsent_results_ = i == n && pull_result();
+
+  execution_time_ += timer.Elapsed();
+
+  if (has_unsent_results_) {
+    return std::nullopt;
+  }
+  summary->insert_or_assign("plan_execution_time", execution_time_.count());
+  // We are finished with pulling all the data, therefore we can send any
+  // metadata about the results i.e. notifications and statistics
+  const bool is_any_counter_set =
+      std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
+                  [](const auto &counter) { return counter > 0; });
+  if (is_any_counter_set) {
+    std::map<std::string, TypedValue> stats;
+    for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
+      stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
+    }
+    summary->insert_or_assign("stats", std::move(stats));
+  }
+  cursor_->Shutdown();
+  ctx_.profile_execution_time = execution_time_;
+  return GetStatsWithTotalTime(ctx_);
+}
 
 std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
                                                                 const std::vector<Symbol> &output_symbols,
                                                                 std::map<std::string, TypedValue> *summary) {
+  if (FLAGS_use_multi_frame) {
+    return PullMultiple(stream, n, output_symbols, summary);
+  }
   // Set up temporary memory for a single Pull. Initial memory comes from the
   // stack. 256 KiB should fit on the stack and should be more than enough for a
   // single `Pull`.
@@ -794,30 +911,24 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
 
 InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
                                        const std::filesystem::path & /*data_directory*/,
-                                       io::Io<io::local_transport::LocalTransport> io,
+                                       std::unique_ptr<RequestRouterFactory> request_router_factory,
                                        coordinator::Address coordinator_addr)
-    : db(db), config(config), io{std::move(io)}, coordinator_address{coordinator_addr} {}
+    : db(db),
+      config(config),
+      coordinator_address{coordinator_addr},
+      request_router_factory_{std::move(request_router_factory)} {}
 
 Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
   MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
-  auto query_io = interpreter_context_->io.ForkLocal();
+
-  shard_request_manager_ = std::make_unique<msgs::ShardRequestManager<io::local_transport::LocalTransport>>(
+  request_router_ =
-      coordinator::CoordinatorClient<io::local_transport::LocalTransport>(
+      interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address);
-          query_io, interpreter_context_->coordinator_address, std::vector{interpreter_context_->coordinator_address}),
+
-      std::move(query_io));
   // Get edge ids
-  coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
+  const auto edge_ids_alloc_min_max_pair =
-  io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
+      request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address);
-  ww.operation = requests;
+  if (edge_ids_alloc_min_max_pair) {
-  auto resp = interpreter_context_->io
+    interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second};
-                  .Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
-                           io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(
-                      interpreter_context_->coordinator_address, ww)
-                  .Wait();
-  if (resp.HasValue()) {
-    const auto alloc_edge_id_reps =
-        std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
-    interpreter_context_->edge_ids_alloc = {alloc_edge_id_reps.low, alloc_edge_id_reps.high};
   }
 }
 
@@ -878,17 +989,16 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
 PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
                                  InterpreterContext *interpreter_context, DbAccessor *dba,
                                  utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
-                                 msgs::ShardRequestManagerInterface *shard_request_manager) {
+                                 RequestRouterInterface *request_router) {
   //                                 TriggerContextCollector *trigger_context_collector = nullptr) {
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
 
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   evaluation_context.timestamp = QueryTimestamp();
   evaluation_context.parameters = parsed_query.parameters;
-  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
-                                storage::v3::View::OLD);
   const auto memory_limit =
       expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
   if (memory_limit) {
@@ -903,9 +1013,9 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
         "convert the parsed row values to the appropriate type. This can be done using the built-in "
         "conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
   }
-  auto plan = CypherQueryToPlan(
+  auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
-      parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, parsed_query.parameters,
+                                parsed_query.parameters,
-      parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, shard_request_manager);
+                                parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
 
   summary->insert_or_assign("cost_estimate", plan->cost());
   auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -924,7 +1034,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
         utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
   }
   auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
-                                              execution_memory, shard_request_manager, memory_limit);
+                                              execution_memory, request_router, memory_limit);
   //                                              execution_memory, trigger_context_collector, memory_limit);
   return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
                        [pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
@@ -938,8 +1048,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
 }
 
 PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
-                                  InterpreterContext *interpreter_context,
+                                  InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
-                                  msgs::ShardRequestManagerInterface *shard_request_manager,
                                   utils::MemoryResource *execution_memory) {
   const std::string kExplainQueryStart = "explain ";
   MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
@@ -958,20 +1067,20 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
   MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
 
-  auto cypher_query_plan = CypherQueryToPlan(
+  auto cypher_query_plan =
-      parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
+      CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
-      parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
+                        cypher_query, parsed_inner_query.parameters,
-      shard_request_manager);
+                        parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
 
   std::stringstream printed_plan;
-  plan::PrettyPrint(*shard_request_manager, &cypher_query_plan->plan(), &printed_plan);
+  plan::PrettyPrint(*request_router, &cypher_query_plan->plan(), &printed_plan);
 
   std::vector<std::vector<TypedValue>> printed_plan_rows;
   for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) {
     printed_plan_rows.push_back(std::vector<TypedValue>{TypedValue(row)});
   }
 
-  summary->insert_or_assign("explain", plan::PlanToJson(*shard_request_manager, &cypher_query_plan->plan()).dump());
+  summary->insert_or_assign("explain", plan::PlanToJson(*request_router, &cypher_query_plan->plan()).dump());
 
   return PreparedQuery{{"QUERY PLAN"},
                        std::move(parsed_query.required_privileges),
@@ -988,7 +1097,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
 PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                   std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
                                   DbAccessor *dba, utils::MemoryResource *execution_memory,
-                                  msgs::ShardRequestManagerInterface *shard_request_manager = nullptr) {
+                                  RequestRouterInterface *request_router = nullptr) {
   const std::string kProfileQueryStart = "profile ";
 
   MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -1026,27 +1135,26 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
 
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
   MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
-  expr::Frame<TypedValue> frame(0);
+  expr::Frame frame(0);
   SymbolTable symbol_table;
   EvaluationContext evaluation_context;
   evaluation_context.timestamp = QueryTimestamp();
   evaluation_context.parameters = parsed_inner_query.parameters;
-  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
-                                storage::v3::View::OLD);
   const auto memory_limit =
       expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
 
-  auto cypher_query_plan = CypherQueryToPlan(
+  auto cypher_query_plan =
-      parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
+      CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
-      parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
+                        cypher_query, parsed_inner_query.parameters,
-      shard_request_manager);
+                        parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
   auto rw_type_checker = plan::ReadWriteTypeChecker();
   rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
 
   return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME", "CUSTOM DATA"},
                        std::move(parsed_query.required_privileges),
                        [plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
-                        summary, dba, interpreter_context, execution_memory, memory_limit, shard_request_manager,
+                        summary, dba, interpreter_context, execution_memory, memory_limit, request_router,
                         // We want to execute the query we are profiling lazily, so we delay
                         // the construction of the corresponding context.
                         stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
@@ -1055,7 +1163,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
                          // No output symbols are given so that nothing is streamed.
                          if (!stats_and_total_time) {
                            stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
-                                                           execution_memory, shard_request_manager, memory_limit)
+                                                           execution_memory, request_router, memory_limit)
                                                       .Pull(stream, {}, {}, summary);
                            pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
                          }
@@ -1182,14 +1290,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
 PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
                                DbAccessor *dba, utils::MemoryResource *execution_memory,
-                               msgs::ShardRequestManagerInterface *manager) {
+                               RequestRouterInterface *request_router) {
   if (in_explicit_transaction) {
     throw UserModificationInMulticommandTxException();
   }
 
   auto *auth_query = utils::Downcast<AuthQuery>(parsed_query.query);
 
-  auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, manager);
+  auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, request_router);
 
   SymbolTable symbol_table;
   std::vector<Symbol> output_symbols;
@@ -1218,14 +1326,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
 
 PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
                                       std::vector<Notification> *notifications, InterpreterContext *interpreter_context,
-                                      msgs::ShardRequestManagerInterface *manager) {
+                                      RequestRouterInterface *request_router) {
   if (in_explicit_transaction) {
     throw ReplicationModificationInMulticommandTxException();
   }
 
   auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
-  auto callback =
+  auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context,
-      HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, manager, notifications);
+                                         request_router, notifications);
 
   return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
                        [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1314,14 +1422,14 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
 }
 
 PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
-                                  msgs::ShardRequestManagerInterface *manager) {
+                                  RequestRouterInterface *request_router) {
   if (in_explicit_transaction) {
     throw SettingConfigInMulticommandTxException{};
   }
 
   auto *setting_query = utils::Downcast<SettingQuery>(parsed_query.query);
   MG_ASSERT(setting_query);
-  auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, manager);
+  auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, request_router);
 
   return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
                        [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1518,7 +1626,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
     if (!in_explicit_transaction_ &&
         (utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
          utils::Downcast<ProfileQuery>(parsed_query.query))) {
-      shard_request_manager_->StartTransaction();
+      request_router_->StartTransaction();
     }
 
     utils::Timer planning_timer;
@@ -1527,14 +1635,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
     if (utils::Downcast<CypherQuery>(parsed_query.query)) {
       prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
                                           &*execution_db_accessor_, &query_execution->execution_memory,
-                                          &query_execution->notifications, shard_request_manager_.get());
+                                          &query_execution->notifications, request_router_.get());
     } else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
       prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
-                                           &*shard_request_manager_, &query_execution->execution_memory_with_exception);
+                                           &*request_router_, &query_execution->execution_memory_with_exception);
     } else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
-      prepared_query = PrepareProfileQuery(
+      prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
-          std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
+                                           interpreter_context_, &*execution_db_accessor_,
-          &*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
+                                           &query_execution->execution_memory_with_exception, request_router_.get());
     } else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
       prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
                                         &query_execution->execution_memory);
@@ -1542,9 +1650,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
       prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_,
                                          &query_execution->notifications, interpreter_context_);
     } else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
-      prepared_query = PrepareAuthQuery(
+      prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
-          std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
+                                        interpreter_context_, &*execution_db_accessor_,
-          &*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
+                                        &query_execution->execution_memory_with_exception, request_router_.get());
     } else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
       prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
                                         interpreter_context_, interpreter_context_->db,
@@ -1555,7 +1663,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
     } else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
       prepared_query =
           PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
-                                  interpreter_context_, shard_request_manager_.get());
+                                  interpreter_context_, request_router_.get());
     } else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
       prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
                                             &*execution_db_accessor_);
@@ -1572,8 +1680,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
       prepared_query =
           PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
     } else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
-      prepared_query =
+      prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, request_router_.get());
-          PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, shard_request_manager_.get());
     } else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
       prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
     } else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
@@ -1614,7 +1721,7 @@ void Interpreter::Commit() {
   // For now, we will not check if there are some unfinished queries.
   // We should document clearly that all results should be pulled to complete
   // a query.
-  shard_request_manager_->Commit();
+  request_router_->Commit();
   if (!db_accessor_) return;
 
   const auto reset_necessary_members = [this]() {

src/query/v2/interpreter.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -16,7 +16,6 @@
 
 #include "coordinator/coordinator.hpp"
 #include "coordinator/coordinator_client.hpp"
-#include "io/local_transport/local_transport.hpp"
 #include "io/transport.hpp"
 #include "query/v2/auth_checker.hpp"
 #include "query/v2/bindings/cypher_main_visitor.hpp"
@@ -172,7 +171,8 @@ struct PreparedQuery {
 struct InterpreterContext {
   explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
                               const std::filesystem::path &data_directory,
-                              io::Io<io::local_transport::LocalTransport> io, coordinator::Address coordinator_addr);
+                              std::unique_ptr<RequestRouterFactory> request_router_factory,
+                              coordinator::Address coordinator_addr);
 
   storage::v3::Shard *db;
 
@@ -188,26 +188,24 @@ struct InterpreterContext {
   const InterpreterConfig config;
   IdAllocator edge_ids_alloc;
 
-  // TODO (antaljanosbenjamin) Figure out an abstraction for io::Io to make it possible to construct an interpreter
-  // context with a simulator transport without templatizing it.
-  io::Io<io::local_transport::LocalTransport> io;
   coordinator::Address coordinator_address;
+  std::unique_ptr<RequestRouterFactory> request_router_factory_;
 
   storage::v3::LabelId NameToLabelId(std::string_view label_name) {
-    return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
+    return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
   }
 
   storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
-    return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
+    return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
   }
 
   storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
-    return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
+    return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
   }
 
  private:
   // TODO Replace with local map of labels, properties and edge type ids
-  storage::v3::NameIdMapper query_id_mapper;
+  storage::v3::NameIdMapper query_id_mapper_;
 };
 
 /// Function that is used to tell all active interpreters that they should stop
@@ -296,13 +294,16 @@ class Interpreter final {
    */
   void Abort();
 
-  const msgs::ShardRequestManagerInterface *GetShardRequestManager() const { return shard_request_manager_.get(); }
+  const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
+  void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
+    request_router_->InstallSimulatorTicker(tick_simulator);
+  }
 
  private:
   struct QueryExecution {
-    std::optional<PreparedQuery> prepared_query;
     utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
     utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
+    std::optional<PreparedQuery> prepared_query;
 
     std::map<std::string, TypedValue> summary;
     std::vector<Notification> notifications;
@@ -342,7 +343,7 @@ class Interpreter final {
   // move this unique_ptr into a shared_ptr.
   std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
   std::optional<DbAccessor> execution_db_accessor_;
-  std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager_;
+  std::unique_ptr<RequestRouterInterface> request_router_;
   bool in_explicit_transaction_{false};
   bool expect_rollback_{false};
 

src/query/v2/multiframe.cpp

@@ -0,0 +1,153 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/v2/multiframe.hpp"
+
+#include <algorithm>
+#include <iterator>
+
+#include "query/v2/bindings/frame.hpp"
+#include "utils/pmr/vector.hpp"
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
+
+namespace memgraph::query::v2 {
+
+static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
+static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
+static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
+static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
+
+MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
+    : frames_(utils::pmr::vector<FrameWithValidity>(
+          number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
+  MG_ASSERT(number_of_frames > 0);
+}
+
+MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
+
+FrameWithValidity &MultiFrame::GetFirstFrame() {
+  MG_ASSERT(!frames_.empty());
+  return frames_.front();
+}
+
+void MultiFrame::MakeAllFramesInvalid() noexcept {
+  std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
+}
+
+bool MultiFrame::HasValidFrame() const noexcept {
+  return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
+}
+
+bool MultiFrame::HasInvalidFrame() const noexcept {
+  return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
+}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+void MultiFrame::DefragmentValidFrames() noexcept {
+  static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
+  static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
+  auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
+  auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
+  while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
+    std::swap(*first_invalid_frame, *following_first_valid);
+    first_invalid_frame++;
+    first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
+    following_first_valid++;
+    following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
+  }
+}
+
+ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
+
+ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
+
+ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
+
+InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
+
+ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
+  /*
+  From: https://en.cppreference.com/w/cpp/algorithm/find
+  Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
+  find_if searches for an element for which predicate p returns true
+  Return value
+    Iterator to the first element satisfying the condition or last if no such element is found.
+
+  -> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
+  */
+  auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
+                         [](const auto &frame) { return !frame.IsValid(); });
+  after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
+}
+
+ValidFramesReader::Iterator ValidFramesReader::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0]};
+  }
+  return end();
+}
+
+ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
+
+ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+ValidFramesModifier::Iterator ValidFramesModifier::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesModifier::Iterator ValidFramesModifier::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+ValidFramesConsumer::~ValidFramesConsumer() noexcept {
+  // TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
+  // cost too much to store it
+  multiframe_->DefragmentValidFrames();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
+  for (auto &frame : multiframe_->frames_) {
+    if (!frame.IsValid()) {
+      return Iterator{&frame};
+    }
+  }
+  return end();
+}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
+}
+
+}  // namespace memgraph::query::v2

src/query/v2/multiframe.hpp

@@ -0,0 +1,308 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <iterator>
+
+#include <gflags/gflags.h>
+
+#include "query/v2/bindings/frame.hpp"
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint64(default_multi_frame_size);
+
+namespace memgraph::query::v2 {
+
+class ValidFramesConsumer;
+class ValidFramesModifier;
+class ValidFramesReader;
+class InvalidFramesPopulator;
+
+class MultiFrame {
+ public:
+  friend class ValidFramesConsumer;
+  friend class ValidFramesModifier;
+  friend class ValidFramesReader;
+  friend class InvalidFramesPopulator;
+
+  MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
+  ~MultiFrame() = default;
+
+  // Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
+  MultiFrame(const MultiFrame &other);
+  MultiFrame(MultiFrame &&other) noexcept;
+  MultiFrame &operator=(const MultiFrame &other) = delete;
+  MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame nor its content with this implementation.
+   */
+  ValidFramesReader GetValidFramesReader();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame with this implementation. One can modify its content.
+   */
+  ValidFramesModifier GetValidFramesModifier();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can modify the validity of the Frame with this implementation.
+   * If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
+   * this is faster.
+   */
+  ValidFramesConsumer GetValidFramesConsumer();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
+   * invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
+   * the Frame with this implementation.
+   */
+  InvalidFramesPopulator GetInvalidFramesPopulator();
+
+  /**
+   * Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
+   * using the iterators instead.
+   * The Frame can be valid or invalid.
+   */
+  FrameWithValidity &GetFirstFrame();
+
+  void MakeAllFramesInvalid() noexcept;
+
+  bool HasValidFrame() const noexcept;
+  bool HasInvalidFrame() const noexcept;
+
+  inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
+
+ private:
+  void DefragmentValidFrames() noexcept;
+
+  utils::pmr::vector<FrameWithValidity> frames_;
+};
+
+class ValidFramesReader {
+ public:
+  explicit ValidFramesReader(MultiFrame &multiframe);
+
+  ~ValidFramesReader() = default;
+  ValidFramesReader(const ValidFramesReader &other) = delete;
+  ValidFramesReader(ValidFramesReader &&other) noexcept = default;
+  ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
+  ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = const Frame;
+    using pointer = value_type *;
+    using reference = const Frame &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ptr_++;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  FrameWithValidity *after_last_valid_frame_;
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesModifier {
+ public:
+  explicit ValidFramesModifier(MultiFrame &multiframe);
+
+  ~ValidFramesModifier() = default;
+  ValidFramesModifier(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
+  ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = Frame;
+    using pointer = value_type *;
+    using reference = Frame &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    // Prefix increment
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesModifier *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesConsumer {
+ public:
+  explicit ValidFramesConsumer(MultiFrame &multiframe);
+
+  ~ValidFramesConsumer() noexcept;
+  ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
+  ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesConsumer *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class InvalidFramesPopulator {
+ public:
+  explicit InvalidFramesPopulator(MultiFrame &multiframe);
+  ~InvalidFramesPopulator() = default;
+
+  InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
+  InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_->MakeValid();
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++ptr_;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+}  // namespace memgraph::query::v2

src/query/v2/plan/cost_estimator.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -149,8 +149,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
     return true;
   }
 
-  // TODO: Cost estimate ScanAllById?
-
 // For the given op first increments the cardinality and then cost.
 #define POST_VISIT_CARD_FIRST(NAME)     \
   bool PostVisit(NAME &) override {     \

src/query/v2/plan/operator.lcp

@@ -28,6 +28,7 @@
 #include "query/v2/bindings/typed_value.hpp"
 #include "query/v2/bindings/frame.hpp"
 #include "query/v2/bindings/symbol_table.hpp"
+#include "query/v2/multiframe.hpp"
 #include "storage/v3/id_types.hpp"
 #include "utils/bound.hpp"
 #include "utils/fnv.hpp"
@@ -71,6 +72,24 @@ class Cursor {
   /// @throws QueryRuntimeException if something went wrong with execution
   virtual bool Pull(Frame &, ExecutionContext &) = 0;
 
+  /// Run an iteration of a @c LogicalOperator with MultiFrame.
+  ///
+  /// Since operators may be chained, the iteration may pull results from
+  /// multiple operators.
+  ///
+  /// @param MultiFrame May be read from or written to while performing the
+  ///     iteration.
+  /// @param ExecutionContext Used to get the position of symbols in frame and
+  ///     other information.
+  /// @return True if the operator was able to populate at least one Frame on the MultiFrame,
+  ///     thus if an operator returns true, that means there is at least one valid Frame in the
+  ///     MultiFrame.
+  ///
+  /// @throws QueryRuntimeException if something went wrong with execution
+  virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
+
+  virtual void PushDown(const MultiFrame&) { MG_ASSERT(false, "PushDownIsNotImplemented"); }
+
   /// Resets the Cursor to its initial state.
   virtual void Reset() = 0;
 
@@ -110,7 +129,7 @@ class ScanAllByLabel;
 class ScanAllByLabelPropertyRange;
 class ScanAllByLabelPropertyValue;
 class ScanAllByLabelProperty;
-class ScanAllById;
+class ScanByPrimaryKey;
 class Expand;
 class ExpandVariable;
 class ConstructNamedPath;
@@ -141,7 +160,7 @@ class Foreach;
 using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
     Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
     ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
-    ScanAllByLabelProperty, ScanAllById,
+    ScanAllByLabelProperty, ScanByPrimaryKey,
     Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
     SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
     EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
@@ -332,11 +351,14 @@ and false on every following Pull.")
    class OnceCursor : public Cursor {
     public:
      OnceCursor() {}
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
+     void PushDown(const MultiFrame&) override;
      bool Pull(Frame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
     private:
+     std::optional<MultiFrame> pushed_down_multi_frame_;
      bool did_pull_{false};
    };
    cpp<#)
@@ -841,19 +863,21 @@ given label and property.
   (:serialize (:slk))
   (:clone))
 
-
+(lcp:define-class scan-by-primary-key (scan-all)
-
+  ((label "::storage::v3::LabelId" :scope :public)
-(lcp:define-class scan-all-by-id (scan-all)
+    (primary-key "std::vector<Expression*>" :scope :public)
-  ((expression "Expression *" :scope :public
+    (expression "Expression *" :scope :public
                :slk-save #'slk-save-ast-pointer
                :slk-load (slk-load-ast-pointer "Expression")))
   (:documentation
-    "ScanAll producing a single node with ID equal to evaluated expression")
+    "ScanAll producing a single node with specified by the label and primary key")
   (:public
     #>cpp
-    ScanAllById() {}
+    ScanByPrimaryKey() {}
-    ScanAllById(const std::shared_ptr<LogicalOperator> &input,
+    ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
-                Symbol output_symbol, Expression *expression,
+                Symbol output_symbol,
+                storage::v3::LabelId label,
+                std::vector<query::v2::Expression*> primary_key,
                 storage::v3::View view = storage::v3::View::OLD);
 
     bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
@@ -1156,6 +1180,7 @@ a boolean value.")
     public:
      FilterCursor(const Filter &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1207,6 +1232,7 @@ RETURN clause) the Produce's pull succeeds exactly once.")
     public:
      ProduceCursor(const Produce &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1254,6 +1280,7 @@ Has a flag for using DETACH DELETE when deleting vertices.")
     public:
      DeleteCursor(const Delete &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1547,6 +1574,7 @@ edge lists).")
      EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
                                 utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1943,27 +1971,6 @@ and returns true, once.")
      input_ = input;
    }
    cpp<#)
-  (:private
-   #>cpp
-   class OptionalCursor : public Cursor {
-    public:
-     OptionalCursor(const Optional &, utils::MemoryResource *);
-     bool Pull(Frame &, ExecutionContext &) override;
-     void Shutdown() override;
-     void Reset() override;
-
-    private:
-     const Optional &self_;
-     const UniqueCursorPtr input_cursor_;
-     const UniqueCursorPtr optional_cursor_;
-     // indicates if the next Pull from this cursor should
-     // perform a Pull from the input_cursor_
-     // this is true when:
-     //  - first pulling from this Cursor
-     //  - previous Pull from this cursor exhausted the optional_cursor_
-     bool pull_input_{true};
-   };
-   cpp<#)
   (:serialize (:slk))
   (:clone))
 

src/query/v2/plan/preprocess.cpp

@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
   auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
     auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
     if (!id_fun) return false;
-    if (id_fun->function_name_ != kId) return false;
+    if (id_fun->function_name_ != functions::kId) return false;
     if (id_fun->arguments_.size() != 1U) return false;
     auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
     if (!ident) return false;

src/query/v2/plan/pretty_print.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,13 +14,13 @@
 
 #include "query/v2/bindings/pretty_print.hpp"
 #include "query/v2/db_accessor.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/string.hpp"
 
 namespace memgraph::query::v2::plan {
 
-PlanPrinter::PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out)
+PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
-    : request_manager_(request_manager), out_(out) {}
+    : request_router_(request_router), out_(out) {}
 
 #define PRE_VISIT(TOp)                                   \
   bool PlanPrinter::PreVisit(TOp &) {                    \
@@ -34,7 +34,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
   WithPrintLn([&](auto &out) {
     out << "* CreateExpand (" << op.input_symbol_.name() << ")"
         << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
-        << op.edge_info_.symbol.name() << ":" << request_manager_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
+        << op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
         << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
         << op.node_info_.symbol.name() << ")";
   });
@@ -54,7 +54,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabel"
-        << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
   });
   return true;
 }
@@ -62,8 +62,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelPropertyValue"
-        << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
-        << request_manager_->PropertyToName(op.property_) << "})";
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
@@ -71,8 +71,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelPropertyRange"
-        << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
-        << request_manager_->PropertyToName(op.property_) << "})";
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
@@ -80,16 +80,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelProperty"
-        << " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
-        << request_manager_->PropertyToName(op.property_) << "})";
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
 
-bool PlanPrinter::PreVisit(ScanAllById &op) {
+bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
   WithPrintLn([&](auto &out) {
-    out << "* ScanAllById"
+    out << "* ScanByPrimaryKey"
-        << " (" << op.output_symbol_.name() << ")";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
   });
   return true;
 }
@@ -100,7 +100,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
           << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
           << op.common_.edge_symbol.name();
     utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
-      stream << ":" << request_manager_->EdgeTypeToName(edge_type);
+      stream << ":" << request_router_->EdgeTypeToName(edge_type);
     });
     *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
           << op.common_.node_symbol.name() << ")";
@@ -129,7 +129,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
           << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
           << op.common_.edge_symbol.name();
     utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
-      stream << ":" << request_manager_->EdgeTypeToName(edge_type);
+      stream << ":" << request_router_->EdgeTypeToName(edge_type);
     });
     *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
           << op.common_.node_symbol.name() << ")";
@@ -263,15 +263,14 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
   --depth_;
 }
 
-void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
+void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
-                 std::ostream *out) {
+  PlanPrinter printer(&request_router, out);
-  PlanPrinter printer(&request_manager, out);
   // FIXME(mtomic): We should make visitors that take const arguments.
   const_cast<LogicalOperator *>(plan_root)->Accept(printer);
 }
 
-nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
+nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
-  impl::PlanToJsonVisitor visitor(&request_manager);
+  impl::PlanToJsonVisitor visitor(&request_router);
   // FIXME(mtomic): We should make visitors that take const arguments.
   const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
   return visitor.output();
@@ -349,16 +348,16 @@ json ToJson(const utils::Bound<Expression *> &bound) {
 
 json ToJson(const Symbol &symbol) { return symbol.name(); }
 
-json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager) {
+json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
-  return request_manager.EdgeTypeToName(edge_type);
+  return request_router.EdgeTypeToName(edge_type);
 }
 
-json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager) {
+json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
-  return request_manager.LabelToName(label);
+  return request_router.LabelToName(label);
 }
 
-json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager) {
+json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
-  return request_manager.PropertyToName(property);
+  return request_router.PropertyToName(property);
 }
 
 json ToJson(NamedExpression *nexpr) {
@@ -369,29 +368,29 @@ json ToJson(NamedExpression *nexpr) {
 }
 
 json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
-            const msgs::ShardRequestManagerInterface &request_manager) {
+            const RequestRouterInterface &request_router) {
   json json;
   for (const auto &prop_pair : properties) {
-    json.emplace(ToJson(prop_pair.first, request_manager), ToJson(prop_pair.second));
+    json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
   }
   return json;
 }
 
-json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager) {
+json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
   json self;
   self["symbol"] = ToJson(node_info.symbol);
-  self["labels"] = ToJson(node_info.labels, request_manager);
+  self["labels"] = ToJson(node_info.labels, request_router);
   const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
-  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
+  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
   return self;
 }
 
-json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager) {
+json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
   json self;
   self["symbol"] = ToJson(edge_info.symbol);
   const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
-  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
+  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
-  self["edge_type"] = ToJson(edge_info.edge_type, request_manager);
+  self["edge_type"] = ToJson(edge_info.edge_type, request_router);
   self["direction"] = ToString(edge_info.direction);
   return self;
 }
@@ -433,7 +432,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
   json self;
   self["name"] = "ScanAllByLabel";
-  self["label"] = ToJson(op.label_, *request_manager_);
+  self["label"] = ToJson(op.label_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
   op.input_->Accept(*this);
@@ -446,8 +445,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
   json self;
   self["name"] = "ScanAllByLabelPropertyRange";
-  self["label"] = ToJson(op.label_, *request_manager_);
+  self["label"] = ToJson(op.label_, *request_router_);
-  self["property"] = ToJson(op.property_, *request_manager_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
   self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
   self["output_symbol"] = ToJson(op.output_symbol_);
@@ -462,8 +461,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
   json self;
   self["name"] = "ScanAllByLabelPropertyValue";
-  self["label"] = ToJson(op.label_, *request_manager_);
+  self["label"] = ToJson(op.label_, *request_router_);
-  self["property"] = ToJson(op.property_, *request_manager_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["expression"] = ToJson(op.expression_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
@@ -477,8 +476,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
   json self;
   self["name"] = "ScanAllByLabelProperty";
-  self["label"] = ToJson(op.label_, *request_manager_);
+  self["label"] = ToJson(op.label_, *request_router_);
-  self["property"] = ToJson(op.property_, *request_manager_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
   op.input_->Accept(*this);
@@ -488,12 +487,15 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
   return false;
 }
 
-bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
+bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
   json self;
-  self["name"] = "ScanAllById";
+  self["name"] = "ScanByPrimaryKey";
+  self["label"] = ToJson(op.label_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
+
   op.input_->Accept(*this);
   self["input"] = PopOutput();
+
   output_ = std::move(self);
   return false;
 }
@@ -501,7 +503,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
 bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
   json self;
   self["name"] = "CreateNode";
-  self["node_info"] = ToJson(op.node_info_, *request_manager_);
+  self["node_info"] = ToJson(op.node_info_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();
@@ -514,8 +516,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
   json self;
   self["name"] = "CreateExpand";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["node_info"] = ToJson(op.node_info_, *request_manager_);
+  self["node_info"] = ToJson(op.node_info_, *request_router_);
-  self["edge_info"] = ToJson(op.edge_info_, *request_manager_);
+  self["edge_info"] = ToJson(op.edge_info_, *request_router_);
   self["existing_node"] = op.existing_node_;
 
   op.input_->Accept(*this);
@@ -531,7 +533,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
   self["input_symbol"] = ToJson(op.input_symbol_);
   self["node_symbol"] = ToJson(op.common_.node_symbol);
   self["edge_symbol"] = ToJson(op.common_.edge_symbol);
-  self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
+  self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
   self["direction"] = ToString(op.common_.direction);
   self["existing_node"] = op.common_.existing_node;
 
@@ -548,7 +550,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
   self["input_symbol"] = ToJson(op.input_symbol_);
   self["node_symbol"] = ToJson(op.common_.node_symbol);
   self["edge_symbol"] = ToJson(op.common_.edge_symbol);
-  self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
+  self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
   self["direction"] = ToString(op.common_.direction);
   self["type"] = ToString(op.type_);
   self["is_reverse"] = op.is_reverse_;
@@ -623,7 +625,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
 bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
   json self;
   self["name"] = "SetProperty";
-  self["property"] = ToJson(op.property_, *request_manager_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lhs"] = ToJson(op.lhs_);
   self["rhs"] = ToJson(op.rhs_);
 
@@ -660,7 +662,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
   json self;
   self["name"] = "SetLabels";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["labels"] = ToJson(op.labels_, *request_manager_);
+  self["labels"] = ToJson(op.labels_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();
@@ -672,7 +674,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
 bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
   json self;
   self["name"] = "RemoveProperty";
-  self["property"] = ToJson(op.property_, *request_manager_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lhs"] = ToJson(op.lhs_);
 
   op.input_->Accept(*this);
@@ -686,7 +688,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
   json self;
   self["name"] = "RemoveLabels";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["labels"] = ToJson(op.labels_, *request_manager_);
+  self["labels"] = ToJson(op.labels_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();

src/query/v2/plan/pretty_print.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,7 @@
 
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/plan/operator.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/request_router.hpp"
 
 namespace memgraph::query::v2 {
 
@@ -27,20 +27,19 @@ namespace plan {
 class LogicalOperator;
 
 /// Pretty print a `LogicalOperator` plan to a `std::ostream`.
-/// ShardRequestManager is needed for resolving label and property names.
+/// RequestRouter is needed for resolving label and property names.
 /// Note that `plan_root` isn't modified, but we can't take it as a const
 /// because we don't have support for visiting a const LogicalOperator.
-void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
+void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
-                 std::ostream *out);
 
 /// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
-inline void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
+inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
-  PrettyPrint(request_manager, plan_root, &std::cout);
+  PrettyPrint(request_router, plan_root, &std::cout);
 }
 
 /// Convert a `LogicalOperator` plan to a JSON representation.
 /// DbAccessor is needed for resolving label and property names.
-nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root);
+nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
 
 class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
  public:
@@ -48,7 +47,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   using HierarchicalLogicalOperatorVisitor::PreVisit;
   using HierarchicalLogicalOperatorVisitor::Visit;
 
-  PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out);
+  PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
 
   bool DefaultPreVisit() override;
 
@@ -68,7 +67,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   bool PreVisit(ScanAllByLabelPropertyValue &) override;
   bool PreVisit(ScanAllByLabelPropertyRange &) override;
   bool PreVisit(ScanAllByLabelProperty &) override;
-  bool PreVisit(ScanAllById &) override;
+  bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
 
   bool PreVisit(Expand &) override;
   bool PreVisit(ExpandVariable &) override;
@@ -115,7 +114,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   void Branch(LogicalOperator &op, const std::string &branch_name = "");
 
   int64_t depth_{0};
-  const msgs::ShardRequestManagerInterface *request_manager_{nullptr};
+  const RequestRouterInterface *request_router_{nullptr};
   std::ostream *out_{nullptr};
 };
 
@@ -133,20 +132,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
 
 nlohmann::json ToJson(const Symbol &symbol);
 
-nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager);
+nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager);
+nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager);
+nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
 
 nlohmann::json ToJson(NamedExpression *nexpr);
 
 nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
-                      const msgs::ShardRequestManagerInterface &request_manager);
+                      const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager);
+nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager);
+nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
 
 nlohmann::json ToJson(const Aggregate::Element &elem);
 
@@ -161,8 +160,7 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
 
 class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
  public:
-  explicit PlanToJsonVisitor(const msgs::ShardRequestManagerInterface *request_manager)
+  explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
-      : request_manager_(request_manager) {}
 
   using HierarchicalLogicalOperatorVisitor::PostVisit;
   using HierarchicalLogicalOperatorVisitor::PreVisit;
@@ -196,7 +194,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
   bool PreVisit(ScanAllByLabelPropertyRange &) override;
   bool PreVisit(ScanAllByLabelPropertyValue &) override;
   bool PreVisit(ScanAllByLabelProperty &) override;
-  bool PreVisit(ScanAllById &) override;
+  bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
 
   bool PreVisit(Produce &) override;
   bool PreVisit(Accumulate &) override;
@@ -218,7 +216,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
 
  protected:
   nlohmann::json output_;
-  const msgs::ShardRequestManagerInterface *request_manager_;
+  const RequestRouterInterface *request_router_;
 
   nlohmann::json PopOutput() {
     nlohmann::json tmp;

src/query/v2/plan/read_write_type_checker.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,10 +11,12 @@
 
 #include "query/v2/plan/read_write_type_checker.hpp"
 
-#define PRE_VISIT(TOp, RWType, continue_visiting) \
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
-  bool ReadWriteTypeChecker::PreVisit(TOp &op) {  \
+#define PRE_VISIT(TOp, RWType, continue_visiting)     \
-    UpdateType(RWType);                           \
+  /*NOLINTNEXTLINE(bugprone-macro-parentheses)*/      \
-    return continue_visiting;                     \
+  bool ReadWriteTypeChecker::PreVisit(TOp & /*op*/) { \
+    UpdateType(RWType);                               \
+    return continue_visiting;                         \
   }
 
 namespace memgraph::query::v2::plan {
@@ -35,7 +37,7 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true)
 PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true)
 PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true)
 PRE_VISIT(ScanAllByLabelProperty, RWType::R, true)
-PRE_VISIT(ScanAllById, RWType::R, true)
+PRE_VISIT(ScanByPrimaryKey, RWType::R, true)
 
 PRE_VISIT(Expand, RWType::R, true)
 PRE_VISIT(ExpandVariable, RWType::R, true)

src/query/v2/plan/read_write_type_checker.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -59,7 +59,7 @@ class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
   bool PreVisit(ScanAllByLabelPropertyValue &) override;
   bool PreVisit(ScanAllByLabelPropertyRange &) override;
   bool PreVisit(ScanAllByLabelProperty &) override;
-  bool PreVisit(ScanAllById &) override;
+  bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
 
   bool PreVisit(Expand &) override;
   bool PreVisit(ExpandVariable &) override;

src/query/v2/plan/rewrite/index_lookup.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,8 @@
 
 #include "utils/flag_validation.hpp"
 
-DEFINE_VALIDATED_HIDDEN_int64(query_vertex_count_to_expand_existing, 10,
+// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_VALIDATED_HIDDEN_int64(query_v2_vertex_count_to_expand_existing, 10,
                               "Maximum count of indexed vertices which provoke "
                               "indexed lookup and then expand to existing, instead of "
                               "a regular expand. Default is 10, to turn off use -1.",

src/query/v2/plan/rewrite/index_lookup.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,10 +25,13 @@
 
 #include <gflags/gflags.h>
 
+#include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/plan/operator.hpp"
 #include "query/v2/plan/preprocess.hpp"
+#include "storage/v3/id_types.hpp"
 
-DECLARE_int64(query_vertex_count_to_expand_existing);
+// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_int64(query_v2_vertex_count_to_expand_existing);
 
 namespace memgraph::query::v2::plan {
 
@@ -98,7 +101,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
       return true;
     }
     ScanAll dst_scan(expand.input(), expand.common_.node_symbol, expand.view_);
-    auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing);
+    auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_v2_vertex_count_to_expand_existing);
     if (indexed_scan) {
       expand.set_input(std::move(indexed_scan));
       expand.common_.existing_node = true;
@@ -127,7 +130,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
       // unconditionally creating an indexed scan.
       indexed_scan = GenScanByIndex(dst_scan);
     } else {
-      indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing);
+      indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_v2_vertex_count_to_expand_existing);
     }
     if (indexed_scan) {
       expand.set_input(std::move(indexed_scan));
@@ -271,11 +274,12 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
     return true;
   }
 
-  bool PreVisit(ScanAllById &op) override {
+  bool PreVisit(ScanByPrimaryKey &op) override {
     prev_ops_.push_back(&op);
     return true;
   }
-  bool PostVisit(ScanAllById &) override {
+
+  bool PostVisit(ScanByPrimaryKey & /*unused*/) override {
     prev_ops_.pop_back();
     return true;
   }
@@ -487,6 +491,12 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
 
   storage::v3::PropertyId GetProperty(PropertyIx prop) { return db_->NameToProperty(prop.name); }
 
+  void EraseLabelFilters(const memgraph::query::v2::Symbol &node_symbol, memgraph::query::v2::LabelIx prim_label) {
+    std::vector<query::v2::Expression *> removed_expressions;
+    filters_.EraseLabelFilter(node_symbol, prim_label, &removed_expressions);
+    filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
+  }
+
   std::optional<LabelIx> FindBestLabelIndex(const std::unordered_set<LabelIx> &labels) {
     MG_ASSERT(!labels.empty(), "Trying to find the best label without any labels.");
     std::optional<LabelIx> best_label;
@@ -559,31 +569,84 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
     const auto &view = scan.view_;
     const auto &modified_symbols = scan.ModifiedSymbols(*symbol_table_);
     std::unordered_set<Symbol> bound_symbols(modified_symbols.begin(), modified_symbols.end());
-    auto are_bound = [&bound_symbols](const auto &used_symbols) {
+
-      for (const auto &used_symbol : used_symbols) {
+    // Try to see if we can use label + primary-key or label + property index.
-        if (!utils::Contains(bound_symbols, used_symbol)) {
+    // If not, try to use just the label index.
-          return false;
-        }
-      }
-      return true;
-    };
-    // First, try to see if we can find a vertex by ID.
-    if (!max_vertex_count || *max_vertex_count >= 1) {
-      for (const auto &filter : filters_.IdFilters(node_symbol)) {
-        if (filter.id_filter->is_symbol_in_value_ || !are_bound(filter.used_symbols)) continue;
-        auto *value = filter.id_filter->value_;
-        filter_exprs_for_removal_.insert(filter.expression);
-        filters_.EraseFilter(filter);
-        return std::make_unique<ScanAllById>(input, node_symbol, value, view);
-      }
-    }
-    // Now try to see if we can use label+property index. If not, try to use
-    // just the label index.
     const auto labels = filters_.FilteredLabels(node_symbol);
     if (labels.empty()) {
       // Without labels, we cannot generate any indexed ScanAll.
       return nullptr;
     }
+
+    // First, try to see if we can find a vertex based on the possibly
+    // supplied primary key.
+    auto property_filters = filters_.PropertyFilters(node_symbol);
+    query::v2::LabelIx prim_label;
+    std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> primary_key;
+
+    auto extract_primary_key = [this](storage::v3::LabelId label,
+                                      std::vector<query::v2::plan::FilterInfo> property_filters)
+        -> std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> {
+      std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk_temp;
+      std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk;
+      std::vector<memgraph::storage::v3::SchemaProperty> schema = db_->GetSchemaForLabel(label);
+
+      std::vector<storage::v3::PropertyId> schema_properties;
+      schema_properties.reserve(schema.size());
+
+      std::transform(schema.begin(), schema.end(), std::back_inserter(schema_properties),
+                     [](const auto &schema_elem) { return schema_elem.property_id; });
+
+      for (const auto &property_filter : property_filters) {
+        if (property_filter.property_filter->type_ != PropertyFilter::Type::EQUAL) {
+          continue;
+        }
+        const auto &property_id = db_->NameToProperty(property_filter.property_filter->property_.name);
+        if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) {
+          pk_temp.emplace_back(std::make_pair(property_filter.expression, property_filter));
+        }
+      }
+
+      // Make sure pk is in the same order as schema_properties.
+      for (const auto &schema_prop : schema_properties) {
+        for (auto &pk_temp_prop : pk_temp) {
+          const auto &property_id = db_->NameToProperty(pk_temp_prop.second.property_filter->property_.name);
+          if (schema_prop == property_id) {
+            pk.push_back(pk_temp_prop);
+          }
+        }
+      }
+      MG_ASSERT(pk.size() == pk_temp.size(),
+                "The two vectors should represent the same primary key with a possibly different order of contained "
+                "elements.");
+
+      return pk.size() == schema_properties.size()
+                 ? pk
+                 : std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>>{};
+    };
+
+    if (!property_filters.empty()) {
+      for (const auto &label : labels) {
+        if (db_->PrimaryLabelExists(GetLabel(label))) {
+          prim_label = label;
+          primary_key = extract_primary_key(GetLabel(prim_label), property_filters);
+          break;
+        }
+      }
+      if (!primary_key.empty()) {
+        // Mark the expressions so they won't be used for an additional, unnecessary filter.
+        for (const auto &primary_property : primary_key) {
+          filter_exprs_for_removal_.insert(primary_property.first);
+          filters_.EraseFilter(primary_property.second);
+        }
+        EraseLabelFilters(node_symbol, prim_label);
+        std::vector<query::v2::Expression *> pk_expressions;
+        std::transform(primary_key.begin(), primary_key.end(), std::back_inserter(pk_expressions),
+                       [](const auto &exp) { return exp.second.property_filter->value_; });
+        return std::make_unique<ScanByPrimaryKey>(input, node_symbol, GetLabel(prim_label), pk_expressions);
+      }
+    }
+
     auto found_index = FindBestLabelPropertyIndex(node_symbol, bound_symbols);
     if (found_index &&
         // Use label+property index if we satisfy max_vertex_count.
@@ -597,9 +660,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
         filter_exprs_for_removal_.insert(found_index->filter.expression);
       }
       filters_.EraseFilter(found_index->filter);
-      std::vector<Expression *> removed_expressions;
+      EraseLabelFilters(node_symbol, found_index->label);
-      filters_.EraseLabelFilter(node_symbol, found_index->label, &removed_expressions);
-      filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
       if (prop_filter.lower_bound_ || prop_filter.upper_bound_) {
         return std::make_unique<ScanAllByLabelPropertyRange>(
             input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_),

src/query/v2/plan/rule_based_planner.hpp

@@ -272,7 +272,7 @@ class RuleBasedPlanner {
           PropertiesMapList vector_props;
           vector_props.reserve(node_properties->size());
           for (const auto &kv : *node_properties) {
-            // TODO(kostasrim) GetProperty should be implemented in terms of ShardRequestManager NameToProperty
+            // TODO(kostasrim) GetProperty should be implemented in terms of RequestRouter NameToProperty
             vector_props.push_back({GetProperty(kv.first), kv.second});
           }
           return std::move(vector_props);

src/query/v2/plan/variable_start_planner.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,7 +17,8 @@
 #include "utils/flag_validation.hpp"
 #include "utils/logging.hpp"
 
-DEFINE_VALIDATED_HIDDEN_uint64(query_max_plans, 1000U, "Maximum number of generated plans for a query.",
+// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_VALIDATED_HIDDEN_uint64(query_v2_max_plans, 1000U, "Maximum number of generated plans for a query.",
                                FLAG_IN_RANGE(1, std::numeric_limits<std::uint64_t>::max()));
 
 namespace memgraph::query::v2::plan::impl {

src/query/v2/plan/variable_start_planner.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,8 @@
 
 #include "query/v2/plan/rule_based_planner.hpp"
 
-DECLARE_uint64(query_max_plans);
+// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint64(query_v2_max_plans);
 
 namespace memgraph::query::v2::plan {
 
@@ -310,7 +311,7 @@ class VariableStartPlanner {
     for (const auto &query_part : query_parts) {
       alternative_query_parts.emplace_back(impl::VaryQueryPartMatching(query_part, symbol_table));
     }
-    return iter::slice(MakeCartesianProduct(std::move(alternative_query_parts)), 0UL, FLAGS_query_max_plans);
+    return iter::slice(MakeCartesianProduct(std::move(alternative_query_parts)), 0UL, FLAGS_query_v2_max_plans);
   }
 
  public:

src/query/v2/plan/vertex_count_cache.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,14 +12,17 @@
 /// @file
 #pragma once
 
+#include <iterator>
 #include <optional>
 
 #include "query/v2/bindings/typed_value.hpp"
-#include "query/v2/shard_request_manager.hpp"
+#include "query/v2/plan/preprocess.hpp"
+#include "query/v2/request_router.hpp"
 #include "storage/v3/conversions.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
 #include "utils/bound.hpp"
+#include "utils/exceptions.hpp"
 #include "utils/fnv.hpp"
 
 namespace memgraph::query::v2::plan {
@@ -29,11 +32,11 @@ namespace memgraph::query::v2::plan {
 template <class TDbAccessor>
 class VertexCountCache {
  public:
-  explicit VertexCountCache(TDbAccessor *shard_request_manager) : shard_request_manager_{shard_request_manager} {}
+  explicit VertexCountCache(TDbAccessor *request_router) : request_router_{request_router} {}
 
-  auto NameToLabel(const std::string &name) { return shard_request_manager_->NameToLabel(name); }
+  auto NameToLabel(const std::string &name) { return request_router_->NameToLabel(name); }
-  auto NameToProperty(const std::string &name) { return shard_request_manager_->NameToProperty(name); }
+  auto NameToProperty(const std::string &name) { return request_router_->NameToProperty(name); }
-  auto NameToEdgeType(const std::string &name) { return shard_request_manager_->NameToEdgeType(name); }
+  auto NameToEdgeType(const std::string &name) { return request_router_->NameToEdgeType(name); }
 
   int64_t VerticesCount() { return 1; }
 
@@ -52,12 +55,17 @@ class VertexCountCache {
     return 1;
   }
 
-  // For now return true if label is primary label
+  bool LabelIndexExists(storage::v3::LabelId label) { return PrimaryLabelExists(label); }
-  bool LabelIndexExists(storage::v3::LabelId label) { return shard_request_manager_->IsPrimaryLabel(label); }
+
+  bool PrimaryLabelExists(storage::v3::LabelId label) { return request_router_->IsPrimaryLabel(label); }
 
   bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; }
 
-  msgs::ShardRequestManagerInterface *shard_request_manager_;
+  const std::vector<memgraph::storage::v3::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) {
+    return request_router_->GetSchemaForLabel(label);
+  }
+
+  RequestRouterInterface *request_router_;
 };
 
 template <class TDbAccessor>

src/query/v2/request_router.hpp

@@ -0,0 +1,817 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <chrono>
+#include <deque>
+#include <iostream>
+#include <iterator>
+#include <map>
+#include <numeric>
+#include <optional>
+#include <random>
+#include <set>
+#include <stdexcept>
+#include <thread>
+#include <unordered_map>
+#include <variant>
+#include <vector>
+
+#include <boost/uuid/uuid.hpp>
+
+#include "coordinator/coordinator.hpp"
+#include "coordinator/coordinator_client.hpp"
+#include "coordinator/coordinator_rsm.hpp"
+#include "coordinator/shard_map.hpp"
+#include "io/address.hpp"
+#include "io/errors.hpp"
+#include "io/local_transport/local_transport.hpp"
+#include "io/notifier.hpp"
+#include "io/rsm/raft.hpp"
+#include "io/rsm/rsm_client.hpp"
+#include "io/rsm/shard_rsm.hpp"
+#include "io/simulator/simulator.hpp"
+#include "io/simulator/simulator_transport.hpp"
+#include "query/v2/accessors.hpp"
+#include "query/v2/requests.hpp"
+#include "storage/v3/id_types.hpp"
+#include "storage/v3/value_conversions.hpp"
+#include "utils/result.hpp"
+
+namespace memgraph::query::v2 {
+
+template <typename TStorageClient>
+class RsmStorageClientManager {
+ public:
+  using CompoundKey = io::rsm::ShardRsmKey;
+  using ShardMetadata = coordinator::ShardMetadata;
+  RsmStorageClientManager() = default;
+  RsmStorageClientManager(const RsmStorageClientManager &) = delete;
+  RsmStorageClientManager(RsmStorageClientManager &&) = delete;
+  RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
+  RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
+  ~RsmStorageClientManager() = default;
+
+  void AddClient(ShardMetadata key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
+
+  bool Exists(const ShardMetadata &key) { return cli_cache_.contains(key); }
+
+  void PurgeCache() { cli_cache_.clear(); }
+
+  TStorageClient &GetClient(const ShardMetadata &key) {
+    auto it = cli_cache_.find(key);
+    MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
+    return it->second;
+  }
+
+ private:
+  std::map<ShardMetadata, TStorageClient> cli_cache_;
+};
+
+template <typename TRequest>
+struct ShardRequestState {
+  memgraph::coordinator::ShardMetadata shard;
+  TRequest request;
+};
+
+// maps from ReadinessToken's internal size_t to the associated state
+template <typename TRequest>
+using RunningRequests = std::unordered_map<size_t, ShardRequestState<TRequest>>;
+
+class RequestRouterInterface {
+ public:
+  using VertexAccessor = query::v2::accessors::VertexAccessor;
+  RequestRouterInterface() = default;
+  RequestRouterInterface(const RequestRouterInterface &) = delete;
+  RequestRouterInterface(RequestRouterInterface &&) = delete;
+  RequestRouterInterface &operator=(const RequestRouterInterface &) = delete;
+  RequestRouterInterface &&operator=(RequestRouterInterface &&) = delete;
+
+  virtual ~RequestRouterInterface() = default;
+
+  virtual void StartTransaction() = 0;
+  virtual void Commit() = 0;
+  virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) = 0;
+  virtual std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) = 0;
+  virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
+  virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
+  virtual std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest request) = 0;
+
+  virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
+  virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
+  virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
+  virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
+  virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
+  virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
+  virtual std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const = 0;
+  virtual std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const = 0;
+  virtual std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const = 0;
+  virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0;
+  virtual bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0;
+
+  virtual std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) = 0;
+  virtual void InstallSimulatorTicker(std::function<bool()> tick_simulator) = 0;
+  virtual const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const = 0;
+};
+
+// TODO(kostasrim)rename this class template
+template <typename TTransport>
+class RequestRouter : public RequestRouterInterface {
+ public:
+  using StorageClient = coordinator::RsmClient<TTransport, msgs::WriteRequests, msgs::WriteResponses,
+                                               msgs::ReadRequests, msgs::ReadResponses>;
+  using CoordinatorWriteRequests = coordinator::CoordinatorWriteRequests;
+  using CoordinatorClient = coordinator::CoordinatorClient<TTransport>;
+  using Address = io::Address;
+  using ShardMetadata = coordinator::ShardMetadata;
+  using ShardMap = coordinator::ShardMap;
+  using CompoundKey = coordinator::PrimaryKey;
+  using VertexAccessor = query::v2::accessors::VertexAccessor;
+  RequestRouter(CoordinatorClient coord, io::Io<TTransport> &&io) : coord_cli_(std::move(coord)), io_(std::move(io)) {}
+
+  RequestRouter(const RequestRouter &) = delete;
+  RequestRouter(RequestRouter &&) = delete;
+  RequestRouter &operator=(const RequestRouter &) = delete;
+  RequestRouter &operator=(RequestRouter &&) = delete;
+
+  ~RequestRouter() override {}
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) override {
+    notifier_.InstallSimulatorTicker(tick_simulator);
+  }
+
+  void StartTransaction() override {
+    coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
+    CoordinatorWriteRequests write_req = req;
+    spdlog::trace("sending hlc request to start transaction");
+    auto write_res = coord_cli_.SendWriteRequest(write_req);
+    spdlog::trace("received hlc response to start transaction");
+    if (write_res.HasError()) {
+      throw std::runtime_error("HLC request failed");
+    }
+    auto coordinator_write_response = write_res.GetValue();
+    auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
+
+    // Transaction ID to be used later...
+    transaction_id_ = hlc_response.new_hlc;
+
+    if (hlc_response.fresher_shard_map) {
+      shards_map_ = hlc_response.fresher_shard_map.value();
+      SetUpNameIdMappers();
+    }
+  }
+
+  void Commit() override {
+    coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
+    CoordinatorWriteRequests write_req = req;
+    spdlog::trace("sending hlc request before committing transaction");
+    auto write_res = coord_cli_.SendWriteRequest(write_req);
+    spdlog::trace("received hlc response before committing transaction");
+    if (write_res.HasError()) {
+      throw std::runtime_error("HLC request for commit failed");
+    }
+    auto coordinator_write_response = write_res.GetValue();
+    auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
+
+    if (hlc_response.fresher_shard_map) {
+      shards_map_ = hlc_response.fresher_shard_map.value();
+      SetUpNameIdMappers();
+    }
+    auto commit_timestamp = hlc_response.new_hlc;
+
+    msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
+
+    for (const auto &[label, space] : shards_map_.label_spaces) {
+      for (const auto &[key, shard] : space.shards) {
+        auto &storage_client = GetStorageClientForShard(shard);
+        // TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
+        // instead.
+        auto commit_response = storage_client.SendWriteRequest(commit_req);
+        // RETRY on timeouts?
+        // Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
+        if (commit_response.HasError()) {
+          throw std::runtime_error("Commit request timed out");
+        }
+        msgs::WriteResponses write_response_variant = commit_response.GetValue();
+        auto &response = std::get<msgs::CommitResponse>(write_response_variant);
+        if (response.error) {
+          throw std::runtime_error("Commit request did not succeed");
+        }
+      }
+    }
+  }
+
+  storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
+    return shards_map_.GetEdgeTypeId(name).value();
+  }
+
+  storage::v3::PropertyId NameToProperty(const std::string &name) const override {
+    return shards_map_.GetPropertyId(name).value();
+  }
+
+  storage::v3::LabelId NameToLabel(const std::string &name) const override {
+    return shards_map_.GetLabelId(name).value();
+  }
+
+  const std::string &PropertyToName(storage::v3::PropertyId id) const override {
+    return properties_.IdToName(id.AsUint());
+  }
+  const std::string &LabelToName(storage::v3::LabelId id) const override { return labels_.IdToName(id.AsUint()); }
+  const std::string &EdgeTypeToName(storage::v3::EdgeTypeId id) const override {
+    return edge_types_.IdToName(id.AsUint());
+  }
+
+  bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
+    const auto schema_it = shards_map_.schemas.find(primary_label);
+    MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
+
+    return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
+             return schema_prop.property_id == property;
+           }) != schema_it->second.end();
+  }
+
+  const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const override {
+    return shards_map_.schemas.at(label);
+  }
+
+  bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); }
+
+  // TODO(kostasrim) Simplify return result
+  std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
+    // create requests
+    auto requests_to_be_sent = RequestsForScanVertices(label);
+
+    spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
+
+    // begin all requests in parallel
+    RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
+    }
+    spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
+
+    // drive requests to completion
+    auto responses = DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
+    spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
+
+    // convert responses into VertexAccessor objects to return
+    std::vector<VertexAccessor> accessors;
+    accessors.reserve(responses.size());
+    for (auto &response : responses) {
+      for (auto &result_row : response.results) {
+        accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
+      }
+    }
+
+    return accessors;
+  }
+
+  std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) override {
+    MG_ASSERT(!new_vertices.empty());
+
+    // create requests
+    std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
+        RequestsForCreateVertices(new_vertices);
+    spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
+
+    // begin all requests in parallel
+    RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      for (auto &new_vertex : request.request.new_vertices) {
+        new_vertex.label_ids.erase(new_vertex.label_ids.begin());
+      }
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), request);
+    }
+    spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
+
+    // drive requests to completion
+    return DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
+  }
+
+  std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) override {
+    MG_ASSERT(!new_edges.empty());
+
+    // create requests
+    std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent =
+        RequestsForCreateExpand(std::move(new_edges));
+
+    // begin all requests in parallel
+    RunningRequests<msgs::CreateExpandRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      msgs::WriteRequests req = request.request;
+      storage_client.SendAsyncWriteRequest(std::move(req), notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), std::move(request));
+    }
+
+    // drive requests to completion
+    return DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
+  }
+
+  std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) override {
+    // TODO(kostasrim)Update to limit the batch size here
+    // Expansions of the destination must be handled by the caller. For example
+    // match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
+    // For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
+    // must be fetched again with an ExpandOne(Edges.dst)
+
+    // create requests
+    std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent =
+        RequestsForExpandOne(std::move(request));
+
+    // begin all requests in parallel
+    RunningRequests<msgs::ExpandOneRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      msgs::ReadRequests req = request.request;
+      storage_client.SendAsyncReadRequest(std::move(req), notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), std::move(request));
+    }
+
+    // drive requests to completion
+    auto responses = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
+
+    // post-process responses
+    std::vector<msgs::ExpandOneResultRow> result_rows;
+    const auto total_row_count = std::accumulate(responses.begin(), responses.end(), 0,
+                                                 [](const int64_t partial_count, const msgs::ExpandOneResponse &resp) {
+                                                   return partial_count + resp.result.size();
+                                                 });
+    result_rows.reserve(total_row_count);
+
+    for (auto &response : responses) {
+      result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
+                         std::make_move_iterator(response.result.end()));
+    }
+
+    return result_rows;
+  }
+
+  std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest requests) override {
+    requests.transaction_id = transaction_id_;
+    // create requests
+    std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
+        RequestsForGetProperties(std::move(requests));
+
+    // begin all requests in parallel
+    RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
+    running_requests.reserve(requests_to_be_sent.size());
+    for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
+      auto &request = requests_to_be_sent[i];
+      io::ReadinessToken readiness_token{i};
+      auto &storage_client = GetStorageClientForShard(request.shard);
+      msgs::ReadRequests req = request.request;
+      storage_client.SendAsyncReadRequest(std::move(req), notifier_, readiness_token);
+      running_requests.emplace(readiness_token.GetId(), std::move(request));
+    }
+
+    // drive requests to completion
+    auto responses = DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
+
+    // post-process responses
+    std::vector<msgs::GetPropertiesResultRow> result_rows;
+
+    for (auto &&response : responses) {
+      std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
+    }
+
+    return result_rows;
+  }
+
+  std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
+    return shards_map_.GetPropertyId(name);
+  }
+
+  std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const override {
+    return shards_map_.GetEdgeTypeId(name);
+  }
+
+  std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const override {
+    return shards_map_.GetLabelId(name);
+  }
+
+ private:
+  std::vector<ShardRequestState<msgs::CreateVerticesRequest>> RequestsForCreateVertices(
+      const std::vector<msgs::NewVertex> &new_vertices) {
+    std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
+
+    for (auto &new_vertex : new_vertices) {
+      MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
+      auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
+                                              storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
+      if (!per_shard_request_table.contains(shard)) {
+        msgs::CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
+        per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
+      }
+      per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
+    }
+
+    std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests = {};
+
+    for (auto &[shard, request] : per_shard_request_table) {
+      ShardRequestState<msgs::CreateVerticesRequest> shard_request_state{
+          .shard = shard,
+          .request = request,
+      };
+      requests.emplace_back(std::move(shard_request_state));
+    }
+
+    return requests;
+  }
+
+  std::vector<ShardRequestState<msgs::CreateExpandRequest>> RequestsForCreateExpand(
+      std::vector<msgs::NewExpand> new_expands) {
+    std::map<ShardMetadata, msgs::CreateExpandRequest> per_shard_request_table;
+    auto ensure_shard_exists_in_table = [&per_shard_request_table,
+                                         transaction_id = transaction_id_](const ShardMetadata &shard) {
+      if (!per_shard_request_table.contains(shard)) {
+        msgs::CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
+        per_shard_request_table.insert({shard, std::move(create_expand_request)});
+      }
+    };
+
+    for (auto &new_expand : new_expands) {
+      const auto shard_src_vertex = shards_map_.GetShardForKey(
+          new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
+      const auto shard_dest_vertex = shards_map_.GetShardForKey(
+          new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
+
+      ensure_shard_exists_in_table(shard_src_vertex);
+
+      if (shard_src_vertex != shard_dest_vertex) {
+        ensure_shard_exists_in_table(shard_dest_vertex);
+        per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
+      }
+      per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
+    }
+
+    std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests = {};
+
+    for (auto &[shard, request] : per_shard_request_table) {
+      ShardRequestState<msgs::CreateExpandRequest> shard_request_state{
+          .shard = shard,
+          .request = request,
+      };
+      requests.emplace_back(std::move(shard_request_state));
+    }
+
+    return requests;
+  }
+
+  std::vector<ShardRequestState<msgs::ScanVerticesRequest>> RequestsForScanVertices(
+      const std::optional<std::string> &label) {
+    std::vector<coordinator::Shards> multi_shards;
+    if (label) {
+      const auto label_id = shards_map_.GetLabelId(*label);
+      MG_ASSERT(label_id);
+      MG_ASSERT(IsPrimaryLabel(*label_id));
+      multi_shards = {shards_map_.GetShardsForLabel(*label)};
+    } else {
+      multi_shards = shards_map_.GetAllShards();
+    }
+
+    std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
+
+    for (auto &shards : multi_shards) {
+      for (auto &[key, shard] : shards) {
+        MG_ASSERT(!shard.peers.empty());
+
+        msgs::ScanVerticesRequest request;
+        request.transaction_id = transaction_id_;
+        request.props_to_return.emplace();
+        request.start_id.second = storage::conversions::ConvertValueVector(key);
+
+        ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
+            .shard = shard,
+            .request = std::move(request),
+        };
+
+        requests.emplace_back(std::move(shard_request_state));
+      }
+    }
+
+    return requests;
+  }
+
+  std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(msgs::ExpandOneRequest &&request) {
+    std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
+    msgs::ExpandOneRequest top_level_rqst_template = request;
+    top_level_rqst_template.transaction_id = transaction_id_;
+    top_level_rqst_template.src_vertices.clear();
+
+    for (auto &vertex : request.src_vertices) {
+      auto shard =
+          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
+      if (!per_shard_request_table.contains(shard)) {
+        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
+      }
+      per_shard_request_table[shard].src_vertices.push_back(std::move(vertex));
+    }
+
+    std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests = {};
+
+    for (auto &[shard, request] : per_shard_request_table) {
+      ShardRequestState<msgs::ExpandOneRequest> shard_request_state{
+          .shard = shard,
+          .request = request,
+      };
+
+      requests.emplace_back(std::move(shard_request_state));
+    }
+
+    return requests;
+  }
+
+  std::vector<ShardRequestState<msgs::GetPropertiesRequest>> RequestsForGetProperties(
+      msgs::GetPropertiesRequest &&request) {
+    std::map<ShardMetadata, msgs::GetPropertiesRequest> per_shard_request_table;
+    auto top_level_rqst_template = request;
+    top_level_rqst_template.transaction_id = transaction_id_;
+    top_level_rqst_template.vertex_ids.clear();
+    top_level_rqst_template.vertices_and_edges.clear();
+
+    for (auto &&vertex : request.vertex_ids) {
+      auto shard =
+          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
+      if (!per_shard_request_table.contains(shard)) {
+        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
+      }
+      per_shard_request_table[shard].vertex_ids.emplace_back(std::move(vertex));
+    }
+
+    for (auto &[vertex, maybe_edge] : request.vertices_and_edges) {
+      auto shard =
+          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
+      if (!per_shard_request_table.contains(shard)) {
+        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
+      }
+      per_shard_request_table[shard].vertices_and_edges.emplace_back(std::move(vertex), maybe_edge);
+    }
+
+    std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests;
+
+    for (auto &[shard, rqst] : per_shard_request_table) {
+      ShardRequestState<msgs::GetPropertiesRequest> shard_request_state{
+          .shard = shard,
+          .request = std::move(rqst),
+      };
+
+      requests.emplace_back(std::move(shard_request_state));
+    }
+
+    return requests;
+  }
+
+  StorageClient &GetStorageClientForShard(ShardMetadata shard) {
+    if (!storage_cli_manager_.Exists(shard)) {
+      AddStorageClientToManager(shard);
+    }
+    return storage_cli_manager_.GetClient(shard);
+  }
+
+  StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
+    auto shard = shards_map_.GetShardForKey(label, key);
+    return GetStorageClientForShard(std::move(shard));
+  }
+
+  void AddStorageClientToManager(ShardMetadata target_shard) {
+    MG_ASSERT(!target_shard.peers.empty());
+    auto leader_addr = target_shard.peers.front();
+    std::vector<Address> addresses;
+    addresses.reserve(target_shard.peers.size());
+    for (auto &address : target_shard.peers) {
+      addresses.push_back(std::move(address.address));
+    }
+    auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
+    storage_cli_manager_.AddClient(target_shard, std::move(cli));
+  }
+
+  template <typename RequestT, typename ResponseT>
+  std::vector<ResponseT> DriveReadResponses(RunningRequests<RequestT> &running_requests) {
+    // Store responses in a map based on the corresponding request
+    // offset, so that they can be reassembled in the correct order
+    // even if they came back in randomized orders.
+    std::map<size_t, ResponseT> response_map;
+
+    spdlog::trace("waiting on readiness for token");
+    while (response_map.size() < running_requests.size()) {
+      auto ready = notifier_.Await();
+      spdlog::trace("got readiness for token {}", ready.GetId());
+      auto &request = running_requests.at(ready.GetId());
+      auto &storage_client = GetStorageClientForShard(request.shard);
+
+      std::optional<utils::BasicResult<io::TimedOut, msgs::ReadResponses>> poll_result =
+          storage_client.PollAsyncReadRequest(ready);
+
+      if (!poll_result.has_value()) {
+        continue;
+      }
+
+      if (poll_result->HasError()) {
+        throw std::runtime_error("RequestRouter Read request timed out");
+      }
+
+      msgs::ReadResponses response_variant = poll_result->GetValue();
+      auto response = std::get<ResponseT>(response_variant);
+      if (response.error) {
+        throw std::runtime_error("RequestRouter Read request did not succeed");
+      }
+
+      // the readiness token has an ID based on the request vector offset
+      response_map.emplace(ready.GetId(), std::move(response));
+    }
+
+    std::vector<ResponseT> responses;
+    responses.reserve(running_requests.size());
+
+    int last = -1;
+    for (auto &&[offset, response] : response_map) {
+      MG_ASSERT(last + 1 == offset);
+      responses.emplace_back(std::forward<ResponseT>(response));
+      last = offset;
+    }
+
+    return responses;
+  }
+
+  template <typename RequestT, typename ResponseT>
+  std::vector<ResponseT> DriveWriteResponses(RunningRequests<RequestT> &running_requests) {
+    // Store responses in a map based on the corresponding request
+    // offset, so that they can be reassembled in the correct order
+    // even if they came back in randomized orders.
+    std::map<size_t, ResponseT> response_map;
+
+    while (response_map.size() < running_requests.size()) {
+      auto ready = notifier_.Await();
+      auto &request = running_requests.at(ready.GetId());
+      auto &storage_client = GetStorageClientForShard(request.shard);
+
+      std::optional<utils::BasicResult<io::TimedOut, msgs::WriteResponses>> poll_result =
+          storage_client.PollAsyncWriteRequest(ready);
+
+      if (!poll_result.has_value()) {
+        continue;
+      }
+
+      if (poll_result->HasError()) {
+        throw std::runtime_error("RequestRouter Write request timed out");
+      }
+
+      msgs::WriteResponses response_variant = poll_result->GetValue();
+      auto response = std::get<ResponseT>(response_variant);
+      if (response.error) {
+        throw std::runtime_error("RequestRouter Write request did not succeed");
+      }
+
+      // the readiness token has an ID based on the request vector offset
+      response_map.emplace(ready.GetId(), std::move(response));
+    }
+
+    std::vector<ResponseT> responses;
+    responses.reserve(running_requests.size());
+
+    int last = -1;
+    for (auto &&[offset, response] : response_map) {
+      MG_ASSERT(last + 1 == offset);
+      responses.emplace_back(std::forward<ResponseT>(response));
+      last = offset;
+    }
+
+    return responses;
+  }
+
+  void SetUpNameIdMappers() {
+    std::unordered_map<uint64_t, std::string> id_to_name;
+    for (const auto &[name, id] : shards_map_.labels) {
+      id_to_name.emplace(id.AsUint(), name);
+    }
+    labels_.StoreMapping(std::move(id_to_name));
+    id_to_name.clear();
+    for (const auto &[name, id] : shards_map_.properties) {
+      id_to_name.emplace(id.AsUint(), name);
+    }
+    properties_.StoreMapping(std::move(id_to_name));
+    id_to_name.clear();
+    for (const auto &[name, id] : shards_map_.edge_types) {
+      id_to_name.emplace(id.AsUint(), name);
+    }
+    edge_types_.StoreMapping(std::move(id_to_name));
+  }
+
+  std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) override {
+    coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
+
+    io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
+    ww.operation = std::move(requests);
+    auto resp = io_.template Request<io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>,
+                                     io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>>(coordinator_address,
+                                                                                                   std::move(ww))
+                    .Wait();
+    if (resp.HasValue()) {
+      const auto alloc_edge_id_reps =
+          std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
+      return std::make_pair(alloc_edge_id_reps.low, alloc_edge_id_reps.high);
+    }
+    return {};
+  }
+
+  ShardMap shards_map_;
+  storage::v3::NameIdMapper properties_;
+  storage::v3::NameIdMapper edge_types_;
+  storage::v3::NameIdMapper labels_;
+  CoordinatorClient coord_cli_;
+  RsmStorageClientManager<StorageClient> storage_cli_manager_;
+  io::Io<TTransport> io_;
+  coordinator::Hlc transaction_id_;
+  io::Notifier notifier_ = {};
+  // TODO(kostasrim) Add batch prefetching
+};
+
+class RequestRouterFactory {
+ public:
+  RequestRouterFactory() = default;
+  RequestRouterFactory(const RequestRouterFactory &) = delete;
+  RequestRouterFactory &operator=(const RequestRouterFactory &) = delete;
+  RequestRouterFactory(RequestRouterFactory &&) = delete;
+  RequestRouterFactory &operator=(RequestRouterFactory &&) = delete;
+
+  virtual ~RequestRouterFactory() = default;
+
+  virtual std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
+      const coordinator::Address &coordinator_address) const = 0;
+};
+
+class LocalRequestRouterFactory : public RequestRouterFactory {
+  using LocalTransportIo = io::Io<io::local_transport::LocalTransport>;
+  LocalTransportIo &io_;
+
+ public:
+  explicit LocalRequestRouterFactory(LocalTransportIo &io) : io_(io) {}
+
+  std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
+      const coordinator::Address &coordinator_address) const override {
+    using TransportType = io::local_transport::LocalTransport;
+
+    auto query_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
+    auto local_transport_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
+
+    return std::make_unique<RequestRouter<TransportType>>(
+        coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
+        std::move(local_transport_io));
+  }
+};
+
+class SimulatedRequestRouterFactory : public RequestRouterFactory {
+  io::simulator::Simulator *simulator_;
+
+ public:
+  explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator) : simulator_(&simulator) {}
+
+  std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
+      const coordinator::Address &coordinator_address) const override {
+    using TransportType = io::simulator::SimulatorTransport;
+    auto actual_transport_handle = simulator_->GetSimulatorHandle();
+
+    boost::uuids::uuid random_uuid;
+    io::Address unique_local_addr_query;
+
+    // The simulated RR should not introduce stochastic behavior.
+    random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
+
+    auto io = simulator_->Register(unique_local_addr_query);
+    auto query_io = io.ForkLocal(random_uuid);
+
+    return std::make_unique<RequestRouter<TransportType>>(
+        coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
+        std::move(io));
+  }
+};
+
+}  // namespace memgraph::query::v2

src/query/v2/requests.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,6 +12,7 @@
 #pragma once
 
 #include <chrono>
+#include <cstdint>
 #include <iostream>
 #include <map>
 #include <memory>
@@ -24,6 +25,8 @@
 #include "coordinator/hybrid_logical_clock.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/result.hpp"
+#include "utils/fnv.hpp"
 
 namespace memgraph::msgs {
 
@@ -35,6 +38,7 @@ struct Value;
 struct Label {
   LabelId id;
   friend bool operator==(const Label &lhs, const Label &rhs) { return lhs.id == rhs.id; }
+  friend bool operator==(const Label &lhs, const LabelId &rhs) { return lhs.id == rhs; }
 };
 
 // TODO(kostasrim) update this with CompoundKey, same for the rest of the file.
@@ -317,12 +321,13 @@ struct Value {
   }
 };
 
-struct Expression {
+struct ShardError {
-  std::string expression;
+  common::ErrorCode code;
+  std::string message;
 };
 
-struct Filter {
+struct Expression {
-  std::string filter_expression;
+  std::string expression;
 };
 
 enum class OrderingDirection { ASCENDING = 1, DESCENDING = 2 };
@@ -361,27 +366,38 @@ struct ScanResultRow {
 };
 
 struct ScanVerticesResponse {
-  bool success;
+  std::optional<ShardError> error;
   std::optional<VertexId> next_start_id;
   std::vector<ScanResultRow> results;
 };
 
-using VertexOrEdgeIds = std::variant<VertexId, EdgeId>;
-
 struct GetPropertiesRequest {
   Hlc transaction_id;
-  // Shouldn't contain mixed vertex and edge ids
+  std::vector<VertexId> vertex_ids;
-  VertexOrEdgeIds vertex_or_edge_ids;
+  std::vector<std::pair<VertexId, EdgeId>> vertices_and_edges;
-  std::vector<PropertyId> property_ids;
+
-  std::vector<Expression> expressions;
+  std::optional<std::vector<PropertyId>> property_ids;
-  bool only_unique = false;
+  std::vector<std::string> expressions;
-  std::optional<std::vector<OrderBy>> order_by;
+
+  std::vector<OrderBy> order_by;
   std::optional<size_t> limit;
-  std::optional<Filter> filter;
+
+  // Return only the properties of the vertices or edges that the filter predicate
+  // evaluates to true
+  std::optional<std::string> filter;
+};
+
+struct GetPropertiesResultRow {
+  VertexId vertex;
+  std::optional<EdgeId> edge;
+
+  std::vector<std::pair<PropertyId, Value>> props;
+  std::vector<Value> evaluated_expressions;
 };
 
 struct GetPropertiesResponse {
-  bool success;
+  std::vector<GetPropertiesResultRow> result_row;
+  std::optional<ShardError> error;
 };
 
 enum class EdgeDirection : uint8_t { OUT = 1, IN = 2, BOTH = 3 };
@@ -403,7 +419,9 @@ struct ExpandOneRequest {
   std::vector<std::string> vertex_expressions;
   std::vector<std::string> edge_expressions;
 
-  std::optional<std::vector<OrderBy>> order_by;
+  std::vector<OrderBy> order_by_vertices;
+  std::vector<OrderBy> order_by_edges;
+
   // Limit the edges or the vertices?
   std::optional<size_t> limit;
   std::vector<std::string> filters;
@@ -446,14 +464,16 @@ struct ExpandOneResultRow {
 };
 
 struct ExpandOneResponse {
-  bool success;
+  std::optional<ShardError> error;
   std::vector<ExpandOneResultRow> result;
 };
 
-struct UpdateVertexProp {
+struct UpdateVertex {
   PrimaryKey primary_key;
-  // This should be a map
+  // Labels are first added and then removed from vertices
-  std::vector<std::pair<PropertyId, Value>> property_updates;
+  std::vector<LabelId> add_labels;
+  std::vector<LabelId> remove_labels;
+  std::map<PropertyId, Value> property_updates;
 };
 
 struct UpdateEdgeProp {
@@ -480,7 +500,7 @@ struct CreateVerticesRequest {
 };
 
 struct CreateVerticesResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 struct DeleteVerticesRequest {
@@ -491,16 +511,16 @@ struct DeleteVerticesRequest {
 };
 
 struct DeleteVerticesResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 struct UpdateVerticesRequest {
   Hlc transaction_id;
-  std::vector<UpdateVertexProp> new_properties;
+  std::vector<UpdateVertex> update_vertices;
 };
 
 struct UpdateVerticesResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 /*
@@ -522,7 +542,7 @@ struct CreateExpandRequest {
 };
 
 struct CreateExpandResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 struct DeleteEdgesRequest {
@@ -531,7 +551,7 @@ struct DeleteEdgesRequest {
 };
 
 struct DeleteEdgesResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 struct UpdateEdgesRequest {
@@ -540,7 +560,7 @@ struct UpdateEdgesRequest {
 };
 
 struct UpdateEdgesResponse {
-  bool success;
+  std::optional<ShardError> error;
 };
 
 struct CommitRequest {
@@ -549,7 +569,17 @@ struct CommitRequest {
 };
 
 struct CommitResponse {
-  bool success;
+  std::optional<ShardError> error;
+};
+
+struct SplitInfo {
+  PrimaryKey split_key;
+  uint64_t shard_version;
+};
+
+struct PerformSplitDataInfo {
+  PrimaryKey split_key;
+  uint64_t shard_version;
 };
 
 using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
@@ -561,3 +591,48 @@ using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesRespon
                                     CreateExpandResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>;
 
 }  // namespace memgraph::msgs
+
+namespace std {
+
+template <>
+struct hash<memgraph::msgs::Value>;
+
+template <>
+struct hash<memgraph::msgs::VertexId> {
+  size_t operator()(const memgraph::msgs::VertexId &id) const {
+    using LabelId = memgraph::storage::v3::LabelId;
+    using Value = memgraph::msgs::Value;
+    return memgraph::utils::HashCombine<LabelId, std::vector<Value>, std::hash<LabelId>,
+                                        memgraph::utils::FnvCollection<std::vector<Value>, Value>>{}(id.first.id,
+                                                                                                     id.second);
+  }
+};
+
+template <>
+struct hash<memgraph::msgs::Value> {
+  size_t operator()(const memgraph::msgs::Value &value) const {
+    using Type = memgraph::msgs::Value::Type;
+    switch (value.type) {
+      case Type::Null:
+        return std::hash<size_t>{}(0U);
+      case Type::Bool:
+        return std::hash<bool>{}(value.bool_v);
+      case Type::Int64:
+        return std::hash<int64_t>{}(value.int_v);
+      case Type::Double:
+        return std::hash<double>{}(value.double_v);
+      case Type::String:
+        return std::hash<std::string>{}(value.string_v);
+      case Type::List:
+        LOG_FATAL("Add hash for lists");
+      case Type::Map:
+        LOG_FATAL("Add hash for maps");
+      case Type::Vertex:
+        LOG_FATAL("Add hash for vertices");
+      case Type::Edge:
+        LOG_FATAL("Add hash for edges");
+    }
+  }
+};
+
+}  // namespace std

src/query/v2/shard_request_manager.hpp

@@ -1,730 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <chrono>
-#include <deque>
-#include <iostream>
-#include <iterator>
-#include <map>
-#include <numeric>
-#include <optional>
-#include <random>
-#include <set>
-#include <stdexcept>
-#include <thread>
-#include <unordered_map>
-#include <vector>
-
-#include "coordinator/coordinator.hpp"
-#include "coordinator/coordinator_client.hpp"
-#include "coordinator/coordinator_rsm.hpp"
-#include "coordinator/shard_map.hpp"
-#include "io/address.hpp"
-#include "io/errors.hpp"
-#include "io/rsm/raft.hpp"
-#include "io/rsm/rsm_client.hpp"
-#include "io/rsm/shard_rsm.hpp"
-#include "io/simulator/simulator.hpp"
-#include "io/simulator/simulator_transport.hpp"
-#include "query/v2/accessors.hpp"
-#include "query/v2/requests.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/value_conversions.hpp"
-#include "utils/result.hpp"
-
-namespace memgraph::msgs {
-template <typename TStorageClient>
-class RsmStorageClientManager {
- public:
-  using CompoundKey = memgraph::io::rsm::ShardRsmKey;
-  using Shard = memgraph::coordinator::Shard;
-  using LabelId = memgraph::storage::v3::LabelId;
-  RsmStorageClientManager() = default;
-  RsmStorageClientManager(const RsmStorageClientManager &) = delete;
-  RsmStorageClientManager(RsmStorageClientManager &&) = delete;
-  RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
-  RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
-  ~RsmStorageClientManager() = default;
-
-  void AddClient(Shard key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
-
-  bool Exists(const Shard &key) { return cli_cache_.contains(key); }
-
-  void PurgeCache() { cli_cache_.clear(); }
-
-  TStorageClient &GetClient(const Shard &key) {
-    auto it = cli_cache_.find(key);
-    MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
-    return it->second;
-  }
-
- private:
-  std::map<Shard, TStorageClient> cli_cache_;
-};
-
-template <typename TRequest>
-struct ExecutionState {
-  using CompoundKey = memgraph::io::rsm::ShardRsmKey;
-  using Shard = memgraph::coordinator::Shard;
-
-  enum State : int8_t { INITIALIZING, EXECUTING, COMPLETED };
-  // label is optional because some operators can create/remove etc, vertices. These kind of requests contain the label
-  // on the request itself.
-  std::optional<std::string> label;
-  // CompoundKey is optional because some operators require to iterate over all the available keys
-  // of a shard. One example is ScanAll, where we only require the field label.
-  std::optional<CompoundKey> key;
-  // Transaction id to be filled by the ShardRequestManager implementation
-  memgraph::coordinator::Hlc transaction_id;
-  // Initialized by ShardRequestManager implementation. This vector is filled with the shards that
-  // the ShardRequestManager impl will send requests to. When a request to a shard exhausts it, meaning that
-  // it pulled all the requested data from the given Shard, it will be removed from the Vector. When the Vector becomes
-  // empty, it means that all of the requests have completed succefully.
-  // TODO(gvolfing)
-  // Maybe make this into a more complex object to be able to keep track of paginated resutls. E.g. instead of a vector
-  // of Shards make it into a std::vector<std::pair<Shard, PaginatedResultType>> (probably a struct instead of a pair)
-  // where PaginatedResultType is an enum signaling the progress on the given request. This way we can easily check if
-  // a partial response on a shard(if there is one) is finished and we can send off the request for the next batch.
-  std::vector<Shard> shard_cache;
-  // 1-1 mapping with `shard_cache`.
-  // A vector that tracks request metadata for each shard (For example, next_id for a ScanAll on Shard A)
-  std::vector<TRequest> requests;
-  State state = INITIALIZING;
-};
-
-class ShardRequestManagerInterface {
- public:
-  using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
-  ShardRequestManagerInterface() = default;
-  ShardRequestManagerInterface(const ShardRequestManagerInterface &) = delete;
-  ShardRequestManagerInterface(ShardRequestManagerInterface &&) = delete;
-  ShardRequestManagerInterface &operator=(const ShardRequestManagerInterface &) = delete;
-  ShardRequestManagerInterface &&operator=(ShardRequestManagerInterface &&) = delete;
-
-  virtual ~ShardRequestManagerInterface() = default;
-
-  virtual void StartTransaction() = 0;
-  virtual void Commit() = 0;
-  virtual std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) = 0;
-  virtual std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
-                                                      std::vector<NewVertex> new_vertices) = 0;
-  virtual std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state,
-                                                  ExpandOneRequest request) = 0;
-  virtual std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
-                                                    std::vector<NewExpand> new_edges) = 0;
-
-  virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
-  virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
-  virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
-  virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
-  virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
-  virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
-  virtual bool IsPrimaryLabel(LabelId label) const = 0;
-  virtual bool IsPrimaryKey(LabelId primary_label, PropertyId property) const = 0;
-};
-
-// TODO(kostasrim)rename this class template
-template <typename TTransport>
-class ShardRequestManager : public ShardRequestManagerInterface {
- public:
-  using StorageClient =
-      memgraph::coordinator::RsmClient<TTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
-  using CoordinatorWriteRequests = memgraph::coordinator::CoordinatorWriteRequests;
-  using CoordinatorClient = memgraph::coordinator::CoordinatorClient<TTransport>;
-  using Address = memgraph::io::Address;
-  using Shard = memgraph::coordinator::Shard;
-  using ShardMap = memgraph::coordinator::ShardMap;
-  using CompoundKey = memgraph::coordinator::PrimaryKey;
-  using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
-  ShardRequestManager(CoordinatorClient coord, memgraph::io::Io<TTransport> &&io)
-      : coord_cli_(std::move(coord)), io_(std::move(io)) {}
-
-  ShardRequestManager(const ShardRequestManager &) = delete;
-  ShardRequestManager(ShardRequestManager &&) = delete;
-  ShardRequestManager &operator=(const ShardRequestManager &) = delete;
-  ShardRequestManager &operator=(ShardRequestManager &&) = delete;
-
-  ~ShardRequestManager() override {}
-
-  void StartTransaction() override {
-    memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
-    CoordinatorWriteRequests write_req = req;
-    auto write_res = coord_cli_.SendWriteRequest(write_req);
-    if (write_res.HasError()) {
-      throw std::runtime_error("HLC request failed");
-    }
-    auto coordinator_write_response = write_res.GetValue();
-    auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
-
-    // Transaction ID to be used later...
-    transaction_id_ = hlc_response.new_hlc;
-
-    if (hlc_response.fresher_shard_map) {
-      shards_map_ = hlc_response.fresher_shard_map.value();
-      SetUpNameIdMappers();
-    }
-  }
-
-  void Commit() override {
-    memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
-    CoordinatorWriteRequests write_req = req;
-    auto write_res = coord_cli_.SendWriteRequest(write_req);
-    if (write_res.HasError()) {
-      throw std::runtime_error("HLC request for commit failed");
-    }
-    auto coordinator_write_response = write_res.GetValue();
-    auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
-
-    if (hlc_response.fresher_shard_map) {
-      shards_map_ = hlc_response.fresher_shard_map.value();
-      SetUpNameIdMappers();
-    }
-    auto commit_timestamp = hlc_response.new_hlc;
-
-    msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
-
-    for (const auto &[label, space] : shards_map_.label_spaces) {
-      for (const auto &[key, shard] : space.shards) {
-        auto &storage_client = GetStorageClientForShard(shard);
-        // TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
-        // instead.
-        auto commit_response = storage_client.SendWriteRequest(commit_req);
-        // RETRY on timeouts?
-        // Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
-        if (commit_response.HasError()) {
-          throw std::runtime_error("Commit request timed out");
-        }
-        WriteResponses write_response_variant = commit_response.GetValue();
-        auto &response = std::get<CommitResponse>(write_response_variant);
-        if (!response.success) {
-          throw std::runtime_error("Commit request did not succeed");
-        }
-      }
-    }
-  }
-
-  storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
-    return shards_map_.GetEdgeTypeId(name).value();
-  }
-
-  storage::v3::PropertyId NameToProperty(const std::string &name) const override {
-    return shards_map_.GetPropertyId(name).value();
-  }
-
-  storage::v3::LabelId NameToLabel(const std::string &name) const override {
-    return shards_map_.GetLabelId(name).value();
-  }
-
-  const std::string &PropertyToName(memgraph::storage::v3::PropertyId id) const override {
-    return properties_.IdToName(id.AsUint());
-  }
-  const std::string &LabelToName(memgraph::storage::v3::LabelId id) const override {
-    return labels_.IdToName(id.AsUint());
-  }
-  const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId id) const override {
-    return edge_types_.IdToName(id.AsUint());
-  }
-
-  bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override {
-    const auto schema_it = shards_map_.schemas.find(primary_label);
-    MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
-
-    return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
-             return schema_prop.property_id == property;
-           }) != schema_it->second.end();
-  }
-
-  bool IsPrimaryLabel(LabelId label) const override { return shards_map_.label_spaces.contains(label); }
-
-  // TODO(kostasrim) Simplify return result
-  std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) override {
-    MaybeInitializeExecutionState(state);
-    std::vector<ScanVerticesResponse> responses;
-
-    SendAllRequests(state);
-    auto all_requests_gathered = [](auto &paginated_rsp_tracker) {
-      return std::ranges::all_of(paginated_rsp_tracker, [](const auto &state) {
-        return state.second == PaginatedResponseState::PartiallyFinished;
-      });
-    };
-
-    std::map<Shard, PaginatedResponseState> paginated_response_tracker;
-    for (const auto &shard : state.shard_cache) {
-      paginated_response_tracker.insert(std::make_pair(shard, PaginatedResponseState::Pending));
-    }
-
-    do {
-      AwaitOnPaginatedRequests(state, responses, paginated_response_tracker);
-    } while (!all_requests_gathered(paginated_response_tracker));
-
-    MaybeCompleteState(state);
-    // TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
-    // result of storage_client.SendReadRequest()).
-    return PostProcess(std::move(responses));
-  }
-
-  std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
-                                              std::vector<NewVertex> new_vertices) override {
-    MG_ASSERT(!new_vertices.empty());
-    MaybeInitializeExecutionState(state, new_vertices);
-    std::vector<CreateVerticesResponse> responses;
-    auto &shard_cache_ref = state.shard_cache;
-
-    // 1. Send the requests.
-    SendAllRequests(state, shard_cache_ref);
-
-    // 2. Block untill all the futures are exhausted
-    do {
-      AwaitOnResponses(state, responses);
-    } while (!state.shard_cache.empty());
-
-    MaybeCompleteState(state);
-    // TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
-    // result of storage_client.SendReadRequest()).
-    return responses;
-  }
-
-  std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
-                                            std::vector<NewExpand> new_edges) override {
-    MG_ASSERT(!new_edges.empty());
-    MaybeInitializeExecutionState(state, new_edges);
-    std::vector<CreateExpandResponse> responses;
-    auto &shard_cache_ref = state.shard_cache;
-    size_t id{0};
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-      WriteRequests req = state.requests[id];
-      auto write_response_result = storage_client.SendWriteRequest(std::move(req));
-      if (write_response_result.HasError()) {
-        throw std::runtime_error("CreateVertices request timedout");
-      }
-      WriteResponses response_variant = write_response_result.GetValue();
-      CreateExpandResponse mapped_response = std::get<CreateExpandResponse>(response_variant);
-
-      if (!mapped_response.success) {
-        throw std::runtime_error("CreateExpand request did not succeed");
-      }
-      responses.push_back(mapped_response);
-      shard_it = shard_cache_ref.erase(shard_it);
-    }
-    // We are done with this state
-    MaybeCompleteState(state);
-    return responses;
-  }
-
-  std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) override {
-    // TODO(kostasrim)Update to limit the batch size here
-    // Expansions of the destination must be handled by the caller. For example
-    // match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
-    // For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
-    // must be fetched again with an ExpandOne(Edges.dst)
-    MaybeInitializeExecutionState(state, std::move(request));
-    std::vector<ExpandOneResponse> responses;
-    auto &shard_cache_ref = state.shard_cache;
-
-    // 1. Send the requests.
-    SendAllRequests(state, shard_cache_ref);
-
-    // 2. Block untill all the futures are exhausted
-    do {
-      AwaitOnResponses(state, responses);
-    } while (!state.shard_cache.empty());
-    std::vector<ExpandOneResultRow> result_rows;
-    const auto total_row_count = std::accumulate(
-        responses.begin(), responses.end(), 0,
-        [](const int64_t partial_count, const ExpandOneResponse &resp) { return partial_count + resp.result.size(); });
-    result_rows.reserve(total_row_count);
-
-    for (auto &response : responses) {
-      result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
-                         std::make_move_iterator(response.result.end()));
-    }
-    MaybeCompleteState(state);
-    return result_rows;
-  }
-
- private:
-  enum class PaginatedResponseState { Pending, PartiallyFinished };
-
-  std::vector<VertexAccessor> PostProcess(std::vector<ScanVerticesResponse> &&responses) const {
-    std::vector<VertexAccessor> accessors;
-    for (auto &response : responses) {
-      for (auto &result_row : response.results) {
-        accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
-      }
-    }
-    return accessors;
-  }
-
-  template <typename ExecutionState>
-  void ThrowIfStateCompleted(ExecutionState &state) const {
-    if (state.state == ExecutionState::COMPLETED) [[unlikely]] {
-      throw std::runtime_error("State is completed and must be reset");
-    }
-  }
-
-  template <typename ExecutionState>
-  void MaybeCompleteState(ExecutionState &state) const {
-    if (state.requests.empty()) {
-      state.state = ExecutionState::COMPLETED;
-    }
-  }
-
-  template <typename ExecutionState>
-  bool ShallNotInitializeState(ExecutionState &state) const {
-    return state.state != ExecutionState::INITIALIZING;
-  }
-
-  void MaybeInitializeExecutionState(ExecutionState<CreateVerticesRequest> &state,
-                                     std::vector<NewVertex> new_vertices) {
-    ThrowIfStateCompleted(state);
-    if (ShallNotInitializeState(state)) {
-      return;
-    }
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, CreateVerticesRequest> per_shard_request_table;
-
-    for (auto &new_vertex : new_vertices) {
-      MG_ASSERT(!new_vertex.label_ids.empty(), "This is error!");
-      auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
-                                              storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
-      if (!per_shard_request_table.contains(shard)) {
-        CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
-        per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
-        state.shard_cache.push_back(shard);
-      }
-      per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
-    }
-
-    for (auto &[shard, rqst] : per_shard_request_table) {
-      state.requests.push_back(std::move(rqst));
-    }
-    state.state = ExecutionState<CreateVerticesRequest>::EXECUTING;
-  }
-
-  void MaybeInitializeExecutionState(ExecutionState<CreateExpandRequest> &state, std::vector<NewExpand> new_expands) {
-    ThrowIfStateCompleted(state);
-    if (ShallNotInitializeState(state)) {
-      return;
-    }
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, CreateExpandRequest> per_shard_request_table;
-    auto ensure_shard_exists_in_table = [&per_shard_request_table,
-                                         transaction_id = transaction_id_](const Shard &shard) {
-      if (!per_shard_request_table.contains(shard)) {
-        CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
-        per_shard_request_table.insert({shard, std::move(create_expand_request)});
-      }
-    };
-
-    for (auto &new_expand : new_expands) {
-      const auto shard_src_vertex = shards_map_.GetShardForKey(
-          new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
-      const auto shard_dest_vertex = shards_map_.GetShardForKey(
-          new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
-
-      ensure_shard_exists_in_table(shard_src_vertex);
-
-      if (shard_src_vertex != shard_dest_vertex) {
-        ensure_shard_exists_in_table(shard_dest_vertex);
-        per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
-      }
-      per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
-    }
-
-    for (auto &[shard, request] : per_shard_request_table) {
-      state.shard_cache.push_back(shard);
-      state.requests.push_back(std::move(request));
-    }
-    state.state = ExecutionState<CreateExpandRequest>::EXECUTING;
-  }
-
-  void MaybeInitializeExecutionState(ExecutionState<ScanVerticesRequest> &state) {
-    ThrowIfStateCompleted(state);
-    if (ShallNotInitializeState(state)) {
-      return;
-    }
-
-    std::vector<coordinator::Shards> multi_shards;
-    state.transaction_id = transaction_id_;
-    if (!state.label) {
-      multi_shards = shards_map_.GetAllShards();
-    } else {
-      const auto label_id = shards_map_.GetLabelId(*state.label);
-      MG_ASSERT(label_id);
-      MG_ASSERT(IsPrimaryLabel(*label_id));
-      multi_shards = {shards_map_.GetShardsForLabel(*state.label)};
-    }
-    for (auto &shards : multi_shards) {
-      for (auto &[key, shard] : shards) {
-        MG_ASSERT(!shard.empty());
-        state.shard_cache.push_back(std::move(shard));
-        ScanVerticesRequest rqst;
-        rqst.transaction_id = transaction_id_;
-        rqst.start_id.second = storage::conversions::ConvertValueVector(key);
-        state.requests.push_back(std::move(rqst));
-      }
-    }
-    state.state = ExecutionState<ScanVerticesRequest>::EXECUTING;
-  }
-
-  void MaybeInitializeExecutionState(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) {
-    ThrowIfStateCompleted(state);
-    if (ShallNotInitializeState(state)) {
-      return;
-    }
-    state.transaction_id = transaction_id_;
-
-    std::map<Shard, ExpandOneRequest> per_shard_request_table;
-    auto top_level_rqst_template = request;
-    top_level_rqst_template.transaction_id = transaction_id_;
-    top_level_rqst_template.src_vertices.clear();
-    state.requests.clear();
-    for (auto &vertex : request.src_vertices) {
-      auto shard =
-          shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
-      if (!per_shard_request_table.contains(shard)) {
-        per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
-        state.shard_cache.push_back(shard);
-      }
-      per_shard_request_table[shard].src_vertices.push_back(vertex);
-    }
-
-    for (auto &[shard, rqst] : per_shard_request_table) {
-      state.requests.push_back(std::move(rqst));
-    }
-    state.state = ExecutionState<ExpandOneRequest>::EXECUTING;
-  }
-
-  StorageClient &GetStorageClientForShard(Shard shard) {
-    if (!storage_cli_manager_.Exists(shard)) {
-      AddStorageClientToManager(shard);
-    }
-    return storage_cli_manager_.GetClient(shard);
-  }
-
-  StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
-    auto shard = shards_map_.GetShardForKey(label, key);
-    return GetStorageClientForShard(std::move(shard));
-  }
-
-  void AddStorageClientToManager(Shard target_shard) {
-    MG_ASSERT(!target_shard.empty());
-    auto leader_addr = target_shard.front();
-    std::vector<Address> addresses;
-    addresses.reserve(target_shard.size());
-    for (auto &address : target_shard) {
-      addresses.push_back(std::move(address.address));
-    }
-    auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
-    storage_cli_manager_.AddClient(target_shard, std::move(cli));
-  }
-
-  void SendAllRequests(ExecutionState<ScanVerticesRequest> &state) {
-    int64_t shard_idx = 0;
-    for (const auto &request : state.requests) {
-      const auto &current_shard = state.shard_cache[shard_idx];
-
-      auto &storage_client = GetStorageClientForShard(current_shard);
-      ReadRequests req = request;
-      storage_client.SendAsyncReadRequest(request);
-
-      ++shard_idx;
-    }
-  }
-
-  void SendAllRequests(ExecutionState<CreateVerticesRequest> &state,
-                       std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
-    size_t id = 0;
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
-      // This is fine because all new_vertices of each request end up on the same shard
-      const auto labels = state.requests[id].new_vertices[0].label_ids;
-      auto req_deep_copy = state.requests[id];
-
-      for (auto &new_vertex : req_deep_copy.new_vertices) {
-        new_vertex.label_ids.erase(new_vertex.label_ids.begin());
-      }
-
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-
-      WriteRequests req = req_deep_copy;
-      storage_client.SendAsyncWriteRequest(req);
-      ++id;
-    }
-  }
-
-  void SendAllRequests(ExecutionState<ExpandOneRequest> &state,
-                       std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
-    size_t id = 0;
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-      ReadRequests req = state.requests[id];
-      storage_client.SendAsyncReadRequest(req);
-      ++id;
-    }
-  }
-
-  void AwaitOnResponses(ExecutionState<CreateVerticesRequest> &state, std::vector<CreateVerticesResponse> &responses) {
-    auto &shard_cache_ref = state.shard_cache;
-    int64_t request_idx = 0;
-
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
-      // This is fine because all new_vertices of each request end up on the same shard
-      const auto labels = state.requests[request_idx].new_vertices[0].label_ids;
-
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-
-      auto poll_result = storage_client.AwaitAsyncWriteRequest();
-      if (!poll_result) {
-        ++shard_it;
-        ++request_idx;
-
-        continue;
-      }
-
-      if (poll_result->HasError()) {
-        throw std::runtime_error("CreateVertices request timed out");
-      }
-
-      WriteResponses response_variant = poll_result->GetValue();
-      auto response = std::get<CreateVerticesResponse>(response_variant);
-
-      if (!response.success) {
-        throw std::runtime_error("CreateVertices request did not succeed");
-      }
-      responses.push_back(response);
-
-      shard_it = shard_cache_ref.erase(shard_it);
-      // Needed to maintain the 1-1 mapping between the ShardCache and the requests.
-      auto it = state.requests.begin() + request_idx;
-      state.requests.erase(it);
-    }
-  }
-
-  void AwaitOnResponses(ExecutionState<ExpandOneRequest> &state, std::vector<ExpandOneResponse> &responses) {
-    auto &shard_cache_ref = state.shard_cache;
-    int64_t request_idx = 0;
-
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-
-      auto poll_result = storage_client.PollAsyncReadRequest();
-      if (!poll_result) {
-        ++shard_it;
-        ++request_idx;
-        continue;
-      }
-
-      if (poll_result->HasError()) {
-        throw std::runtime_error("ExpandOne request timed out");
-      }
-
-      ReadResponses response_variant = poll_result->GetValue();
-      auto response = std::get<ExpandOneResponse>(response_variant);
-      // -NOTE-
-      // Currently a boolean flag for signaling the overall success of the
-      // ExpandOne request does not exist. But it should, so here we assume
-      // that it is already in place.
-      if (!response.success) {
-        throw std::runtime_error("ExpandOne request did not succeed");
-      }
-
-      responses.push_back(std::move(response));
-      shard_it = shard_cache_ref.erase(shard_it);
-      // Needed to maintain the 1-1 mapping between the ShardCache and the requests.
-      auto it = state.requests.begin() + request_idx;
-      state.requests.erase(it);
-    }
-  }
-
-  void AwaitOnPaginatedRequests(ExecutionState<ScanVerticesRequest> &state,
-                                std::vector<ScanVerticesResponse> &responses,
-                                std::map<Shard, PaginatedResponseState> &paginated_response_tracker) {
-    auto &shard_cache_ref = state.shard_cache;
-
-    // Find the first request that is not holding a paginated response.
-    int64_t request_idx = 0;
-    for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
-      if (paginated_response_tracker.at(*shard_it) != PaginatedResponseState::Pending) {
-        ++shard_it;
-        ++request_idx;
-        continue;
-      }
-
-      auto &storage_client = GetStorageClientForShard(*shard_it);
-
-      auto await_result = storage_client.AwaitAsyncReadRequest();
-
-      if (!await_result) {
-        // Redirection has occured.
-        ++shard_it;
-        ++request_idx;
-        continue;
-      }
-
-      if (await_result->HasError()) {
-        throw std::runtime_error("ScanAll request timed out");
-      }
-
-      ReadResponses read_response_variant = await_result->GetValue();
-      auto response = std::get<ScanVerticesResponse>(read_response_variant);
-      if (!response.success) {
-        throw std::runtime_error("ScanAll request did not succeed");
-      }
-
-      if (!response.next_start_id) {
-        paginated_response_tracker.erase((*shard_it));
-        shard_cache_ref.erase(shard_it);
-        // Needed to maintain the 1-1 mapping between the ShardCache and the requests.
-        auto it = state.requests.begin() + request_idx;
-        state.requests.erase(it);
-
-      } else {
-        state.requests[request_idx].start_id.second = response.next_start_id->second;
-        paginated_response_tracker[*shard_it] = PaginatedResponseState::PartiallyFinished;
-      }
-      responses.push_back(std::move(response));
-    }
-  }
-
-  void SetUpNameIdMappers() {
-    std::unordered_map<uint64_t, std::string> id_to_name;
-    for (const auto &[name, id] : shards_map_.labels) {
-      id_to_name.emplace(id.AsUint(), name);
-    }
-    labels_.StoreMapping(std::move(id_to_name));
-    id_to_name.clear();
-    for (const auto &[name, id] : shards_map_.properties) {
-      id_to_name.emplace(id.AsUint(), name);
-    }
-    properties_.StoreMapping(std::move(id_to_name));
-    id_to_name.clear();
-    for (const auto &[name, id] : shards_map_.edge_types) {
-      id_to_name.emplace(id.AsUint(), name);
-    }
-    edge_types_.StoreMapping(std::move(id_to_name));
-  }
-
-  ShardMap shards_map_;
-  storage::v3::NameIdMapper properties_;
-  storage::v3::NameIdMapper edge_types_;
-  storage::v3::NameIdMapper labels_;
-  CoordinatorClient coord_cli_;
-  RsmStorageClientManager<StorageClient> storage_cli_manager_;
-  memgraph::io::Io<TTransport> io_;
-  memgraph::coordinator::Hlc transaction_id_;
-  // TODO(kostasrim) Add batch prefetching
-};
-}  // namespace memgraph::msgs

src/storage/v2/constraints.cpp

@@ -97,15 +97,15 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
         if (delta->label == label) {
           MG_ASSERT(!has_label, "Invalid database state!");
           has_label = true;
-          break;
         }
+        break;
       }
       case Delta::Action::REMOVE_LABEL: {
         if (delta->label == label) {
           MG_ASSERT(has_label, "Invalid database state!");
           has_label = false;
-          break;
         }
+        break;
       }
       case Delta::Action::ADD_IN_EDGE:
       case Delta::Action::ADD_OUT_EDGE:

src/storage/v3/CMakeLists.txt

@@ -9,19 +9,17 @@ set(storage_v3_src_files
     temporal.cpp
     edge_accessor.cpp
     indices.cpp
-    key_store.cpp
-    lexicographically_ordered_vertex.cpp
     property_store.cpp
     vertex_accessor.cpp
     schemas.cpp
     schema_validator.cpp
     shard.cpp
-    storage.cpp
     shard_rsm.cpp
     bindings/typed_value.cpp
     expr.cpp
-    request_helper.cpp
+    vertex.cpp
-    storage.cpp)
+    splitter.cpp
+    request_helper.cpp)
 
 # ######################
 find_package(gflags REQUIRED)
@@ -33,4 +31,4 @@ target_link_libraries(mg-storage-v3 Threads::Threads mg-utils gflags)
 target_include_directories(mg-storage-v3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
 
 add_dependencies(mg-storage-v3 generate_lcp_storage)
-target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io)
+target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io mg-functions)

src/storage/v3/bindings/ast/ast.lcp

@@ -24,6 +24,8 @@
 #include "storage/v3/conversions.hpp"
 #include "storage/v3/path.hpp"
 #include "utils/typeinfo.hpp"
+#include "utils/exceptions.hpp"
+#include "functions/awesome_memgraph_functions.hpp"
 
 cpp<#
 
@@ -178,14 +180,6 @@ cpp<#
   (:serialize (:slk :load-args '((storage "storage::v3::AstStorage *")))))
 
 #>cpp
-struct FunctionContext {
-  DbAccessor *db_accessor;
-  utils::MemoryResource *memory;
-  int64_t timestamp;
-  std::unordered_map<std::string, int64_t> *counters;
-  View view;
-};
-
 inline bool operator==(const LabelIx &a, const LabelIx &b) {
   return a.ix == b.ix && a.name == b.name;
 }
@@ -845,16 +839,24 @@ cpp<#
               :slk-load (slk-load-ast-vector "Expression"))
    (function-name "std::string" :scope :public)
    (function "std::function<TypedValue(const TypedValue *, int64_t,
-                                       const FunctionContext &)>"
+                                       const functions::FunctionContext<memgraph::storage::v3::DbAccessor> &)>"
              :scope :public
              :dont-save t
              :clone :copy
              :slk-load (lambda (member)
                         #>cpp
+                        self->${member} = functions::NameToFunction<memgraph::storage::v3::TypedValue,
+                                            functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
+                                            functions::StorageEngineTag, Conv>(self->function_name_);
                         cpp<#)))
   (:public
     #>cpp
     Function() = default;
+    using Conv = decltype(PropertyToTypedValueFunctor<TypedValue>);
+    class SemanticException : public memgraph::utils::BasicException {
+      using utils::BasicException::BasicException;
+    };
+
 
     DEFVISITABLE(ExpressionVisitor<TypedValue>);
     DEFVISITABLE(ExpressionVisitor<void>);
@@ -872,7 +874,13 @@ cpp<#
     Function(const std::string &function_name,
              const std::vector<Expression *> &arguments)
         : arguments_(arguments),
-          function_name_(function_name) {
+          function_name_(function_name),
+          function_(functions::NameToFunction<memgraph::storage::v3::TypedValue,
+                                            functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
+                                            functions::StorageEngineTag, Conv>(function_name_)) {
+      if (!function_) {
+        throw SemanticException("Function '{}' doesn't exist.", function_name);
+      }
     }
     cpp<#)
   (:private

src/storage/v3/bindings/db_accessor.hpp

@@ -78,8 +78,8 @@ class DbAccessor final {
     return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
   }
 
-  storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
+  storage::v3::ShardResult<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
-                                               const storage::v3::EdgeTypeId &edge_type) {
+                                                    const storage::v3::EdgeTypeId &edge_type) {
     static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
     auto maybe_edge = accessor_->CreateEdge(from->Id(storage::v3::View::NEW).GetValue(),
                                             to->Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
@@ -87,8 +87,8 @@ class DbAccessor final {
     return EdgeAccessor(*maybe_edge);
   }
 
-  storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
+  storage::v3::ShardResult<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
-    auto res = accessor_->DeleteEdge(edge->FromVertex(), edge->ToVertex(), edge->Gid());
+    auto res = accessor_->DeleteEdge(edge->From(), edge->To(), edge->Gid());
     if (res.HasError()) {
       return res.GetError();
     }
@@ -101,7 +101,7 @@ class DbAccessor final {
     return std::make_optional<EdgeAccessor>(*value);
   }
 
-  storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
+  storage::v3::ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
       VertexAccessor *vertex_accessor) {
     using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
 
@@ -125,7 +125,7 @@ class DbAccessor final {
     return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
   }
 
-  storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
+  storage::v3::ShardResult<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
     auto res = accessor_->DeleteVertex(vertex_accessor);
     if (res.HasError()) {
       return res.GetError();

src/storage/v3/bindings/eval.hpp

@@ -21,6 +21,7 @@
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_store.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/result.hpp"
 #include "storage/v3/view.hpp"
 #include "utils/memory.hpp"
 
@@ -87,6 +88,6 @@ struct EvaluationContext {
 using ExpressionEvaluator =
     memgraph::expr::ExpressionEvaluator<TypedValue, EvaluationContext, DbAccessor, storage::v3::View,
                                         storage::v3::LabelId, storage::v3::PropertyStore, PropertyToTypedValueConverter,
-                                        memgraph::storage::v3::Error>;
+                                        common::ErrorCode>;
 
 }  // namespace memgraph::storage::v3

src/storage/v3/bindings/frame.hpp

@@ -17,5 +17,5 @@
 #include "storage/v3/bindings/typed_value.hpp"
 
 namespace memgraph::storage::v3 {
-using Frame = memgraph::expr::Frame<TypedValue>;
+using Frame = memgraph::expr::Frame;
 }  // namespace memgraph::storage::v3

src/storage/v3/config.hpp

@@ -30,6 +30,10 @@ struct Config {
     io::Duration reclamation_interval{};
   } gc;
 
+  struct Split {
+    uint64_t max_shard_vertex_size{500'000};
+  } split;
+
   struct Items {
     bool properties_on_edges{true};
   } items;

src/storage/v3/conversions.hpp

@@ -69,6 +69,10 @@ TTypedValue PropertyToTypedValue(const PropertyValue &value) {
   LOG_FATAL("Unsupported type");
 }
 
+template <typename TypedValueT>
+inline const auto PropertyToTypedValueFunctor =
+    [](const PropertyValue &value) { return PropertyToTypedValue<TypedValueT>(value); };
+
 template <typename TTypedValue>
 TTypedValue PropertyToTypedValue(const PropertyValue &value, utils::MemoryResource *mem) {
   switch (value.type()) {

src/storage/v3/delta.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,21 +11,29 @@
 
 #pragma once
 
+#include <cstdint>
 #include <memory>
+
 #include "storage/v3/edge_ref.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/vertex.hpp"
 #include "storage/v3/vertex_id.hpp"
 #include "utils/logging.hpp"
+#include "utils/synchronized.hpp"
 
 namespace memgraph::storage::v3 {
 
 // Forward declarations because we only store pointers here.
-struct Vertex;
 struct Edge;
 struct Delta;
 struct CommitInfo;
 
+inline uint64_t GetNextDeltaId() {
+  static utils::Synchronized<uint64_t, utils::SpinLock> delta_id{0};
+  return delta_id.WithLock([](auto &id) { return id++; });
+}
+
 // This class stores one of three pointers (`Delta`, `Vertex` and `Edge`)
 // without using additional memory for storing the type. The type is stored in
 // the pointer itself in the lower bits. All of those structures contain large
@@ -129,7 +137,7 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer
 inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { return !(a == b); }
 
 struct Delta {
-  enum class Action {
+  enum class Action : uint8_t {
     // Used for both Vertex and Edge
     DELETE_OBJECT,
     RECREATE_OBJECT,
@@ -157,46 +165,54 @@ struct Delta {
   struct RemoveInEdgeTag {};
   struct RemoveOutEdgeTag {};
 
-  Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
+  Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::DELETE_OBJECT), commit_info(commit_info), command_id(command_id) {}
+      : action(Action::DELETE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
 
-  Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
+  Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::RECREATE_OBJECT), commit_info(commit_info), command_id(command_id) {}
+      : action(Action::RECREATE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
 
-  Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
+  Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::ADD_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
+      : action(Action::ADD_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
 
-  Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
+  Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
-      : action(Action::REMOVE_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
+      : action(Action::REMOVE_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
 
   Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, CommitInfo *commit_info,
-        uint64_t command_id)
+        uint64_t delta_id, uint64_t command_id)
-      : action(Action::SET_PROPERTY), commit_info(commit_info), command_id(command_id), property({key, value}) {}
+      : action(Action::SET_PROPERTY),
+        id(delta_id),
+        commit_info(commit_info),
+        command_id(command_id),
+        property({key, value}) {}
 
   Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t command_id)
+        uint64_t delta_id, uint64_t command_id)
       : action(Action::ADD_IN_EDGE),
+        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t command_id)
+        uint64_t delta_id, uint64_t command_id)
       : action(Action::ADD_OUT_EDGE),
+        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t command_id)
+        uint64_t delta_id, uint64_t command_id)
       : action(Action::REMOVE_IN_EDGE),
+        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
 
   Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
-        uint64_t command_id)
+        uint64_t delta_id, uint64_t command_id)
       : action(Action::REMOVE_OUT_EDGE),
+        id(delta_id),
         commit_info(commit_info),
         command_id(command_id),
         vertex_edge({edge_type, std::move(vertex_id), edge}) {}
@@ -225,8 +241,10 @@ struct Delta {
     }
   }
 
-  Action action;
+  friend bool operator==(const Delta &lhs, const Delta &rhs) noexcept { return lhs.id == rhs.id; }
 
+  Action action;
+  uint64_t id;
   // TODO: optimize with in-place copy
   CommitInfo *commit_info;
   uint64_t command_id;

src/storage/v3/edge.hpp

@@ -21,7 +21,7 @@
 
 namespace memgraph::storage::v3 {
 
-struct Vertex;
+using EdgeContainer = std::map<Gid, Edge>;
 
 struct Edge {
   Edge(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {

src/storage/v3/edge_accessor.cpp

@@ -15,6 +15,7 @@
 
 #include "storage/v3/mvcc.hpp"
 #include "storage/v3/property_value.hpp"
+#include "storage/v3/result.hpp"
 #include "storage/v3/schema_validator.hpp"
 #include "storage/v3/vertex_accessor.hpp"
 #include "utils/memory_tracker.hpp"
@@ -50,17 +51,17 @@ bool EdgeAccessor::IsVisible(const View view) const {
   return exists && (for_deleted_ || !deleted);
 }
 
-const VertexId &EdgeAccessor::FromVertex() const { return from_vertex_; }
+const VertexId &EdgeAccessor::From() const { return from_vertex_; }
 
-const VertexId &EdgeAccessor::ToVertex() const { return to_vertex_; }
+const VertexId &EdgeAccessor::To() const { return to_vertex_; }
 
-Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
+ShardResult<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
-  if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
+  if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
 
-  if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
+  if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
+  if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
   auto current_value = edge_.ptr->properties.GetProperty(property);
   // We could skip setting the value if the previous one is the same to the new
@@ -75,12 +76,12 @@ Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const Prope
   return std::move(current_value);
 }
 
-Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
+ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
-  if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
+  if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
 
-  if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
+  if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
 
-  if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
+  if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
 
   auto properties = edge_.ptr->properties.Properties();
   for (const auto &property : properties) {
@@ -92,11 +93,11 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
   return std::move(properties);
 }
 
-Result<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
+ShardResult<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
   return GetProperty(property, view);
 }
 
-Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
+ShardResult<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
   if (!config_.properties_on_edges) return PropertyValue();
   auto exists = true;
   auto deleted = edge_.ptr->deleted;
@@ -128,12 +129,12 @@ Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view)
         break;
     }
   });
-  if (!exists) return Error::NONEXISTENT_OBJECT;
+  if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
-  if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
+  if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
   return std::move(value);
 }
 
-Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
+ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
   if (!config_.properties_on_edges) return std::map<PropertyId, PropertyValue>{};
   auto exists = true;
   auto deleted = edge_.ptr->deleted;
@@ -174,9 +175,12 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view)
         break;
     }
   });
-  if (!exists) return Error::NONEXISTENT_OBJECT;
+  if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
-  if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
+  if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
   return std::move(properties);
 }
 
+// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
+size_t EdgeAccessor::CypherId() const { return Gid().AsUint(); }
+
 }  // namespace memgraph::storage::v3

src/storage/v3/edge_accessor.hpp

@@ -25,7 +25,6 @@
 
 namespace memgraph::storage::v3 {
 
-struct Vertex;
 class VertexAccessor;
 struct Indices;
 
@@ -48,27 +47,27 @@ class EdgeAccessor final {
   /// @return true if the object is visible from the current transaction
   bool IsVisible(View view) const;
 
-  const VertexId &FromVertex() const;
+  const VertexId &From() const;
 
-  const VertexId &ToVertex() const;
+  const VertexId &To() const;
 
   EdgeTypeId EdgeType() const { return edge_type_; }
 
   /// Set a property value and return the old value.
   /// @throw std::bad_alloc
-  Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
+  ShardResult<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
 
   /// Remove all properties and return old values for each removed property.
   /// @throw std::bad_alloc
-  Result<std::map<PropertyId, PropertyValue>> ClearProperties();
+  ShardResult<std::map<PropertyId, PropertyValue>> ClearProperties();
 
   /// @throw std::bad_alloc
-  Result<PropertyValue> GetProperty(PropertyId property, View view) const;
+  ShardResult<PropertyValue> GetProperty(PropertyId property, View view) const;
 
-  Result<PropertyValue> GetProperty(View view, PropertyId property) const;
+  ShardResult<PropertyValue> GetProperty(View view, PropertyId property) const;
 
   /// @throw std::bad_alloc
-  Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
+  ShardResult<std::map<PropertyId, PropertyValue>> Properties(View view) const;
 
   Gid Gid() const noexcept {
     if (config_.properties_on_edges) {
@@ -84,6 +83,9 @@ class EdgeAccessor final {
   }
   bool operator!=(const EdgeAccessor &other) const noexcept { return !(*this == other); }
 
+  // Dummy function
+  size_t CypherId() const;
+
  private:
   EdgeRef edge_;
   EdgeTypeId edge_type_;

src/storage/v3/expr.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -35,7 +35,7 @@ msgs::Value ConstructValueVertex(const VertexAccessor &acc, View view) {
   memgraph::msgs::Label value_label{.id = prim_label};
 
   auto prim_key = conversions::ConvertValueVector(acc.PrimaryKey(view).GetValue());
-  memgraph::msgs::VertexId vertex_id = std::make_pair(value_label, prim_key);
+  memgraph::msgs::VertexId vertex_id = std::make_pair(value_label, std::move(prim_key));
 
   // Get the labels
   auto vertex_labels = acc.Labels(view).GetValue();
@@ -45,20 +45,19 @@ msgs::Value ConstructValueVertex(const VertexAccessor &acc, View view) {
   std::transform(vertex_labels.begin(), vertex_labels.end(), std::back_inserter(value_labels),
                  [](const auto &label) { return msgs::Label{.id = label}; });
 
-  return msgs::Value({.id = vertex_id, .labels = value_labels});
+  return msgs::Value({.id = std::move(vertex_id), .labels = std::move(value_labels)});
 }
 
 msgs::Value ConstructValueEdge(const EdgeAccessor &acc, View view) {
   msgs::EdgeType type = {.id = acc.EdgeType()};
   msgs::EdgeId gid = {.gid = acc.Gid().AsUint()};
 
-  msgs::Label src_prim_label = {.id = acc.FromVertex().primary_label};
+  msgs::Label src_prim_label = {.id = acc.From().primary_label};
   memgraph::msgs::VertexId src_vertex =
-      std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.FromVertex().primary_key));
+      std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.From().primary_key));
 
-  msgs::Label dst_prim_label = {.id = acc.ToVertex().primary_label};
+  msgs::Label dst_prim_label = {.id = acc.To().primary_label};
-  msgs::VertexId dst_vertex =
+  msgs::VertexId dst_vertex = std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.To().primary_key));
-      std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.ToVertex().primary_key));
 
   auto properties = acc.Properties(view);
 
@@ -165,7 +164,7 @@ std::any ParseExpression(const std::string &expr, memgraph::expr::AstStorage &st
   return visitor.visit(ast);
 }
 
-TypedValue ComputeExpression(DbAccessor &dba, const std::optional<memgraph::storage::v3::VertexAccessor> &v_acc,
+TypedValue ComputeExpression(DbAccessor &dba, const memgraph::storage::v3::VertexAccessor &v_acc,
                              const std::optional<memgraph::storage::v3::EdgeAccessor> &e_acc,
                              const std::string &expression, std::string_view node_name, std::string_view edge_name) {
   AstStorage storage;
@@ -192,10 +191,11 @@ TypedValue ComputeExpression(DbAccessor &dba, const std::optional<memgraph::stor
                              return position_symbol_pair.second.name() == node_name;
                            }) != symbol_table.table().end());
 
-    frame[symbol_table.at(node_identifier)] = *v_acc;
+    frame[symbol_table.at(node_identifier)] = v_acc;
   }
 
   if (edge_identifier.symbol_pos_ != -1) {
+    MG_ASSERT(e_acc.has_value());
     MG_ASSERT(std::find_if(symbol_table.table().begin(), symbol_table.table().end(),
                            [&edge_name](const std::pair<int32_t, Symbol> &position_symbol_pair) {
                              return position_symbol_pair.second.name() == edge_name;

src/storage/v3/expr.hpp

@@ -9,6 +9,8 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
+#pragma once
+
 #include <vector>
 
 #include "db_accessor.hpp"
@@ -48,8 +50,7 @@ auto Eval(TExpression *expr, EvaluationContext &ctx, AstStorage &storage, Expres
 
 std::any ParseExpression(const std::string &expr, AstStorage &storage);
 
-TypedValue ComputeExpression(DbAccessor &dba, const std::optional<VertexAccessor> &v_acc,
+TypedValue ComputeExpression(DbAccessor &dba, const VertexAccessor &v_acc, const std::optional<EdgeAccessor> &e_acc,
-                             const std::optional<EdgeAccessor> &e_acc, const std::string &expression,
+                             const std::string &expression, std::string_view node_name, std::string_view edge_name);
-                             std::string_view node_name, std::string_view edge_name);
 
 }  // namespace memgraph::storage::v3

src/storage/v3/indices.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,16 @@
 
 #include "indices.hpp"
 
+#include <algorithm>
+#include <functional>
 #include <limits>
 
+#include "storage/v3/delta.hpp"
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/mvcc.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/schemas.hpp"
+#include "storage/v3/vertex.hpp"
 #include "utils/bound.hpp"
 #include "utils/logging.hpp"
 #include "utils/memory_tracker.hpp"
@@ -53,9 +57,9 @@ bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t timestamp)
   bool deleted{false};
   const Delta *delta{nullptr};
   {
-    has_label = utils::Contains(vertex.labels, label);
+    has_label = utils::Contains(vertex.second.labels, label);
-    deleted = vertex.deleted;
+    deleted = vertex.second.deleted;
-    delta = vertex.delta;
+    delta = vertex.second.delta;
   }
   if (!deleted && has_label) {
     return true;
@@ -105,10 +109,10 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, PropertyId
   bool deleted{false};
   const Delta *delta{nullptr};
   {
-    has_label = utils::Contains(vertex.labels, label);
+    has_label = utils::Contains(vertex.second.labels, label);
-    current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
+    current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
-    deleted = vertex.deleted;
+    deleted = vertex.second.deleted;
-    delta = vertex.delta;
+    delta = vertex.second.delta;
   }
 
   if (!deleted && has_label && current_value_equal_to_value) {
@@ -163,9 +167,9 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
   bool has_label{false};
   const Delta *delta{nullptr};
   {
-    deleted = vertex.deleted;
+    deleted = vertex.second.deleted;
-    has_label = utils::Contains(vertex.labels, label);
+    has_label = utils::Contains(vertex.second.labels, label);
-    delta = vertex.delta;
+    delta = vertex.second.delta;
   }
   ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
     switch (delta.action) {
@@ -214,10 +218,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
   bool current_value_equal_to_value = value.IsNull();
   const Delta *delta{nullptr};
   {
-    deleted = vertex.deleted;
+    deleted = vertex.second.deleted;
-    has_label = utils::Contains(vertex.labels, label);
+    has_label = utils::Contains(vertex.second.labels, label);
-    current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
+    current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
-    delta = vertex.delta;
+    delta = vertex.second.delta;
   }
   ApplyDeltasForRead(transaction, delta, view,
                      [&deleted, &has_label, &current_value_equal_to_value, key, label, &value](const Delta &delta) {
@@ -265,11 +269,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
 void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
   auto it = index_.find(label);
   if (it == index_.end()) return;
-  auto acc = it->second.access();
+  it->second.insert(Entry{vertex, tx.start_timestamp.logical_id});
-  acc.insert(Entry{vertex, tx.start_timestamp.logical_id});
 }
 
-bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
+bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
   auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
   if (!emplaced) {
@@ -277,13 +280,11 @@ bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices)
     return false;
   }
   try {
-    auto acc = it->second.access();
+    for (auto &vertex : vertices) {
-    for (auto &lgo_vertex : vertices) {
+      if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
-      auto &vertex = lgo_vertex.vertex;
-      if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
         continue;
       }
-      acc.insert(Entry{&vertex, 0});
+      it->second.insert(Entry{&vertex, 0});
     }
   } catch (const utils::OutOfMemoryException &) {
     utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -304,7 +305,7 @@ std::vector<LabelId> LabelIndex::ListIndices() const {
 
 void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
   for (auto &label_storage : index_) {
-    auto vertices_acc = label_storage.second.access();
+    auto &vertices_acc = label_storage.second;
     for (auto it = vertices_acc.begin(); it != vertices_acc.end();) {
       auto next_it = it;
       ++next_it;
@@ -316,7 +317,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
 
       if ((next_it != vertices_acc.end() && it->vertex == next_it->vertex) ||
           !AnyVersionHasLabel(*it->vertex, label_storage.first, clean_up_before_timestamp)) {
-        vertices_acc.remove(*it);
+        vertices_acc.erase(*it);
       }
 
       it = next_it;
@@ -324,7 +325,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
   }
 }
 
-LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
+LabelIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -339,7 +340,7 @@ LabelIndex::Iterable::Iterator &LabelIndex::Iterable::Iterator::operator++() {
 }
 
 void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
-  for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
+  for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
     if (index_iterator_->vertex == current_vertex_) {
       continue;
     }
@@ -352,10 +353,9 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
   }
 }
 
-LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view,
+LabelIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction,
-                               Transaction *transaction, Indices *indices, Config::Items config,
+                               Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
-                               const VertexValidator &vertex_validator)
+    : index_container_(&index_container),
-    : index_accessor_(std::move(index_accessor)),
       label_(label),
       view_(view),
       transaction_(transaction),
@@ -363,12 +363,6 @@ LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor,
       config_(config),
       vertex_validator_(&vertex_validator) {}
 
-void LabelIndex::RunGC() {
-  for (auto &index_entry : index_) {
-    index_entry.second.run_gc();
-  }
-}
-
 bool LabelPropertyIndex::Entry::operator<(const Entry &rhs) const {
   if (value < rhs.value) {
     return true;
@@ -388,14 +382,13 @@ bool LabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) const { retu
 bool LabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) const { return value == rhs; }
 
 void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
-  for (auto &[label_prop, storage] : index_) {
+  for (auto &[label_prop, index] : index_) {
     if (label_prop.first != label) {
       continue;
     }
-    auto prop_value = vertex->properties.GetProperty(label_prop.second);
+    auto prop_value = vertex->second.properties.GetProperty(label_prop.second);
     if (!prop_value.IsNull()) {
-      auto acc = storage.access();
+      index.emplace(Entry{prop_value, vertex, tx.start_timestamp.logical_id});
-      acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp.logical_id});
     }
   }
 }
@@ -405,18 +398,17 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
   if (value.IsNull()) {
     return;
   }
-  for (auto &[label_prop, storage] : index_) {
+  for (auto &[label_prop, index] : index_) {
     if (label_prop.second != property) {
       continue;
     }
-    if (utils::Contains(vertex->labels, label_prop.first)) {
+    if (VertexHasLabel(*vertex, label_prop.first)) {
-      auto acc = storage.access();
+      index.emplace(Entry{value, vertex, tx.start_timestamp.logical_id});
-      acc.insert(Entry{value, vertex, tx.start_timestamp.logical_id});
     }
   }
 }
 
-bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
+bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
   auto [it, emplaced] =
       index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
@@ -425,17 +417,15 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, Vertice
     return false;
   }
   try {
-    auto acc = it->second.access();
+    for (auto &vertex : vertices) {
-    for (auto &lgo_vertex : vertices) {
+      if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
-      auto &vertex = lgo_vertex.vertex;
-      if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
         continue;
       }
-      auto value = vertex.properties.GetProperty(property);
+      auto value = vertex.second.properties.GetProperty(property);
       if (value.IsNull()) {
         continue;
       }
-      acc.insert(Entry{std::move(value), &vertex, 0});
+      it->second.emplace(Entry{value, &vertex, 0});
     }
   } catch (const utils::OutOfMemoryException &) {
     utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -456,8 +446,7 @@ std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ListIndices() co
 
 void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
   for (auto &[label_property, index] : index_) {
-    auto index_acc = index.access();
+    for (auto it = index.begin(); it != index.end();) {
-    for (auto it = index_acc.begin(); it != index_acc.end();) {
       auto next_it = it;
       ++next_it;
 
@@ -466,17 +455,17 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
         continue;
       }
 
-      if ((next_it != index_acc.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
+      if ((next_it != index.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
           !AnyVersionHasLabelProperty(*it->vertex, label_property.first, label_property.second, it->value,
                                       clean_up_before_timestamp)) {
-        index_acc.remove(*it);
+        index.erase(it);
       }
       it = next_it;
     }
   }
 }
 
-LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
+LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
     : self_(self),
       index_iterator_(index_iterator),
       current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -491,7 +480,7 @@ LabelPropertyIndex::Iterable::Iterator &LabelPropertyIndex::Iterable::Iterator::
 }
 
 void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
-  for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
+  for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
     if (index_iterator_->vertex == current_vertex_) {
       continue;
     }
@@ -506,11 +495,11 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
     }
     if (self_->upper_bound_) {
       if (self_->upper_bound_->value() < index_iterator_->value) {
-        index_iterator_ = self_->index_accessor_.end();
+        index_iterator_ = self_->index_container_->end();
         break;
       }
       if (!self_->upper_bound_->IsInclusive() && index_iterator_->value == self_->upper_bound_->value()) {
-        index_iterator_ = self_->index_accessor_.end();
+        index_iterator_ = self_->index_container_->end();
         break;
       }
     }
@@ -537,13 +526,12 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
 const PropertyValue kSmallestTemporalData =
     PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
 
-LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label,
+LabelPropertyIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
-                                       PropertyId property,
                                        const std::optional<utils::Bound<PropertyValue>> &lower_bound,
                                        const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
                                        Transaction *transaction, Indices *indices, Config::Items config,
                                        const VertexValidator &vertex_validator)
-    : index_accessor_(std::move(index_accessor)),
+    : index_container_(&index_container),
       label_(label),
       property_(property),
       lower_bound_(lower_bound),
@@ -651,49 +639,55 @@ LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::begin() {
   // If the bounds are set and don't have comparable types we don't yield any
   // items from the index.
   if (!bounds_valid_) {
-    return {this, index_accessor_.end()};
+    return {this, index_container_->end()};
   }
-  auto index_iterator = index_accessor_.begin();
   if (lower_bound_) {
-    index_iterator = index_accessor_.find_equal_or_greater(lower_bound_->value());
+    return {this, std::ranges::lower_bound(*index_container_, lower_bound_->value(), std::less{}, &Entry::value)};
   }
-  return {this, index_iterator};
+  return {this, index_container_->begin()};
 }
 
-LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_accessor_.end()}; }
+LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_container_->end()}; }
 
-int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
+int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
-                                                   const PropertyValue &value) const {
   auto it = index_.find({label, property});
   MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
-  auto acc = it->second.access();
+  MG_ASSERT(!value.IsNull(), "Null is not supported!");
-  if (!value.IsNull()) {
-    return static_cast<int64_t>(
-        acc.estimate_count(value, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
-  }
-  // The value `Null` won't ever appear in the index because it indicates that
-  // the property shouldn't exist. Instead, this value is used as an indicator
-  // to estimate the average number of equal elements in the list (for any
-  // given value).
-  return static_cast<int64_t>(acc.estimate_average_number_of_equals(
-      [](const auto &first, const auto &second) { return first.value == second.value; },
-      static_cast<int>(utils::SkipListLayerForAverageEqualsEstimation(acc.size()))));
-}
 
-int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
+  // TODO(jbajic) This can be improved by exiting early
-                                                   const std::optional<utils::Bound<PropertyValue>> &lower,
+  auto start_it = std::ranges::lower_bound(it->second, value, std::less{}, &Entry::value);
-                                                   const std::optional<utils::Bound<PropertyValue>> &upper) const {
-  auto it = index_.find({label, property});
-  MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
-  auto acc = it->second.access();
   return static_cast<int64_t>(
-      acc.estimate_range_count(lower, upper, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
+      std::ranges::count_if(start_it, it->second.end(), [&value](const auto &elem) { return elem.value == value; }));
 }
 
-void LabelPropertyIndex::RunGC() {
+int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property,
-  for (auto &index_entry : index_) {
+                                        const std::optional<utils::Bound<PropertyValue>> &lower,
-    index_entry.second.run_gc();
+                                        const std::optional<utils::Bound<PropertyValue>> &upper) const {
-  }
+  auto it = index_.find({label, property});
+  MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
+  const auto lower_it = std::invoke(
+      [&index = it->second](const auto value, const auto def) {
+        if (value) {
+          if (value->IsInclusive()) {
+            return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
+          }
+          return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
+        }
+        return def;
+      },
+      lower, it->second.begin());
+  const auto upper_it = std::invoke(
+      [&index = it->second](const auto value, const auto def) {
+        if (value) {
+          if (value->IsInclusive()) {
+            return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
+          }
+          return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
+        }
+        return def;
+      },
+      upper, it->second.end());
+  return static_cast<int64_t>(std::distance(lower_it, upper_it));
 }
 
 void RemoveObsoleteEntries(Indices *indices, const uint64_t clean_up_before_timestamp) {

src/storage/v3/indices.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,14 +13,16 @@
 
 #include <cstdint>
 #include <optional>
+#include <set>
 #include <tuple>
 #include <utility>
 
 #include "storage/v3/config.hpp"
+#include "storage/v3/id_types.hpp"
+#include "storage/v3/key_store.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/transaction.hpp"
 #include "storage/v3/vertex_accessor.hpp"
-#include "storage/v3/vertices_skip_list.hpp"
 #include "utils/bound.hpp"
 #include "utils/logging.hpp"
 #include "utils/skip_list.hpp"
@@ -30,7 +32,6 @@ namespace memgraph::storage::v3 {
 struct Indices;
 
 class LabelIndex {
- private:
   struct Entry {
     Vertex *vertex;
     uint64_t timestamp;
@@ -41,25 +42,23 @@ class LabelIndex {
     bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
   };
 
-  struct LabelStorage {
+  using IndexType = LabelId;
-    LabelId label;
-    utils::SkipList<Entry> vertices;
-
-    bool operator<(const LabelStorage &rhs) const { return label < rhs.label; }
-    bool operator<(LabelId rhs) const { return label < rhs; }
-    bool operator==(const LabelStorage &rhs) const { return label == rhs.label; }
-    bool operator==(LabelId rhs) const { return label == rhs; }
-  };
 
  public:
+  using IndexContainer = std::set<Entry>;
+
   LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
 
+  LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator,
+             std::map<LabelId, IndexContainer> &data)
+      : index_{std::move(data)}, indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
+
   /// @throw std::bad_alloc
   void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
+  bool CreateIndex(LabelId label, VertexContainer &vertices);
 
   /// Returns false if there was no index to drop
   bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
@@ -72,12 +71,12 @@ class LabelIndex {
 
   class Iterable {
    public:
-    Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view, Transaction *transaction,
+    Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
-             Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
+             Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
+      Iterator(Iterable *self, IndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -90,16 +89,16 @@ class LabelIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      utils::SkipList<Entry>::Iterator index_iterator_;
+      IndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
 
-    Iterator begin() { return {this, index_accessor_.begin()}; }
+    Iterator begin() { return {this, index_container_->begin()}; }
-    Iterator end() { return {this, index_accessor_.end()}; }
+    Iterator end() { return {this, index_container_->end()}; }
 
    private:
-    utils::SkipList<Entry>::Accessor index_accessor_;
+    IndexContainer *index_container_;
     LabelId label_;
     View view_;
     Transaction *transaction_;
@@ -112,7 +111,7 @@ class LabelIndex {
   Iterable Vertices(LabelId label, View view, Transaction *transaction) {
     auto it = index_.find(label);
     MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint());
-    return {it->second.access(), label, view, transaction, indices_, config_, *vertex_validator_};
+    return {it->second, label, view, transaction, indices_, config_, *vertex_validator_};
   }
 
   int64_t ApproximateVertexCount(LabelId label) {
@@ -123,17 +122,35 @@ class LabelIndex {
 
   void Clear() { index_.clear(); }
 
-  void RunGC();
+  std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
+    std::map<IndexType, IndexContainer> cloned_indices;
+    for (auto &[index_type_val, index] : index_) {
+      auto entry_it = index.begin();
+      auto &cloned_indices_container = cloned_indices[index_type_val];
+      while (entry_it != index.end()) {
+        // We need to save the next iterator since the current one will be
+        // invalidated after extract
+        auto next_entry_it = std::next(entry_it);
+        if (entry_it->vertex->first > split_key) {
+          [[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
+              cloned_indices_container.insert(index.extract(entry_it));
+          MG_ASSERT(inserted, "Failed to extract index entry!");
+        }
+        entry_it = next_entry_it;
+      }
+    }
+
+    return cloned_indices;
+  }
 
  private:
-  std::map<LabelId, utils::SkipList<Entry>> index_;
+  std::map<LabelId, IndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;
 };
 
 class LabelPropertyIndex {
- private:
   struct Entry {
     PropertyValue value;
     Vertex *vertex;
@@ -145,8 +162,11 @@ class LabelPropertyIndex {
     bool operator<(const PropertyValue &rhs) const;
     bool operator==(const PropertyValue &rhs) const;
   };
+  using IndexType = std::pair<LabelId, PropertyId>;
 
  public:
+  using IndexContainer = std::set<Entry>;
+
   LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
       : indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
 
@@ -157,7 +177,7 @@ class LabelPropertyIndex {
   void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
+  bool CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices);
 
   bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
 
@@ -169,14 +189,14 @@ class LabelPropertyIndex {
 
   class Iterable {
    public:
-    Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, PropertyId property,
+    Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
              const std::optional<utils::Bound<PropertyValue>> &lower_bound,
              const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
              Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
 
     class Iterator {
      public:
-      Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
+      Iterator(Iterable *self, IndexContainer::iterator index_iterator);
 
       VertexAccessor operator*() const { return current_vertex_accessor_; }
 
@@ -189,7 +209,7 @@ class LabelPropertyIndex {
       void AdvanceUntilValid();
 
       Iterable *self_;
-      utils::SkipList<Entry>::Iterator index_iterator_;
+      IndexContainer::iterator index_iterator_;
       VertexAccessor current_vertex_accessor_;
       Vertex *current_vertex_;
     };
@@ -198,7 +218,7 @@ class LabelPropertyIndex {
     Iterator end();
 
    private:
-    utils::SkipList<Entry>::Accessor index_accessor_;
+    IndexContainer *index_container_;
     LabelId label_;
     PropertyId property_;
     std::optional<utils::Bound<PropertyValue>> lower_bound_;
@@ -217,11 +237,11 @@ class LabelPropertyIndex {
     auto it = index_.find({label, property});
     MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
               property.AsUint());
-    return {it->second.access(), label,    property, lower_bound,       upper_bound, view,
+    return {it->second, label,       property, lower_bound, upper_bound,
-            transaction,         indices_, config_,  *vertex_validator_};
+            view,       transaction, indices_, config_,     *vertex_validator_};
   }
 
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
+  int64_t VertexCount(LabelId label, PropertyId property) const {
     auto it = index_.find({label, property});
     MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
               property.AsUint());
@@ -232,18 +252,36 @@ class LabelPropertyIndex {
   /// an estimated count of nodes which have their property's value set to
   /// `value`. If the `value` specified is `Null`, then an average number of
   /// equal elements is returned.
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
+  int64_t VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
 
-  int64_t ApproximateVertexCount(LabelId label, PropertyId property,
+  int64_t VertexCount(LabelId label, PropertyId property, const std::optional<utils::Bound<PropertyValue>> &lower,
-                                 const std::optional<utils::Bound<PropertyValue>> &lower,
+                      const std::optional<utils::Bound<PropertyValue>> &upper) const;
-                                 const std::optional<utils::Bound<PropertyValue>> &upper) const;
 
   void Clear() { index_.clear(); }
 
-  void RunGC();
+  std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
+    std::map<IndexType, IndexContainer> cloned_indices;
+    for (auto &[index_type_val, index] : index_) {
+      auto entry_it = index.begin();
+      auto &cloned_index_container = cloned_indices[index_type_val];
+      while (entry_it != index.end()) {
+        // We need to save the next iterator since the current one will be
+        // invalidated after extract
+        auto next_entry_it = std::next(entry_it);
+        if (entry_it->vertex->first > split_key) {
+          [[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
+              cloned_index_container.insert(index.extract(entry_it));
+          MG_ASSERT(inserted, "Failed to extract index entry!");
+        }
+        entry_it = next_entry_it;
+      }
+    }
+
+    return cloned_indices;
+  }
 
  private:
-  std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>> index_;
+  std::map<std::pair<LabelId, PropertyId>, IndexContainer> index_;
   Indices *indices_;
   Config::Items config_;
   const VertexValidator *vertex_validator_;

src/storage/v3/key_store.cpp

@@ -1,43 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <algorithm>
-#include <iterator>
-#include <ranges>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/key_store.hpp"
-#include "storage/v3/property_value.hpp"
-
-namespace memgraph::storage::v3 {
-
-KeyStore::KeyStore(const PrimaryKey &key_values) {
-  for (auto i = 0; i < key_values.size(); ++i) {
-    MG_ASSERT(!key_values[i].IsNull());
-    store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
-  }
-}
-
-PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
-
-PropertyValue KeyStore::GetKey(const PropertyId property_id) const { return store_.GetProperty(property_id); }
-
-PrimaryKey KeyStore::Keys() const {
-  auto keys_map = store_.Properties();
-  PrimaryKey keys;
-  keys.reserve(keys_map.size());
-  std::ranges::transform(
-      keys_map, std::back_inserter(keys),
-      [](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
-  return keys;
-}
-
-}  // namespace memgraph::storage::v3

src/storage/v3/key_store.hpp

@@ -11,12 +11,6 @@
 
 #pragma once
 
-#include <algorithm>
-#include <compare>
-#include <functional>
-
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/property_store.hpp"
 #include "storage/v3/property_value.hpp"
 
 namespace memgraph::storage::v3 {
@@ -24,39 +18,4 @@ namespace memgraph::storage::v3 {
 // Primary key is a collection of primary properties.
 using PrimaryKey = std::vector<PropertyValue>;
 
-class KeyStore {
- public:
-  explicit KeyStore(const PrimaryKey &key_values);
-
-  KeyStore(const KeyStore &) = delete;
-  KeyStore(KeyStore &&other) noexcept = default;
-  KeyStore &operator=(const KeyStore &) = delete;
-  KeyStore &operator=(KeyStore &&other) noexcept = default;
-
-  ~KeyStore() = default;
-
-  PropertyValue GetKey(size_t index) const;
-
-  PropertyValue GetKey(PropertyId property) const;
-
-  PrimaryKey Keys() const;
-
-  friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
-    return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
-  }
-
-  friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
-    return std::ranges::equal(lhs.Keys(), rhs.Keys());
-  }
-
-  friend bool operator<(const KeyStore &lhs, const PrimaryKey &rhs) {
-    return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
-  }
-
-  friend bool operator==(const KeyStore &lhs, const PrimaryKey &rhs) { return std::ranges::equal(lhs.Keys(), rhs); }
-
- private:
-  PropertyStore store_;
-};
-
 }  // namespace memgraph::storage::v3

src/storage/v3/lexicographically_ordered_vertex.hpp

@@ -1,42 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <concepts>
-#include <type_traits>
-
-#include "storage/v3/vertex.hpp"
-#include "utils/concepts.hpp"
-
-namespace memgraph::storage::v3 {
-
-struct LexicographicallyOrderedVertex {
-  Vertex vertex;
-
-  friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
-    return lhs.vertex.keys == rhs.vertex.keys;
-  }
-
-  friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
-    return lhs.vertex.keys < rhs.vertex.keys;
-  }
-
-  // TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
-  friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
-    return lhs.vertex.keys == rhs;
-  }
-
-  friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
-    return lhs.vertex.keys < rhs;
-  }
-};
-}  // namespace memgraph::storage::v3

src/storage/v3/mvcc.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,21 @@
 
 #pragma once
 
+#include <type_traits>
+
+#include "storage/v3/edge.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/transaction.hpp"
+#include "storage/v3/vertex.hpp"
 #include "storage/v3/view.hpp"
+#include "utils/concepts.hpp"
 
 namespace memgraph::storage::v3 {
 
+inline VertexData *GetDeltaHolder(Vertex *vertex) { return &vertex->second; }
+
+inline Edge *GetDeltaHolder(Edge *edge) { return edge; }
+
 /// This function iterates through the undo buffers from an object (starting
 /// from the supplied delta) and determines what deltas should be applied to get
 /// the currently visible version of the object. When the function finds a delta
@@ -77,10 +86,12 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie
 /// transaction) and returns a `bool` value indicating whether the caller can
 /// proceed with a write operation.
 template <typename TObj>
+requires utils::SameAsAnyOf<TObj, Edge, Vertex>
 inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
-  if (object->delta == nullptr) return true;
+  auto *delta_holder = GetDeltaHolder(object);
+  if (delta_holder->delta == nullptr) return true;
 
-  const auto &delta_commit_info = *object->delta->commit_info;
+  const auto &delta_commit_info = *delta_holder->delta->commit_info;
   if (delta_commit_info.start_or_commit_timestamp == transaction->commit_info->start_or_commit_timestamp ||
       (delta_commit_info.is_locally_committed &&
        delta_commit_info.start_or_commit_timestamp < transaction->start_timestamp)) {
@@ -97,7 +108,7 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
 /// a `DELETE_OBJECT` delta).
 /// @throw std::bad_alloc
 inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
-  return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(),
+  return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(), GetNextDeltaId(),
                                            transaction->command_id);
 }
 
@@ -105,9 +116,11 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
 /// the delta into the object's delta list.
 /// @throw std::bad_alloc
 template <typename TObj, class... Args>
+requires utils::SameAsAnyOf<TObj, Edge, Vertex>
 inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
   auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
-                                                 transaction->command_id);
+                                                 GetNextDeltaId(), transaction->command_id);
+  auto *delta_holder = GetDeltaHolder(object);
 
   // The operations are written in such order so that both `next` and `prev`
   // chains are valid at all times. The chains must be valid at all times
@@ -118,21 +131,21 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
   // TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in
   // `Shard::Accessor::CreateEdge`.
   // NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
-  delta->next = object->delta;
+  delta->next = delta_holder->delta;
   // 2. We need to set the previous delta of the new delta to the object.
   delta->prev.Set(object);
   // 3. We need to set the previous delta of the existing delta to the new
   // delta. After this point the garbage collector will be able to see the new
   // delta but won't modify it until we are done with all of our modifications.
-  if (object->delta) {
+  if (delta_holder->delta) {
-    object->delta->prev.Set(delta);
+    delta_holder->delta->prev.Set(delta);
   }
   // 4. Finally, we need to set the object's delta to the new delta. The garbage
   // collector and other transactions will acquire the object lock to read the
   // delta from the object. Because the lock is held during the whole time this
   // modification is being done, everybody else will wait until we are fully
   // done with our modification before they read the object's delta value.
-  object->delta = delta;
+  delta_holder->delta = delta;
 }
 
 }  // namespace memgraph::storage::v3