Merge pull request #719 from memgraph/T1149-MG-split-shard

- Implement shard split on storage side

.github/workflows/daily_benchmark.yaml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)

.github/workflows/diff.yaml

@@ -26,7 +26,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -64,7 +64,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -75,14 +75,39 @@ jobs:
       - name: Fetch all history for all tags and branches
         run: git fetch
 
+      - name: Initialize deps
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+          # Initialize dependencies.
+          ./init
+
+      - name: Set base branch
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
+
+      - name: Set base branch # if we manually dispatch or push to master
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
+
+      - name: Python code analysis
+        run: |
+          CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
+          for file in ${CHANGED_FILES}; do
+            echo ${file}
+            if [[ ${file} == *.py ]]; then
+              python3 -m black --check --diff ${file}
+              python3 -m isort --check-only --profile "black" --diff ${file}
+            fi
+          done
+
       - name: Build combined ASAN, UBSAN and coverage binaries
         run: |
           # Activate toolchain.
           source /opt/toolchain-v4/activate
 
-          # Initialize dependencies.
-          ./init
-
           cd build
           cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
           make -j$THREADS memgraph__unit
@@ -110,7 +135,7 @@ jobs:
           tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
 
       - name: Save code coverage
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
@@ -145,7 +170,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -164,52 +189,23 @@ jobs:
           cmake ..
           make -j$THREADS
 
-      - name: Run leftover CTest tests
+      - name: Run simulation tests
         run: |
           # Activate toolchain.
           source /opt/toolchain-v4/activate
 
-          # Run leftover CTest tests (all except unit and benchmark tests).
+          # Run simulation tests.
           cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
+          ctest -R memgraph__simulation --output-on-failure -j$THREADS
 
-      - name: Run drivers tests
-        run: |
-          ./tests/drivers/run.sh
-
-      - name: Run integration tests
-        run: |
-          cd tests/integration
-          for name in *; do
-            if [ ! -d $name ]; then continue; fi
-            pushd $name >/dev/null
-            echo "Running: $name"
-            if [ -x prepare.sh ]; then
-              ./prepare.sh
-            fi
-            if [ -x runner.py ]; then
-              ./runner.py
-            elif [ -x runner.sh ]; then
-              ./runner.sh
-            fi
-            echo
-            popd >/dev/null
-          done
-
-      - name: Run cppcheck and clang-format
+      - name: Run single benchmark test
         run: |
           # Activate toolchain.
           source /opt/toolchain-v4/activate
 
-          # Run cppcheck and clang-format.
-          cd tools/github
-          ./cppcheck_and_clang_format diff
-
-      - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v2
-        with:
-          name: "Code coverage"
-          path: tools/github/cppcheck_and_clang_format.txt
+          # Run simulation tests.
+          cd tests/mgbench
+          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
 
   release_build:
     name: "Release build"
@@ -221,7 +217,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -240,19 +236,6 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=release ..
           make -j$THREADS
 
-      - name: Run GQL Behave tests
-        run: |
-          cd tests/gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v2
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
       - name: Run unit tests
         run: |
           # Activate toolchain.
@@ -267,10 +250,19 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v4/activate
 
-          # Run unit tests.
+          # Run simulation tests.
           cd build
           ctest -R memgraph__simulation --output-on-failure -j$THREADS
 
+      - name: Run single benchmark test
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run simulation tests.
+          cd tests/mgbench
+          ./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg  "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
+
       - name: Run e2e tests
         run: |
           # TODO(gitbuda): Setup mgclient and pymgclient properly.
@@ -278,131 +270,12 @@ jobs:
           ./setup.sh
           source ve3/bin/activate
           cd e2e
-          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
 
-      - name: Run stress test (plain)
+      - name: Run query performance tests
         run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
-      - name: Run durability test
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v2
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Save test data
-        uses: actions/upload-artifact@v2
-        if: always()
-        with:
-          name: "Test data"
-          path: |
-            # multiple paths could be defined
-            build/logs
-
-  release_jepsen_test:
-    name: "Release Jepsen Test"
-    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
-    #continue-on-error: true
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v2
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binarie.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS memgraph
-
-      - name: Run Jepsen tests
-        run: |
-          cd tests/jepsen
-          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
-
-      - name: Save Jepsen report
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: "Jepsen Report"
-          path: tests/jepsen/Jepsen.tar.gz
-
-  release_benchmarks:
-    name: "Release benchmarks"
-    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v2
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Run macro benchmarks
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
+          cd tests/manual
+          ./query_performance_runner.py
 
       - name: Get branch name (merge)
         if: github.event_name != 'pull_request'
@@ -420,25 +293,8 @@ jobs:
           virtualenv -p python3 ve3
           source ve3/bin/activate
           pip install -r requirements.txt
-          ./main.py --benchmark-name "macro_benchmark" \
-                    --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
-                    --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}" \
-                    --head-branch-name "${{ env.BRANCH_NAME }}"
-
-      - name: Run mgbench
-        run: |
-          cd tests/mgbench
-          ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
-
-      - name: Upload mgbench results
-        run: |
-          cd tools/bench-graph-client
-          virtualenv -p python3 ve3
-          source ve3/bin/activate
-          pip install -r requirements.txt
-          ./main.py --benchmark-name "mgbench" \
-                    --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
+          ./main.py --benchmark-name "query_performance" \
+                    --benchmark-results-path "../../build/tests/manual/query_performance_benchmark/summary.json" \
                     --github-run-id "${{ github.run_id }}" \
                     --github-run-number "${{ github.run_number }}" \
                     --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/full_clang_tidy.yaml

@@ -14,7 +14,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)

.github/workflows/package_all.yaml

@@ -17,7 +17,7 @@ jobs:
         run: |
           ./release/package/run.sh package centos-7
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: centos-7
           path: build/output/centos-7/memgraph*.rpm
@@ -34,7 +34,7 @@ jobs:
         run: |
           ./release/package/run.sh package centos-9
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: centos-9
           path: build/output/centos-9/memgraph*.rpm
@@ -51,7 +51,7 @@ jobs:
         run: |
           ./release/package/run.sh package debian-10
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: debian-10
           path: build/output/debian-10/memgraph*.deb
@@ -68,7 +68,7 @@ jobs:
         run: |
           ./release/package/run.sh package debian-11
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: debian-11
           path: build/output/debian-11/memgraph*.deb
@@ -87,7 +87,7 @@ jobs:
           ./run.sh package debian-11 --for-docker
           ./run.sh docker
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: docker
           path: build/output/docker/memgraph*.tar.gz
@@ -104,7 +104,7 @@ jobs:
         run: |
           ./release/package/run.sh package ubuntu-18.04
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ubuntu-1804
           path: build/output/ubuntu-18.04/memgraph*.deb
@@ -121,7 +121,7 @@ jobs:
         run: |
           ./release/package/run.sh package ubuntu-20.04
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ubuntu-2004
           path: build/output/ubuntu-20.04/memgraph*.deb
@@ -138,7 +138,7 @@ jobs:
         run: |
           ./release/package/run.sh package ubuntu-22.04
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ubuntu-2204
           path: build/output/ubuntu-22.04/memgraph*.deb
@@ -155,7 +155,7 @@ jobs:
         run: |
           ./release/package/run.sh package debian-11 --for-platform
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: debian-11-platform
           path: build/output/debian-11/memgraph*.deb
@@ -172,7 +172,7 @@ jobs:
         run: |
           ./release/package/run.sh package debian-11-arm
       - name: "Upload package"
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: debian-11-arm
           path: build/output/debian-11-arm/memgraph*.deb

.github/workflows/release_centos8.yaml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
           tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
 
       - name: Save code coverage
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
           ./cppcheck_and_clang_format diff
 
       - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -225,7 +225,7 @@ jobs:
           rpmlint memgraph*.rpm
 
       - name: Save enterprise RPM package
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Enterprise RPM package"
           path: build/output/memgraph*.rpm
@@ -262,7 +262,7 @@ jobs:
           ./continuous_integration
 
       - name: Save quality assurance status
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "GQL Behave Status"
           path: |

.github/workflows/release_debian10.yaml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
           tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
 
       - name: Save code coverage
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
           ./cppcheck_and_clang_format diff
 
       - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
           cpack -G DEB --config ../CPackConfig.cmake
 
       - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Enterprise DEB package"
           path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
           ./continuous_integration
 
       - name: Save quality assurance status
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "GQL Behave Status"
           path: |
@@ -324,7 +324,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -349,7 +349,7 @@ jobs:
           ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
 
       - name: Save Jepsen report
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
           name: "Jepsen Report"

.github/workflows/release_docker.yaml

@@ -19,7 +19,7 @@ jobs:
       DOCKER_REPOSITORY_NAME: memgraph
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1

.github/workflows/release_ubuntu2004.yaml

@@ -17,7 +17,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
           tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
 
       - name: Save code coverage
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
           ./cppcheck_and_clang_format diff
 
       - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Code coverage"
           path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
 
     steps:
       - name: Set up repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           # Number of commits to fetch. `0` indicates all history for all
           # branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
           cpack -G DEB --config ../CPackConfig.cmake
 
       - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "Enterprise DEB package"
           path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
           ./continuous_integration
 
       - name: Save quality assurance status
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: "GQL Behave Status"
           path: |

.gitignore

@@ -54,6 +54,7 @@ src/distributed/token_sharing_rpc_messages.hpp
 src/distributed/updates_rpc_messages.hpp
 src/query/v2/frontend/ast/ast.hpp
 src/query/frontend/ast/ast.hpp
+src/storage/v3/bindings/ast/ast.hpp
 src/query/distributed/frontend/ast/ast_serialization.hpp
 src/query/v2/distributed/frontend/ast/ast_serialization.hpp
 src/durability/distributed/state_delta.hpp
@@ -61,12 +62,15 @@ src/durability/single_node/state_delta.hpp
 src/durability/single_node_ha/state_delta.hpp
 src/query/frontend/semantic/symbol.hpp
 src/query/v2/frontend/semantic/symbol.hpp
+src/expr/semantic/symbol.hpp
 src/query/distributed/frontend/semantic/symbol_serialization.hpp
 src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
 src/query/distributed/plan/ops.hpp
 src/query/v2/distributed/plan/ops.hpp
 src/query/plan/operator.hpp
 src/query/v2/plan/operator.hpp
+src/parser/opencypher/generated
+src/expr/semantic/symbol.hpp
 src/raft/log_entry.hpp
 src/raft/raft_rpc_messages.hpp
 src/raft/snapshot_metadata.hpp

.pre-commit-config.yaml

@@ -6,18 +6,15 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.10.0
     hooks:
       - id: black
-        args: # arguments to configure black
-          - --line-length=120
-          - --include='\.pyi?$'
-          # these folders wont be formatted by black
-          - --exclude="""\.git |
-            \.__pycache__|
-            build|
-            libs|
-            .cache"""
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)
+        args: ["--profile", "black"]
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v13.0.0
     hooks:

CMakeLists.txt

@@ -182,7 +182,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # c99-designator is disabled because of required mixture of designated and
 # non-designated initializers in Python Query Module code (`py_module.cpp`).
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-    -Werror=switch -Werror=switch-bool -Werror=return-type \
+    -Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
+    -Werror=return-type \
     -Werror=return-stack-address \
     -Wno-c99-designator \
     -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")

config/flags.yaml

@@ -83,13 +83,9 @@ modifications:
     value: "true"
     override: true
 
-  - name: "query_modules_directory"
-    value: "/usr/lib/memgraph/query_modules"
-    override: true
-
-  - name: "auth_module_executable"
-    value: "/usr/lib/memgraph/auth_module/example.py"
-    override: false
+  # - name: "query_modules_directory"
+  #   value: "/usr/lib/memgraph/query_modules"
+  #   override: true
 
   - name: "memory_limit"
     value: "0"

init

@@ -140,5 +140,12 @@ done;
 python3 -m pip install pre-commit
 python3 -m pre_commit install
 
+# Install py format tools
+echo "Install black formatter"
+python3 -m pip install black==22.10.*
+echo "Install isort"
+python3 -m pip install isort==5.10.*
+
+
 # Link `include/mgp.py` with `release/mgp/mgp.py`
 ln -v -f include/mgp.py release/mgp/mgp.py

libs/setup.sh

@@ -116,7 +116,7 @@ declare -A primary_urls=(
   ["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
   ["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
   ["spdlog"]="http://$local_cache_host/git/spdlog"
-  ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
+  ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
   ["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
   ["protobuf"]="http://$local_cache_host/git/protobuf.git"
@@ -141,7 +141,7 @@ declare -A secondary_urls=(
   ["pymgclient"]="https://github.com/memgraph/pymgclient.git"
   ["mgconsole"]="http://github.com/memgraph/mgconsole.git"
   ["spdlog"]="https://github.com/gabime/spdlog"
-  ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
+  ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
   ["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="https://github.com/edenhill/librdkafka.git"
   ["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
 repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
 
 # google test
-googletest_tag="release-1.8.0"
+googletest_tag="release-1.12.1"
 repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
 
 # libbcrypt

pyproject.toml

@@ -0,0 +1,12 @@
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+    | .git
+    | .__pycache__
+    | build
+    | libs
+    | .cache
+)/
+'''

src/CMakeLists.txt

@@ -21,6 +21,7 @@ add_subdirectory(auth)
 add_subdirectory(parser)
 add_subdirectory(expr)
 add_subdirectory(coordinator)
+add_subdirectory(functions)
 
 if (MG_ENTERPRISE)
   add_subdirectory(audit)
@@ -37,13 +38,13 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # Memgraph Single Node v2 Executable
 # ----------------------------------------------------------------------------
 set(mg_single_node_v2_sources
-    glue/communication.cpp
-    memgraph.cpp
-    glue/auth.cpp
+        glue/v2/communication.cpp
+        memgraph.cpp
+        glue/v2/auth.cpp
 )
 
 set(mg_single_node_v2_libs stdc++fs Threads::Threads
-    telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
+    telemetry_lib mg-query-v2 mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-io mg-coordinator)
 if (MG_ENTERPRISE)
   # These are enterprise subsystems
   set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
@@ -126,19 +127,3 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
 # ----------------------------------------------------------------------------
 # Memgraph CSV Import Tool Executable
 # ----------------------------------------------------------------------------
-
-add_executable(mg_import_csv mg_import_csv.cpp)
-target_link_libraries(mg_import_csv mg-storage-v2)
-
-# Strip the executable in release build.
-if (lower_build_type STREQUAL "release")
-  add_custom_command(TARGET mg_import_csv POST_BUILD
-                     COMMAND strip -s mg_import_csv
-                     COMMENT "Stripping symbols and sections from mg_import_csv")
-endif()
-
-install(TARGETS mg_import_csv RUNTIME DESTINATION bin)
-
-# ----------------------------------------------------------------------------
-# Memgraph CSV Import Tool Executable
-# ----------------------------------------------------------------------------

src/common/errors.hpp

@@ -0,0 +1,68 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <cstdint>
+#include <string_view>
+
+namespace memgraph::common {
+
+enum class ErrorCode : uint8_t {
+  SERIALIZATION_ERROR,
+  NONEXISTENT_OBJECT,
+  DELETED_OBJECT,
+  VERTEX_HAS_EDGES,
+  PROPERTIES_DISABLED,
+  VERTEX_ALREADY_INSERTED,
+  // Schema Violations
+  SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
+  SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
+  SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
+  SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
+  SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
+  SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
+
+  OBJECT_NOT_FOUND,
+};
+
+constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
+  switch (code) {
+    case ErrorCode::SERIALIZATION_ERROR:
+      return "SERIALIZATION_ERROR";
+    case ErrorCode::NONEXISTENT_OBJECT:
+      return "NONEXISTENT_OBJECT";
+    case ErrorCode::DELETED_OBJECT:
+      return "DELETED_OBJECT";
+    case ErrorCode::VERTEX_HAS_EDGES:
+      return "VERTEX_HAS_EDGES";
+    case ErrorCode::PROPERTIES_DISABLED:
+      return "PROPERTIES_DISABLED";
+    case ErrorCode::VERTEX_ALREADY_INSERTED:
+      return "VERTEX_ALREADY_INSERTED";
+    case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
+      return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
+    case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
+      return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
+    case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
+      return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
+    case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
+      return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
+    case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
+      return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
+    case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
+      return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
+    case ErrorCode::OBJECT_NOT_FOUND:
+      return "OBJECT_NOT_FOUND";
+  }
+}
+
+}  // namespace memgraph::common

src/communication/bolt/v1/states/executing.hpp

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
     }
     case Signature::Route: {
       if constexpr (bolt_minor >= 3) {
-        if (signature == Signature::Route) return HandleRoute<TSession>(session);
+        return HandleRoute<TSession>(session);
       } else {
         spdlog::trace("Supported only in bolt v4.3");
         return State::Close;

src/communication/v2/listener.hpp

@@ -30,6 +30,7 @@
 #include "communication/context.hpp"
 #include "communication/v2/pool.hpp"
 #include "communication/v2/session.hpp"
+#include "utils/message.hpp"
 #include "utils/spin_lock.hpp"
 #include "utils/synchronized.hpp"
 
@@ -58,10 +59,10 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
   bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
 
  private:
-  Listener(boost::asio::io_context &io_context, TSessionData *data, ServerContext *server_context,
+  Listener(boost::asio::io_context &io_context, TSessionData &data, ServerContext *server_context,
            tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
       : io_context_(io_context),
-        data_(data),
+        data_(&data),
         server_context_(server_context),
         acceptor_(io_context_),
         endpoint_{endpoint},
@@ -110,7 +111,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
       return OnError(ec, "accept");
     }
 
-    auto session = SessionHandler::Create(std::move(socket), data_, *server_context_, endpoint_, inactivity_timeout_,
+    auto session = SessionHandler::Create(std::move(socket), *data_, *server_context_, endpoint_, inactivity_timeout_,
                                           service_name_);
     session->Start();
     DoAccept();

src/communication/v2/server.hpp

@@ -72,7 +72,7 @@ class Server final {
    * Constructs and binds server to endpoint, operates on session data and
    * invokes workers_count workers
    */
-  Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context,
+  Server(ServerEndpoint &endpoint, TSessionData &session_data, ServerContext *server_context,
          const int inactivity_timeout_sec, const std::string_view service_name,
          size_t workers_count = std::thread::hardware_concurrency())
       : endpoint_{endpoint},

src/communication/v2/session.hpp

@@ -41,6 +41,7 @@
 #include <boost/beast/websocket/rfc6455.hpp>
 #include <boost/system/detail/error_code.hpp>
 
+#include "communication/buffer.hpp"
 #include "communication/context.hpp"
 #include "communication/exceptions.hpp"
 #include "utils/logging.hpp"
@@ -139,7 +140,7 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
 
  private:
   // Take ownership of the socket
-  explicit WebsocketSession(tcp::socket &&socket, TSessionData *data, tcp::endpoint endpoint,
+  explicit WebsocketSession(tcp::socket &&socket, TSessionData &data, tcp::endpoint endpoint,
                             std::string_view service_name)
       : ws_(std::move(socket)),
         strand_{boost::asio::make_strand(ws_.get_executor())},
@@ -311,13 +312,13 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
   }
 
  private:
-  explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &server_context, tcp::endpoint endpoint,
+  explicit Session(tcp::socket &&socket, TSessionData &data, ServerContext &server_context, tcp::endpoint endpoint,
                    const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
       : socket_(CreateSocket(std::move(socket), server_context)),
         strand_{boost::asio::make_strand(GetExecutor())},
         output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
         session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
-        data_{data},
+        data_{&data},
         endpoint_{endpoint},
         remote_endpoint_{GetRemoteEndpoint()},
         service_name_{service_name},
@@ -373,7 +374,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
         spdlog::info("Switching {} to websocket connection", remote_endpoint_);
         if (std::holds_alternative<TCPSocket>(socket_)) {
           auto sock = std::get<TCPSocket>(std::move(socket_));
-          WebsocketSession<TSession, TSessionData>::Create(std::move(sock), data_, endpoint_, service_name_)
+          WebsocketSession<TSession, TSessionData>::Create(std::move(sock), *data_, endpoint_, service_name_)
               ->DoAccept(parser.release());
           execution_active_ = false;
           return;
@@ -465,7 +466,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
     if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
       // The deadline has passed. Stop the session. The other actors will
       // terminate as soon as possible.
-      spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
+      spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_.count());
       DoShutdown();
     } else {
       // Put the actor back to sleep.

src/coordinator/CMakeLists.txt

@@ -1,10 +1,9 @@
 set(coordinator_src_files
-    coordinator.hpp
-    shard_map.hpp
-    hybrid_logical_clock.hpp)
+    coordinator.cpp
+    shard_map.cpp)
 
 find_package(fmt REQUIRED)
 find_package(Threads REQUIRED)
 
 add_library(mg-coordinator STATIC ${coordinator_src_files})
-target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils)
+target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils mg-storage-v3)

src/coordinator/coordinator.cpp

@@ -0,0 +1,129 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <coordinator/coordinator.hpp>
+
+namespace memgraph::coordinator {
+
+CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
+  spdlog::info("Coordinator handling HeartbeatRequest");
+
+  // add this storage engine to any under-replicated shards that it is not already a part of
+
+  auto initializing_rsms_for_shard_manager =
+      shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
+
+  return HeartbeatResponse{
+      .shards_to_initialize = initializing_rsms_for_shard_manager,
+  };
+}
+
+CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
+  HlcResponse res{};
+
+  auto hlc_shard_map = shard_map_.GetHlc();
+
+  MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
+
+  res.new_hlc = Hlc{
+      .logical_id = ++highest_allocated_timestamp_,
+      // TODO(tyler) probably pass some more context to the Coordinator here
+      // so that we can use our wall clock and enforce monotonicity.
+      // .coordinator_wall_clock = io_.Now(),
+  };
+
+  // Allways return fresher shard_map for now.
+  res.fresher_shard_map = std::make_optional(shard_map_);
+
+  return res;
+}
+
+CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
+  AllocateEdgeIdBatchResponse res{};
+
+  uint64_t low = highest_allocated_edge_id_;
+
+  highest_allocated_edge_id_ += ahr.batch_size;
+
+  uint64_t high = highest_allocated_edge_id_;
+
+  res.low = low;
+  res.high = high;
+
+  return res;
+}
+
+/// This splits the shard immediately beneath the provided
+/// split key, keeping the assigned peers identical for now,
+/// but letting them be gradually migrated over time.
+CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
+  SplitShardResponse res{};
+
+  if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
+    res.success = false;
+  } else {
+    res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
+                                        split_shard_request.split_key);
+  }
+
+  return res;
+}
+
+/// This adds the provided storage engine to the standby storage engine pool,
+/// which can be used to rebalance storage over time.
+CoordinatorWriteResponses Coordinator::ApplyWrite(
+    RegisterStorageEngineRequest && /* register_storage_engine_request */) {
+  RegisterStorageEngineResponse res{};
+  // TODO
+
+  return res;
+}
+
+/// This begins the process of draining the provided storage engine from all raft
+/// clusters that it might be participating in.
+CoordinatorWriteResponses Coordinator::ApplyWrite(
+    DeregisterStorageEngineRequest && /* register_storage_engine_request */) {
+  DeregisterStorageEngineResponse res{};
+  // TODO
+
+  return res;
+}
+
+CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
+  InitializeLabelResponse res{};
+
+  std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
+      initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
+      initialize_label_request.last_shard_map_version);
+
+  if (new_label_id) {
+    res.new_label_id = new_label_id.value();
+    res.fresher_shard_map = std::nullopt;
+    res.success = true;
+  } else {
+    res.fresher_shard_map = shard_map_;
+    res.success = false;
+  }
+
+  return res;
+}
+
+CoordinatorWriteResponses Coordinator::ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request) {
+  AllocatePropertyIdsResponse res{};
+
+  auto property_ids = shard_map_.AllocatePropertyIds(allocate_property_ids_request.property_names);
+
+  res.property_ids = property_ids;
+
+  return res;
+}
+
+}  // namespace memgraph::coordinator

src/coordinator/coordinator.hpp

@@ -12,26 +12,30 @@
 #pragma once
 
 #include <optional>
+#include <set>
 #include <string>
 #include <unordered_set>
 #include <variant>
 #include <vector>
 
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "coordinator/shard_map.hpp"
-#include "io/simulator/simulator.hpp"
-#include "io/time.hpp"
-#include "io/transport.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/schemas.hpp"
+#include <boost/uuid/uuid.hpp>
+
+#include <coordinator/hybrid_logical_clock.hpp>
+#include <coordinator/shard_map.hpp>
+#include <io/simulator/simulator.hpp>
+#include <io/time.hpp>
+#include <io/transport.hpp>
+#include <storage/v3/id_types.hpp>
+#include <storage/v3/schemas.hpp>
 
 namespace memgraph::coordinator {
 
+using memgraph::io::Address;
 using memgraph::storage::v3::LabelId;
 using memgraph::storage::v3::PropertyId;
-using Address = memgraph::io::Address;
-using SimT = memgraph::io::simulator::SimulatorTransport;
 using memgraph::storage::v3::SchemaProperty;
+using SimT = memgraph::io::simulator::SimulatorTransport;
+using PrimaryKey = std::vector<PropertyValue>;
 
 struct HlcRequest {
   Hlc last_shard_map_version;
@@ -50,6 +54,11 @@ struct GetShardMapResponse {
   ShardMap shard_map;
 };
 
+struct AllocateHlcBatchRequest {
+  Hlc low;
+  Hlc high;
+};
+
 struct AllocateHlcBatchResponse {
   bool success;
   Hlc low;
@@ -76,7 +85,7 @@ struct AllocatePropertyIdsResponse {
 struct SplitShardRequest {
   Hlc previous_shard_map_version;
   LabelId label_id;
-  CompoundKey split_key;
+  PrimaryKey split_key;
 };
 
 struct SplitShardResponse {
@@ -102,26 +111,34 @@ struct DeregisterStorageEngineResponse {
 struct InitializeLabelRequest {
   std::string label_name;
   std::vector<SchemaProperty> schema;
+  size_t replication_factor;
   Hlc last_shard_map_version;
 };
 
 struct InitializeLabelResponse {
   bool success;
+  LabelId new_label_id;
   std::optional<ShardMap> fresher_shard_map;
 };
 
-struct HeartbeatRequest {};
-struct HeartbeatResponse {};
+struct HeartbeatRequest {
+  Address from_storage_manager;
+  std::set<boost::uuids::uuid> initialized_rsms;
+};
+
+struct HeartbeatResponse {
+  std::vector<ShardToInitialize> shards_to_initialize;
+};
 
 using CoordinatorWriteRequests =
     std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
-                 DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest>;
-using CoordinatorWriteResponses =
-    std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse, RegisterStorageEngineResponse,
-                 DeregisterStorageEngineResponse, InitializeLabelResponse, AllocatePropertyIdsResponse>;
+                 DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
+using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
+                                               RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
+                                               InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
 
-using CoordinatorReadRequests = std::variant<GetShardMapRequest, HeartbeatRequest>;
-using CoordinatorReadResponses = std::variant<GetShardMapResponse, HeartbeatResponse>;
+using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
+using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
 
 class Coordinator {
  public:
@@ -141,14 +158,10 @@ class Coordinator {
 
  private:
   ShardMap shard_map_;
-  uint64_t highest_allocated_timestamp_;
+  uint64_t highest_allocated_timestamp_{0};
 
   /// Query engines need to periodically request batches of unique edge IDs.
-  uint64_t highest_allocated_edge_id_;
-
-  static CoordinatorReadResponses HandleRead(HeartbeatRequest && /* heartbeat_request */) {
-    return HeartbeatResponse{};
-  }
+  uint64_t highest_allocated_edge_id_{0};
 
   CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
     GetShardMapResponse res;
@@ -156,104 +169,28 @@ class Coordinator {
     return res;
   }
 
-  CoordinatorWriteResponses ApplyWrite(HlcRequest &&hlc_request) {
-    HlcResponse res{};
+  CoordinatorWriteResponses ApplyWrite(HeartbeatRequest &&heartbeat_request);
 
-    auto hlc_shard_map = shard_map_.GetHlc();
+  CoordinatorWriteResponses ApplyWrite(HlcRequest &&hlc_request);
 
-    MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
-
-    res.new_hlc = Hlc{
-        .logical_id = ++highest_allocated_timestamp_,
-        // TODO(tyler) probably pass some more context to the Coordinator here
-        // so that we can use our wall clock and enforce monotonicity.
-        // .coordinator_wall_clock = io_.Now(),
-    };
-
-    // Allways return fresher shard_map for now.
-    res.fresher_shard_map = std::make_optional(shard_map_);
-
-    return res;
-  }
-
-  CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
-    AllocateEdgeIdBatchResponse res{};
-
-    uint64_t low = highest_allocated_edge_id_;
-
-    highest_allocated_edge_id_ += ahr.batch_size;
-
-    uint64_t high = highest_allocated_edge_id_;
-
-    res.low = low;
-    res.high = high;
-
-    return res;
-  }
+  CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
 
   /// This splits the shard immediately beneath the provided
   /// split key, keeping the assigned peers identical for now,
   /// but letting them be gradually migrated over time.
-  CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request) {
-    SplitShardResponse res{};
-
-    if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
-      res.success = false;
-    } else {
-      res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
-                                          split_shard_request.split_key);
-    }
-
-    return res;
-  }
+  CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
 
   /// This adds the provided storage engine to the standby storage engine pool,
   /// which can be used to rebalance storage over time.
-  static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */) {
-    RegisterStorageEngineResponse res{};
-    // TODO
-
-    return res;
-  }
+  static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);
 
   /// This begins the process of draining the provided storage engine from all raft
   /// clusters that it might be participating in.
-  static CoordinatorWriteResponses ApplyWrite(DeregisterStorageEngineRequest && /* register_storage_engine_request */) {
-    DeregisterStorageEngineResponse res{};
-    // TODO
-    // const Address &address = register_storage_engine_request.address;
-    // storage_engine_pool_.erase(address);
-    // res.success = true;
+  static CoordinatorWriteResponses ApplyWrite(DeregisterStorageEngineRequest && /* register_storage_engine_request */);
 
-    return res;
-  }
+  CoordinatorWriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request);
 
-  CoordinatorWriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
-    InitializeLabelResponse res{};
-
-    bool success = shard_map_.InitializeNewLabel(initialize_label_request.label_name, initialize_label_request.schema,
-                                                 initialize_label_request.last_shard_map_version);
-
-    if (success) {
-      res.fresher_shard_map = shard_map_;
-      res.success = false;
-    } else {
-      res.fresher_shard_map = std::nullopt;
-      res.success = true;
-    }
-
-    return res;
-  }
-
-  CoordinatorWriteResponses ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request) {
-    AllocatePropertyIdsResponse res{};
-
-    auto property_ids = shard_map_.AllocatePropertyIds(allocate_property_ids_request.property_names);
-
-    res.property_ids = property_ids;
-
-    return res;
-  }
+  CoordinatorWriteResponses ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request);
 };
 
 }  // namespace memgraph::coordinator

src/coordinator/coordinator_client.hpp

@@ -21,5 +21,4 @@ using memgraph::io::rsm::RsmClient;
 template <typename IoImpl>
 using CoordinatorClient = RsmClient<IoImpl, CoordinatorWriteRequests, CoordinatorWriteResponses,
                                     CoordinatorReadRequests, CoordinatorReadResponses>;
-
 }  // namespace memgraph::coordinator

src/coordinator/coordinator_worker.hpp

@@ -0,0 +1,168 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <chrono>
+#include <deque>
+#include <memory>
+#include <queue>
+#include <variant>
+
+#include "coordinator/coordinator.hpp"
+#include "coordinator/coordinator_rsm.hpp"
+#include "coordinator/shard_map.hpp"
+#include "io/address.hpp"
+#include "io/future.hpp"
+#include "io/messages.hpp"
+#include "io/rsm/raft.hpp"
+#include "io/time.hpp"
+#include "io/transport.hpp"
+#include "query/v2/requests.hpp"
+
+namespace memgraph::coordinator::coordinator_worker {
+
+/// Obligations:
+/// * ShutDown
+/// * Cron
+/// * RouteMessage
+
+using coordinator::Coordinator;
+using coordinator::CoordinatorRsm;
+using io::Address;
+using io::RequestId;
+using io::Time;
+using io::messages::CoordinatorMessages;
+using msgs::ReadRequests;
+using msgs::ReadResponses;
+using msgs::WriteRequests;
+using msgs::WriteResponses;
+
+struct ShutDown {};
+
+struct Cron {};
+
+struct RouteMessage {
+  CoordinatorMessages message;
+  RequestId request_id;
+  Address to;
+  Address from;
+};
+
+using Message = std::variant<RouteMessage, Cron, ShutDown>;
+
+struct QueueInner {
+  std::mutex mu{};
+  std::condition_variable cv;
+  // TODO(tyler) handle simulator communication std::shared_ptr<std::atomic<int>> blocked;
+
+  // TODO(tyler) investigate using a priority queue that prioritizes messages in a way that
+  // improves overall QoS. For example, maybe we want to schedule raft Append messages
+  // ahead of Read messages or generally writes before reads for lowering the load on the
+  // overall system faster etc... When we do this, we need to make sure to avoid
+  // starvation by sometimes randomizing priorities, rather than following a strict
+  // prioritization.
+  std::deque<Message> queue;
+
+  uint64_t submitted = 0;
+  uint64_t calls_to_pop = 0;
+};
+
+/// There are two reasons to implement our own Queue instead of using
+/// one off-the-shelf:
+/// 1. we will need to know in the simulator when all threads are waiting
+/// 2. we will want to implement our own priority queue within this for QoS
+class Queue {
+  std::shared_ptr<QueueInner> inner_ = std::make_shared<QueueInner>();
+
+ public:
+  void Push(Message &&message) {
+    {
+      MG_ASSERT(inner_.use_count() > 0);
+      std::unique_lock<std::mutex> lock(inner_->mu);
+
+      inner_->submitted++;
+
+      inner_->queue.emplace_back(std::move(message));
+    }  // lock dropped before notifying condition variable
+
+    inner_->cv.notify_all();
+  }
+
+  Message Pop() {
+    MG_ASSERT(inner_.use_count() > 0);
+    std::unique_lock<std::mutex> lock(inner_->mu);
+
+    inner_->calls_to_pop++;
+    inner_->cv.notify_all();
+
+    while (inner_->queue.empty()) {
+      inner_->cv.wait(lock);
+    }
+
+    Message message = std::move(inner_->queue.front());
+    inner_->queue.pop_front();
+
+    return message;
+  }
+
+  void BlockOnQuiescence() const {
+    MG_ASSERT(inner_.use_count() > 0);
+    std::unique_lock<std::mutex> lock(inner_->mu);
+
+    while (inner_->calls_to_pop <= inner_->submitted) {
+      inner_->cv.wait(lock);
+    }
+  }
+};
+
+/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
+template <typename IoImpl>
+class CoordinatorWorker {
+  io::Io<IoImpl> io_;
+  Queue queue_;
+  CoordinatorRsm<IoImpl> coordinator_;
+
+  bool Process(ShutDown && /*shut_down*/) { return false; }
+
+  bool Process(Cron && /* cron */) {
+    coordinator_.Cron();
+    return true;
+  }
+
+  bool Process(RouteMessage &&route_message) {
+    coordinator_.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
+
+    return true;
+  }
+
+ public:
+  CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
+      : io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
+
+  CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
+  CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
+  CoordinatorWorker(const CoordinatorWorker &) = delete;
+  CoordinatorWorker &operator=(const CoordinatorWorker &) = delete;
+  ~CoordinatorWorker() = default;
+
+  void Run() {
+    bool should_continue = true;
+    while (should_continue) {
+      Message message = queue_.Pop();
+
+      should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
+                                   std::move(message));
+    }
+  }
+};
+
+}  // namespace memgraph::coordinator::coordinator_worker

src/coordinator/hybrid_logical_clock.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,6 +11,11 @@
 
 #pragma once
 
+#include <chrono>
+#include <compare>
+#include <ctime>
+#include <iomanip>
+
 #include "io/time.hpp"
 
 namespace memgraph::coordinator {
@@ -18,11 +23,26 @@ namespace memgraph::coordinator {
 using Time = memgraph::io::Time;
 
 /// Hybrid-logical clock
-struct Hlc {
-  uint64_t logical_id;
-  Time coordinator_wall_clock;
+struct Hlc final {
+  uint64_t logical_id{0};
+  Time coordinator_wall_clock = Time::min();
 
-  bool operator==(const Hlc &other) const = default;
+  auto operator<=>(const Hlc &other) const noexcept { return logical_id <=> other.logical_id; }
+
+  bool operator==(const Hlc &other) const noexcept = default;
+  bool operator<(const Hlc &other) const noexcept = default;
+  bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
+  bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
+  bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
+
+  friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
+    auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);
+    in << "Hlc { logical_id: " << hlc.logical_id;
+    in << ", coordinator_wall_clock: " << std::put_time(std::localtime(&wall_clock), "%F %T");
+    in << " }";
+
+    return in;
+  }
 };
 
 }  // namespace memgraph::coordinator

src/coordinator/shard_map.cpp

@@ -0,0 +1,576 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <optional>
+#include <unordered_map>
+#include <vector>
+
+#include "common/types.hpp"
+#include "coordinator/shard_map.hpp"
+#include "spdlog/spdlog.h"
+#include "storage/v3/schemas.hpp"
+#include "storage/v3/temporal.hpp"
+#include "utils/cast.hpp"
+#include "utils/exceptions.hpp"
+#include "utils/string.hpp"
+
+namespace memgraph::coordinator {
+
+using memgraph::common::SchemaType;
+using memgraph::storage::v3::TemporalData;
+using memgraph::storage::v3::TemporalType;
+
+PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
+  PrimaryKey ret{};
+
+  const int64_t min_int = std::numeric_limits<int64_t>::min();
+
+  const TemporalData date{TemporalType::Date, min_int};
+  const TemporalData local_time{TemporalType::LocalTime, min_int};
+  const TemporalData local_date_time{TemporalType::LocalDateTime, min_int};
+  const TemporalData duration{TemporalType::Duration, min_int};
+
+  for (const auto &schema_property : schema) {
+    switch (schema_property.type) {
+      case SchemaType::BOOL:
+        ret.emplace_back(PropertyValue(false));
+        break;
+      case SchemaType::INT:
+        ret.emplace_back(PropertyValue(min_int));
+        break;
+      case SchemaType::STRING:
+        ret.emplace_back(PropertyValue(""));
+        break;
+      case SchemaType::DATE:
+        ret.emplace_back(PropertyValue(date));
+        break;
+      case SchemaType::LOCALTIME:
+        ret.emplace_back(PropertyValue(local_time));
+        break;
+      case SchemaType::LOCALDATETIME:
+        ret.emplace_back(PropertyValue(local_date_time));
+        break;
+      case SchemaType::DURATION:
+        ret.emplace_back(PropertyValue(duration));
+        break;
+    }
+  }
+
+  return ret;
+}
+
+ShardMap ShardMap::Parse(std::istream &input_stream) {
+  ShardMap shard_map;
+  const auto read_size = [&input_stream] {
+    size_t size{0};
+    input_stream >> size;
+    return size;
+  };
+
+  // Reads a string until the next whitespace
+  const auto read_word = [&input_stream] {
+    std::string word;
+    input_stream >> word;
+    return word;
+  };
+
+  const auto read_names = [&read_size, &read_word] {
+    const auto number_of_names = read_size();
+    spdlog::trace("Reading {} names", number_of_names);
+    std::vector<std::string> names;
+    names.reserve(number_of_names);
+
+    for (auto name_index = 0; name_index < number_of_names; ++name_index) {
+      names.push_back(read_word());
+      spdlog::trace("Read '{}'", names.back());
+    }
+    return names;
+  };
+
+  const auto read_line = [&input_stream] {
+    std::string line;
+    std::getline(input_stream, line);
+    return line;
+  };
+
+  const auto parse_type = [](const std::string &type) {
+    static const auto type_map = std::unordered_map<std::string, common::SchemaType>{
+        {"string", common::SchemaType::STRING}, {"int", common::SchemaType::INT}, {"bool", common::SchemaType::BOOL}};
+    const auto lower_case_type = utils::ToLowerCase(type);
+    auto it = type_map.find(lower_case_type);
+    MG_ASSERT(it != type_map.end(), "Invalid type in split files: {}", type);
+    return it->second;
+  };
+
+  const auto parse_property_value = [](std::string text, const common::SchemaType type) {
+    if (type == common::SchemaType::STRING) {
+      return storage::v3::PropertyValue{std::move(text)};
+    }
+    if (type == common::SchemaType::INT) {
+      size_t processed{0};
+      int64_t value = std::stoll(text, &processed);
+      MG_ASSERT(processed == text.size() || text[processed] == ' ', "Invalid integer format: '{}'", text);
+      return storage::v3::PropertyValue{value};
+    }
+    LOG_FATAL("Not supported type: {}", utils::UnderlyingCast(type));
+  };
+
+  spdlog::debug("Reading properties");
+  const auto properties = read_names();
+  MG_ASSERT(shard_map.AllocatePropertyIds(properties).size() == properties.size(),
+            "Unexpected number of properties created!");
+
+  spdlog::debug("Reading edge types");
+  const auto edge_types = read_names();
+  MG_ASSERT(shard_map.AllocateEdgeTypeIds(edge_types).size() == edge_types.size(),
+            "Unexpected number of properties created!");
+
+  spdlog::debug("Reading primary labels");
+  const auto number_of_primary_labels = read_size();
+  spdlog::debug("Reading {} primary labels", number_of_primary_labels);
+
+  for (auto label_index = 0; label_index < number_of_primary_labels; ++label_index) {
+    const auto primary_label = read_word();
+    spdlog::debug("Reading primary label named '{}'", primary_label);
+    const auto number_of_primary_properties = read_size();
+    spdlog::debug("Reading {} primary properties", number_of_primary_properties);
+    std::vector<std::string> pp_names;
+    std::vector<common::SchemaType> pp_types;
+    pp_names.reserve(number_of_primary_properties);
+    pp_types.reserve(number_of_primary_properties);
+    for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
+      pp_names.push_back(read_word());
+      spdlog::debug("Reading primary property named '{}'", pp_names.back());
+      pp_types.push_back(parse_type(read_word()));
+    }
+    auto pp_mapping = shard_map.AllocatePropertyIds(pp_names);
+    std::vector<SchemaProperty> schema;
+    schema.reserve(number_of_primary_properties);
+
+    for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
+      schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
+    }
+    const auto hlc = shard_map.GetHlc();
+    MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
+              "Cannot initialize new label: {}", primary_label);
+
+    const auto number_of_split_points = read_size();
+    spdlog::debug("Reading {} split points", number_of_split_points);
+
+    [[maybe_unused]] const auto remainder_from_last_line = read_line();
+    for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
+      const auto line = read_line();
+      spdlog::debug("Read split point '{}'", line);
+      MG_ASSERT(line.front() == '[', "Invalid split file format!");
+      MG_ASSERT(line.back() == ']', "Invalid split file format!");
+      std::string_view line_view{line};
+      line_view.remove_prefix(1);
+      line_view.remove_suffix(1);
+      static constexpr std::string_view kDelimiter{","};
+      auto pk_values_as_text = utils::Split(line_view, kDelimiter);
+      std::vector<PropertyValue> pk;
+      pk.reserve(number_of_primary_properties);
+      MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
+                "Split point contains invalid number of values '{}'", line);
+
+      for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
+        pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
+      }
+      shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
+    }
+  }
+
+  return shard_map;
+}
+
+std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map) {
+  using utils::print_helpers::operator<<;
+
+  in << "ShardMap { shard_map_version: " << shard_map.shard_map_version;
+  in << ", max_property_id: " << shard_map.max_property_id;
+  in << ", max_edge_type_id: " << shard_map.max_edge_type_id;
+  in << ", properties: " << shard_map.properties;
+  in << ", edge_types: " << shard_map.edge_types;
+  in << ", max_label_id: " << shard_map.max_label_id;
+  in << ", labels: " << shard_map.labels;
+  in << ", label_spaces: " << shard_map.label_spaces;
+  in << ", schemas: " << shard_map.schemas;
+  in << "}";
+  return in;
+}
+
+Shards ShardMap::GetShardsForLabel(const LabelName &label) const {
+  const auto id = labels.at(label);
+  const auto &shards = label_spaces.at(id).shards;
+  return shards;
+}
+
+std::vector<Shards> ShardMap::GetAllShards() const {
+  std::vector<Shards> all_shards;
+  all_shards.reserve(label_spaces.size());
+  std::transform(label_spaces.begin(), label_spaces.end(), std::back_inserter(all_shards),
+                 [](const auto &label_space) { return label_space.second.shards; });
+  return all_shards;
+}
+
+// TODO(gabor) later we will want to update the wallclock time with
+// the given Io<impl>'s time as well
+Hlc ShardMap::IncrementShardMapVersion() noexcept {
+  ++shard_map_version.logical_id;
+  return shard_map_version;
+}
+
+// TODO(antaljanosbenjamin) use a single map for all name id
+// mapping and a single counter to maintain the next id
+std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
+  std::unordered_map<uint64_t, std::string> id_to_names;
+
+  const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) {
+    for (const auto &[name, id] : name_to_id_type) {
+      id_to_names.emplace(id.AsUint(), name);
+    }
+  };
+
+  map_type_ids(edge_types);
+  map_type_ids(labels);
+  map_type_ids(properties);
+
+  return id_to_names;
+}
+
+Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
+
+boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
+  return boost::uuids::uuid{0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            0,
+                            static_cast<unsigned char>(shard_id >> 56U),
+                            static_cast<unsigned char>(shard_id >> 48U),
+                            static_cast<unsigned char>(shard_id >> 40U),
+                            static_cast<unsigned char>(shard_id >> 32U),
+                            static_cast<unsigned char>(shard_id >> 24U),
+                            static_cast<unsigned char>(shard_id >> 16U),
+                            static_cast<unsigned char>(shard_id >> 8U),
+                            static_cast<unsigned char>(shard_id)};
+}
+
+std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
+                                                      std::set<boost::uuids::uuid> initialized) {
+  std::vector<ShardToInitialize> ret{};
+
+  bool mutated = false;
+
+  for (auto &[label_id, label_space] : label_spaces) {
+    for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
+      auto &[low_key, shard] = *it;
+      std::optional<PrimaryKey> high_key;
+      if (const auto next_it = std::next(it); next_it != label_space.shards.end()) {
+        high_key = next_it->first;
+      }
+      // TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
+      bool machine_contains_shard = false;
+
+      for (auto &aas : shard.peers) {
+        if (initialized.contains(aas.address.unique_id)) {
+          machine_contains_shard = true;
+          if (aas.status != Status::CONSENSUS_PARTICIPANT) {
+            mutated = true;
+            spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
+            aas.status = Status::CONSENSUS_PARTICIPANT;
+          }
+        } else {
+          const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
+                                    aas.address.last_known_port == storage_manager.last_known_port;
+          if (same_machine) {
+            machine_contains_shard = true;
+            spdlog::info("reminding shard manager that they should begin participating in shard");
+
+            ret.push_back(ShardToInitialize{
+                .uuid = aas.address.unique_id,
+                .label_id = label_id,
+                .min_key = low_key,
+                .max_key = high_key,
+                .schema = schemas[label_id],
+                .config = Config{},
+                .id_to_names = IdToNames(),
+            });
+          }
+        }
+      }
+
+      if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
+        // increment version for each new uuid for deterministic creation
+        IncrementShardMapVersion();
+
+        Address address = storage_manager;
+
+        // TODO(tyler) use deterministic UUID so that coordinators don't diverge here
+        address.unique_id = NewShardUuid(shard_map_version.logical_id);
+
+        spdlog::info("assigning shard manager to shard");
+
+        ret.push_back(ShardToInitialize{
+            .uuid = address.unique_id,
+            .label_id = label_id,
+            .min_key = low_key,
+            .max_key = high_key,
+            .schema = schemas[label_id],
+            .config = Config{},
+            .id_to_names = IdToNames(),
+        });
+
+        AddressAndStatus aas = {
+            .address = address,
+            .status = Status::INITIALIZING,
+        };
+
+        shard.peers.emplace_back(aas);
+      }
+    }
+  }
+
+  if (mutated) {
+    IncrementShardMapVersion();
+  }
+  return ret;
+}
+
+bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
+  if (previous_shard_map_version != shard_map_version) {
+    return false;
+  }
+
+  auto &label_space = label_spaces.at(label_id);
+  auto &shards_in_map = label_space.shards;
+
+  MG_ASSERT(!shards_in_map.empty());
+  MG_ASSERT(!shards_in_map.contains(key));
+  MG_ASSERT(label_spaces.contains(label_id));
+
+  // Finding the ShardMetadata that the new PrimaryKey should map to.
+  auto prev = std::prev(shards_in_map.upper_bound(key));
+  ShardMetadata duplicated_shard = prev->second;
+
+  // Apply the split
+  shards_in_map[key] = duplicated_shard;
+
+  IncrementShardMapVersion();
+
+  return true;
+}
+
+std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
+                                                    size_t replication_factor, Hlc last_shard_map_version) {
+  if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
+    return std::nullopt;
+  }
+
+  const LabelId label_id = LabelId::FromUint(++max_label_id);
+
+  labels.emplace(std::move(label_name), label_id);
+
+  PrimaryKey initial_key = SchemaToMinKey(schema);
+  ShardMetadata empty_shard = {};
+
+  Shards shards = {
+      {initial_key, empty_shard},
+  };
+
+  LabelSpace label_space{
+      .schema = schema,
+      .shards = shards,
+      .replication_factor = replication_factor,
+  };
+  schemas[label_id] = std::move(schema);
+
+  label_spaces.emplace(label_id, label_space);
+
+  IncrementShardMapVersion();
+
+  return label_id;
+}
+
+void ShardMap::AddServer(Address server_address) {
+  // Find a random place for the server to plug in
+}
+
+std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
+  if (const auto it = labels.find(label); it != labels.end()) {
+    return it->second;
+  }
+  return std::nullopt;
+}
+
+const std::string &ShardMap::GetLabelName(const LabelId label) const {
+  if (const auto it =
+          std::ranges::find_if(labels, [label](const auto &name_id_pair) { return name_id_pair.second == label; });
+      it != labels.end()) {
+    return it->first;
+  }
+  throw utils::BasicException("GetLabelName fails on the given label id!");
+}
+
+std::optional<PropertyId> ShardMap::GetPropertyId(const std::string &property_name) const {
+  if (const auto it = properties.find(property_name); it != properties.end()) {
+    return it->second;
+  }
+  return std::nullopt;
+}
+
+const std::string &ShardMap::GetPropertyName(const PropertyId property) const {
+  if (const auto it = std::ranges::find_if(
+          properties, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
+      it != properties.end()) {
+    return it->first;
+  }
+  throw utils::BasicException("PropertyId not found!");
+}
+
+std::optional<EdgeTypeId> ShardMap::GetEdgeTypeId(const std::string &edge_type) const {
+  if (const auto it = edge_types.find(edge_type); it != edge_types.end()) {
+    return it->second;
+  }
+  return std::nullopt;
+}
+
+const std::string &ShardMap::GetEdgeTypeName(const EdgeTypeId property) const {
+  if (const auto it = std::ranges::find_if(
+          edge_types, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
+      it != edge_types.end()) {
+    return it->first;
+  }
+  throw utils::BasicException("EdgeTypeId not found!");
+}
+
+Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key,
+                                   const PrimaryKey &end_key) const {
+  MG_ASSERT(start_key <= end_key);
+  MG_ASSERT(labels.contains(label_name));
+
+  LabelId label_id = labels.at(label_name);
+
+  const auto &label_space = label_spaces.at(label_id);
+
+  const auto &shards_for_label = label_space.shards;
+
+  MG_ASSERT(shards_for_label.begin()->first <= start_key,
+            "the ShardMap must always contain a minimal key that is less than or equal to any requested key");
+
+  auto it = std::prev(shards_for_label.upper_bound(start_key));
+  const auto end_it = shards_for_label.upper_bound(end_key);
+
+  Shards shards{};
+
+  std::copy(it, end_it, std::inserter(shards, shards.end()));
+
+  return shards;
+}
+
+ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
+  MG_ASSERT(labels.contains(label_name));
+
+  LabelId label_id = labels.at(label_name);
+
+  const auto &label_space = label_spaces.at(label_id);
+
+  MG_ASSERT(label_space.shards.begin()->first <= key,
+            "the ShardMap must always contain a minimal key that is less than or equal to any requested key");
+
+  return std::prev(label_space.shards.upper_bound(key))->second;
+}
+
+ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
+  MG_ASSERT(label_spaces.contains(label_id));
+
+  const auto &label_space = label_spaces.at(label_id);
+
+  MG_ASSERT(label_space.shards.begin()->first <= key,
+            "the ShardMap must always contain a minimal key that is less than or equal to any requested key");
+
+  return std::prev(label_space.shards.upper_bound(key))->second;
+}
+
+PropertyMap ShardMap::AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
+  PropertyMap ret{};
+
+  bool mutated = false;
+
+  for (const auto &property_name : new_properties) {
+    if (properties.contains(property_name)) {
+      auto property_id = properties.at(property_name);
+      ret.emplace(property_name, property_id);
+    } else {
+      mutated = true;
+
+      const PropertyId property_id = PropertyId::FromUint(++max_property_id);
+      ret.emplace(property_name, property_id);
+      properties.emplace(property_name, property_id);
+    }
+  }
+
+  if (mutated) {
+    IncrementShardMapVersion();
+  }
+
+  return ret;
+}
+
+EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types) {
+  EdgeTypeIdMap ret;
+
+  bool mutated = false;
+
+  for (const auto &edge_type_name : new_edge_types) {
+    if (edge_types.contains(edge_type_name)) {
+      auto edge_type_id = edge_types.at(edge_type_name);
+      ret.emplace(edge_type_name, edge_type_id);
+    } else {
+      mutated = true;
+
+      const EdgeTypeId edge_type_id = EdgeTypeId::FromUint(++max_edge_type_id);
+      ret.emplace(edge_type_name, edge_type_id);
+      edge_types.emplace(edge_type_name, edge_type_id);
+    }
+  }
+
+  if (mutated) {
+    IncrementShardMapVersion();
+  }
+
+  return ret;
+}
+
+bool ShardMap::ClusterInitialized() const {
+  for (const auto &[label_id, label_space] : label_spaces) {
+    for (const auto &[low_key, shard] : label_space.shards) {
+      if (shard.peers.size() < label_space.replication_factor) {
+        spdlog::info("label_space below desired replication factor");
+        return false;
+      }
+
+      for (const auto &aas : shard.peers) {
+        if (aas.status != Status::CONSENSUS_PARTICIPANT) {
+          spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
+          return false;
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+}  // namespace memgraph::coordinator

src/coordinator/shard_map.hpp

@@ -11,20 +11,37 @@
 
 #pragma once
 
+#include <algorithm>
+#include <limits>
 #include <map>
+#include <set>
 #include <vector>
 
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+
+#include "common/types.hpp"
 #include "coordinator/hybrid_logical_clock.hpp"
 #include "io/address.hpp"
+#include "storage/v3/config.hpp"
 #include "storage/v3/id_types.hpp"
+#include "storage/v3/name_id_mapper.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/schemas.hpp"
+#include "storage/v3/temporal.hpp"
+#include "utils/exceptions.hpp"
+#include "utils/print_helpers.hpp"
 
 namespace memgraph::coordinator {
 
+constexpr int64_t kNotExistingId{0};
+
 using memgraph::io::Address;
+using memgraph::storage::v3::Config;
+using memgraph::storage::v3::EdgeTypeId;
 using memgraph::storage::v3::LabelId;
 using memgraph::storage::v3::PropertyId;
+using memgraph::storage::v3::PropertyValue;
 using memgraph::storage::v3::SchemaProperty;
 
 enum class Status : uint8_t {
@@ -38,152 +55,150 @@ enum class Status : uint8_t {
 struct AddressAndStatus {
   memgraph::io::Address address;
   Status status;
+
+  friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
+
+  friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
+    in << "AddressAndStatus { address: ";
+    in << address_and_status.address;
+    if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
+      in << ", status: CONSENSUS_PARTICIPANT }";
+    } else {
+      in << ", status: INITIALIZING }";
+    }
+
+    return in;
+  }
+
+  friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
+    return lhs.address == rhs.address;
+  }
 };
 
-using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
-using Shard = std::vector<AddressAndStatus>;
-using Shards = std::map<CompoundKey, Shard>;
+using PrimaryKey = std::vector<PropertyValue>;
+
+struct ShardMetadata {
+  std::vector<AddressAndStatus> peers;
+  uint64_t version;
+
+  friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
+    using utils::print_helpers::operator<<;
+
+    in << "ShardMetadata { peers: ";
+    in << shard.peers;
+    in << " version: ";
+    in << shard.version;
+    in << " }";
+
+    return in;
+  }
+
+  friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
+
+  friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
+    if (lhs.peers != rhs.peers) {
+      return lhs.peers < rhs.peers;
+    }
+
+    return lhs.version < rhs.version;
+  }
+};
+
+using Shards = std::map<PrimaryKey, ShardMetadata>;
 using LabelName = std::string;
 using PropertyName = std::string;
+using EdgeTypeName = std::string;
 using PropertyMap = std::map<PropertyName, PropertyId>;
+using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
+
+struct ShardToInitialize {
+  boost::uuids::uuid uuid;
+  LabelId label_id;
+  PrimaryKey min_key;
+  std::optional<PrimaryKey> max_key;
+  std::vector<SchemaProperty> schema;
+  Config config;
+  std::unordered_map<uint64_t, std::string> id_to_names;
+};
+
+PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
 
 struct LabelSpace {
   std::vector<SchemaProperty> schema;
-  std::map<CompoundKey, Shard> shards;
+  // Maps between the smallest primary key stored in the shard and the shard
+  std::map<PrimaryKey, ShardMetadata> shards;
+  size_t replication_factor;
+
+  friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
+    using utils::print_helpers::operator<<;
+
+    in << "LabelSpace { schema: ";
+    in << label_space.schema;
+    in << ", shards: ";
+    in << label_space.shards;
+    in << ", replication_factor: " << label_space.replication_factor << "}";
+
+    return in;
+  }
 };
 
 struct ShardMap {
   Hlc shard_map_version;
-  uint64_t max_property_id;
+  uint64_t max_property_id{kNotExistingId};
+  uint64_t max_edge_type_id{kNotExistingId};
   std::map<PropertyName, PropertyId> properties;
-  uint64_t max_label_id;
-  std::map<LabelName, LabelId> labels;
+  std::map<EdgeTypeName, EdgeTypeId> edge_types;
+  uint64_t max_label_id{kNotExistingId};
   std::map<LabelId, LabelSpace> label_spaces;
   std::map<LabelId, std::vector<SchemaProperty>> schemas;
+  std::map<LabelName, LabelId> labels;
+
+  [[nodiscard]] static ShardMap Parse(std::istream &input_stream);
+  friend std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map);
+
+  Shards GetShardsForLabel(const LabelName &label) const;
+
+  std::vector<Shards> GetAllShards() const;
 
   // TODO(gabor) later we will want to update the wallclock time with
   // the given Io<impl>'s time as well
-  Hlc IncrementShardMapVersion() noexcept {
-    ++shard_map_version.logical_id;
-    return shard_map_version;
-  }
+  Hlc IncrementShardMapVersion() noexcept;
+  Hlc GetHlc() const noexcept;
 
-  Hlc GetHlc() const noexcept { return shard_map_version; }
+  std::unordered_map<uint64_t, std::string> IdToNames();
 
-  bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const CompoundKey &key) {
-    if (previous_shard_map_version != shard_map_version) {
-      return false;
-    }
+  // Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
+  std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
 
-    auto &label_space = label_spaces.at(label_id);
-    auto &shards_in_map = label_space.shards;
+  bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
 
-    MG_ASSERT(!shards_in_map.contains(key));
-    MG_ASSERT(label_spaces.contains(label_id));
+  std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
+                                            size_t replication_factor, Hlc last_shard_map_version);
 
-    // Finding the Shard that the new CompoundKey should map to.
-    auto prev = std::prev(shards_in_map.upper_bound(key));
-    Shard duplicated_shard = prev->second;
+  void AddServer(Address server_address);
 
-    // Apply the split
-    shards_in_map[key] = duplicated_shard;
+  std::optional<LabelId> GetLabelId(const std::string &label) const;
+  // TODO(antaljanosbenjamin): Remove this and instead use NameIdMapper
+  const std::string &GetLabelName(LabelId label) const;
+  std::optional<PropertyId> GetPropertyId(const std::string &property_name) const;
+  const std::string &GetPropertyName(PropertyId property) const;
+  std::optional<EdgeTypeId> GetEdgeTypeId(const std::string &edge_type) const;
+  const std::string &GetEdgeTypeName(EdgeTypeId property) const;
 
-    return true;
-  }
+  Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
 
-  bool InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema, Hlc last_shard_map_version) {
-    if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
-      return false;
-    }
+  ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
 
-    const LabelId label_id = LabelId::FromUint(++max_label_id);
+  ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
 
-    labels.emplace(std::move(label_name), label_id);
+  PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
 
-    LabelSpace label_space{
-        .schema = std::move(schema),
-        .shards = Shards{},
-    };
+  EdgeTypeIdMap AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types);
 
-    label_spaces.emplace(label_id, label_space);
-
-    IncrementShardMapVersion();
-
-    return true;
-  }
-
-  void AddServer(Address server_address) {
-    // Find a random place for the server to plug in
-  }
-
-  Shards GetShardsForRange(const LabelName &label_name, const CompoundKey &start_key, const CompoundKey &end_key) const {
-    MG_ASSERT(start_key <= end_key);
-    MG_ASSERT(labels.contains(label_name));
-
-    LabelId label_id = labels.at(label_name);
-
-    const auto &label_space = label_spaces.at(label_id);
-
-    const auto &shards_for_label = label_space.shards;
-
-    MG_ASSERT(shards_for_label.begin()->first <= start_key,
-              "the ShardMap must always contain a minimal key that is less than or equal to any requested key");
-
-    auto it = std::prev(shards_for_label.upper_bound(start_key));
-    const auto end_it = shards_for_label.upper_bound(end_key);
-
-    Shards shards{};
-
-    std::copy(it, end_it, std::inserter(shards, shards.end()));
-
-    return shards;
-  }
-
-  Shard GetShardForKey(const LabelName &label_name, const CompoundKey &key) const {
-    MG_ASSERT(labels.contains(label_name));
-
-    LabelId label_id = labels.at(label_name);
-
-    const auto &label_space = label_spaces.at(label_id);
-
-    MG_ASSERT(label_space.shards.begin()->first <= key,
-              "the ShardMap must always contain a minimal key that is less than or equal to any requested key");
-
-    return std::prev(label_space.shards.upper_bound(key))->second;
-  }
-
-  PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
-    PropertyMap ret{};
-
-    bool mutated = false;
-
-    for (const auto &property_name : new_properties) {
-      if (properties.contains(property_name)) {
-        auto property_id = properties.at(property_name);
-        ret.emplace(property_name, property_id);
-      } else {
-        mutated = true;
-
-        const PropertyId property_id = PropertyId::FromUint(++max_property_id);
-        ret.emplace(property_name, property_id);
-        properties.emplace(property_name, property_id);
-      }
-    }
-
-    if (mutated) {
-      IncrementShardMapVersion();
-    }
-
-    return ret;
-  }
-
-  std::optional<PropertyId> GetPropertyId(const std::string &property_name) const {
-    if (properties.contains(property_name)) {
-      return properties.at(property_name);
-    }
-
-    return std::nullopt;
-  }
+  /// Returns true if all shards have the desired number of replicas and they are in
+  /// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
+  /// there is also an active leader for each shard.
+  bool ClusterInitialized() const;
 };
 
 }  // namespace memgraph::coordinator

src/expr/CMakeLists.txt

@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
 target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
-target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)
+target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)

src/expr/ast.hpp

@@ -13,7 +13,7 @@
 #ifndef MG_AST_INCLUDE_PATH
 #ifdef MG_CLANG_TIDY_CHECK
 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
-#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"
+#include "query/v2/bindings/bindings.hpp"
 #else
 #error Missing AST include path
 #endif
@@ -21,8 +21,6 @@
 
 #ifndef MG_INJECTED_NAMESPACE_NAME
 #ifdef MG_CLANG_TIDY_CHECK
-// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
-#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2
 #else
 #error Missing AST namespace
 #endif

src/expr/ast/cypher_main_visitor.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ constexpr char kId[] = "ID";
 
 namespace MG_INJECTED_NAMESPACE_NAME {
 namespace detail {
-using antlropencypher::MemgraphCypher;
+using antlropencypher::v2::MemgraphCypher;
 
 template <typename TVisitor>
 std::optional<std::pair<Expression *, size_t>> VisitMemoryLimit(MemgraphCypher::MemoryLimitContext *memory_limit_ctx,
@@ -211,13 +211,13 @@ inline std::string_view ToString(const PulsarConfigKey key) {
 }
 }  // namespace detail
 
-using antlropencypher::MemgraphCypher;
+using antlropencypher::v2::MemgraphCypher;
 
 struct ParsingContext {
   bool is_query_cached = false;
 };
 
-class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
+class CypherMainVisitor : public antlropencypher::v2::MemgraphCypherBaseVisitor {
  public:
   explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {}
 

src/expr/ast/pretty_print_ast_to_original_expression.hpp

@@ -0,0 +1,279 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <iostream>
+#include <type_traits>
+
+#include "expr/ast.hpp"
+#include "expr/typed_value.hpp"
+#include "utils/algorithm.hpp"
+#include "utils/logging.hpp"
+#include "utils/string.hpp"
+
+namespace memgraph::expr {
+inline constexpr const char *identifier_node_symbol = "MG_SYMBOL_NODE";
+inline constexpr const char *identifier_edge_symbol = "MG_SYMBOL_EDGE";
+
+namespace detail {
+template <typename T>
+void PrintObject(std::ostream *out, const T &arg) {
+  static_assert(!std::is_convertible<T, Expression *>::value,
+                "This overload shouldn't be called with pointers convertible "
+                "to Expression *. This means your other PrintObject overloads aren't "
+                "being called for certain AST nodes when they should (or perhaps such "
+                "overloads don't exist yet).");
+  *out << arg;
+}
+
+inline void PrintObject(std::ostream *out, const std::string &str) { *out << str; }
+
+inline void PrintObject(std::ostream * /*out*/, Aggregation::Op /*op*/) {
+  throw utils::NotYetImplemented("PrintObject: Aggregation::Op");
+}
+
+inline void PrintObject(std::ostream * /*out*/, Expression * /*expr*/);
+
+inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
+
+template <typename T>
+void PrintObject(std::ostream * /*out*/, const std::vector<T> & /*vec*/) {
+  throw utils::NotYetImplemented("PrintObject: vector<T>");
+}
+
+template <typename T>
+void PrintObject(std::ostream * /*out*/, const std::vector<T, utils::Allocator<T>> & /*vec*/) {
+  throw utils::NotYetImplemented("PrintObject: vector<T, utils::Allocator<T>>");
+}
+
+template <typename K, typename V>
+void PrintObject(std::ostream * /*out*/, const std::map<K, V> & /*map*/) {
+  throw utils::NotYetImplemented("PrintObject: map<K, V>");
+}
+
+template <typename T>
+void PrintObject(std::ostream * /*out*/, const utils::pmr::map<utils::pmr::string, T> & /*map*/) {
+  throw utils::NotYetImplemented("PrintObject: map<utils::pmr::string, V>");
+}
+
+template <typename T1, typename T2, typename T3>
+inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
+  using TypedValue = TypedValueT<T1, T2, T3>;
+  switch (value.type()) {
+    case TypedValue::Type::Null:
+      *out << "null";
+      break;
+    case TypedValue::Type::String:
+      PrintObject(out, value.ValueString());
+      break;
+    case TypedValue::Type::Bool:
+      *out << (value.ValueBool() ? "true" : "false");
+      break;
+    case TypedValue::Type::Int:
+      PrintObject(out, value.ValueInt());
+      break;
+    case TypedValue::Type::Double:
+      PrintObject(out, value.ValueDouble());
+      break;
+    case TypedValue::Type::List:
+      PrintObject(out, value.ValueList());
+      break;
+    case TypedValue::Type::Map:
+      PrintObject(out, value.ValueMap());
+      break;
+    case TypedValue::Type::Date:
+      PrintObject(out, value.ValueDate());
+      break;
+    case TypedValue::Type::Duration:
+      PrintObject(out, value.ValueDuration());
+      break;
+    case TypedValue::Type::LocalTime:
+      PrintObject(out, value.ValueLocalTime());
+      break;
+    case TypedValue::Type::LocalDateTime:
+      PrintObject(out, value.ValueLocalDateTime());
+      break;
+    default:
+      MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
+  }
+}
+
+template <typename T>
+void PrintOperatorArgs(const std::string & /*name*/, std::ostream *out, bool with_parenthesis, const T &arg) {
+  PrintObject(out, arg);
+  if (with_parenthesis) {
+    *out << ")";
+  }
+}
+
+template <typename T, typename... Ts>
+void PrintOperatorArgs(const std::string &name, std::ostream *out, bool with_parenthesis, const T &arg,
+                       const Ts &...args) {
+  PrintObject(out, arg);
+  *out << " " << name << " ";
+  PrintOperatorArgs(name, out, with_parenthesis, args...);
+}
+
+template <typename... Ts>
+void PrintOperator(const std::string &name, std::ostream *out, bool with_parenthesis, const Ts &...args) {
+  if (with_parenthesis) {
+    *out << "(";
+  }
+  PrintOperatorArgs(name, out, with_parenthesis, args...);
+}
+
+// new
+template <typename T>
+void PrintOperatorArgs(std::ostream *out, const T &arg) {
+  PrintObject(out, arg);
+}
+
+template <typename T, typename... Ts>
+void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
+  PrintObject(out, arg);
+  PrintOperatorArgs(out, args...);
+}
+
+template <typename... Ts>
+void PrintOperator(std::ostream *out, const Ts &...args) {
+  PrintOperatorArgs(out, args...);
+}
+}  // namespace detail
+
+class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
+ public:
+  explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
+
+  // Unary operators
+  // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR)      \
+  /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
+  void Visit(OP_NODE &op) override { detail::PrintOperator(OP_STR, out_, false /*with_parenthesis*/, op.expression_); }
+
+  UNARY_OPERATOR_VISIT(NotOperator, "Not");
+  UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
+  UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
+  UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
+
+#undef UNARY_OPERATOR_VISIT
+
+  // Binary operators
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR)                                                        \
+  /* NOLINTNEXTLINE(bugprone-macro-parentheses) */                                                    \
+  void Visit(OP_NODE &op) override {                                                                  \
+    detail::PrintOperator(OP_STR, out_, true /*with_parenthesis*/, op.expression1_, op.expression2_); \
+  }
+// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
+#define BINARY_OPERATOR_VISIT_NOT_IMPL(OP_NODE, OP_STR) \
+  /* NOLINTNEXTLINE(bugprone-macro-parentheses) */      \
+  void Visit(OP_NODE & /*op*/) override { throw utils::NotYetImplemented("OP_NODE"); }
+
+  BINARY_OPERATOR_VISIT(OrOperator, "Or");
+  BINARY_OPERATOR_VISIT(XorOperator, "Xor");
+  BINARY_OPERATOR_VISIT(AndOperator, "And");
+  BINARY_OPERATOR_VISIT(AdditionOperator, "+");
+  BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
+  BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
+  BINARY_OPERATOR_VISIT(DivisionOperator, "/");
+  BINARY_OPERATOR_VISIT(ModOperator, "%");
+  BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
+  BINARY_OPERATOR_VISIT(EqualOperator, "=");
+  BINARY_OPERATOR_VISIT(LessOperator, "<");
+  BINARY_OPERATOR_VISIT(GreaterOperator, ">");
+  BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
+  BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
+  BINARY_OPERATOR_VISIT_NOT_IMPL(InListOperator, "In");
+  BINARY_OPERATOR_VISIT_NOT_IMPL(SubscriptOperator, "Subscript");
+
+#undef BINARY_OPERATOR_VISIT
+#undef BINARY_OPERATOR_VISIT_NOT_IMPL
+
+  // Other
+  void Visit(ListSlicingOperator & /*op*/) override { throw utils::NotYetImplemented("ListSlicingOperator"); }
+
+  void Visit(IfOperator & /*op*/) override { throw utils::NotYetImplemented("IfOperator"); }
+
+  void Visit(ListLiteral & /*op*/) override { throw utils::NotYetImplemented("ListLiteral"); }
+
+  void Visit(MapLiteral & /*op*/) override { throw utils::NotYetImplemented("MapLiteral"); }
+
+  void Visit(LabelsTest & /*op*/) override { throw utils::NotYetImplemented("LabelsTest"); }
+
+  void Visit(Aggregation & /*op*/) override { throw utils::NotYetImplemented("Aggregation"); }
+
+  void Visit(Function & /*op*/) override { throw utils::NotYetImplemented("Function"); }
+
+  void Visit(Reduce & /*op*/) override { throw utils::NotYetImplemented("Reduce"); }
+
+  void Visit(Coalesce & /*op*/) override { throw utils::NotYetImplemented("Coalesce"); }
+
+  void Visit(Extract & /*op*/) override { throw utils::NotYetImplemented("Extract"); }
+
+  void Visit(All & /*op*/) override { throw utils::NotYetImplemented("All"); }
+
+  void Visit(Single & /*op*/) override { throw utils::NotYetImplemented("Single"); }
+
+  void Visit(Any & /*op*/) override { throw utils::NotYetImplemented("Any"); }
+
+  void Visit(None & /*op*/) override { throw utils::NotYetImplemented("None"); }
+
+  void Visit(Identifier &op) override {
+    auto is_node = true;
+    auto is_edge = false;
+    auto is_other = false;
+    if (is_node) {
+      detail::PrintOperator(out_, identifier_node_symbol);
+    } else if (is_edge) {
+      detail::PrintOperator(out_, identifier_edge_symbol);
+    } else {
+      MG_ASSERT(is_other);
+      detail::PrintOperator(out_, op.name_);
+    }
+  }
+
+  void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
+
+  void Visit(PropertyLookup &op) override { detail::PrintOperator(out_, op.expression_, ".", op.property_.name); }
+
+  void Visit(ParameterLookup & /*op*/) override { throw utils::NotYetImplemented("ParameterLookup"); }
+
+  void Visit(NamedExpression & /*op*/) override { throw utils::NotYetImplemented("NamedExpression"); }
+
+  void Visit(RegexMatch & /*op*/) override { throw utils::NotYetImplemented("RegexMatch"); }
+
+ private:
+  std::ostream *out_;
+};
+
+namespace detail {
+inline void PrintObject(std::ostream *out, Expression *expr) {
+  if (expr) {
+    ExpressionPrettyPrinter printer{out};
+    expr->Accept(printer);
+  } else {
+    *out << "<null>";
+  }
+}
+}  // namespace detail
+
+inline void PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(Expression *expr, std::ostream *out) {
+  ExpressionPrettyPrinter printer{out};
+  expr->Accept(printer);
+}
+
+inline std::string ExpressiontoStringWhileReplacingNodeAndEdgeSymbols(Expression *expr) {
+  std::ostringstream ss;
+  expr::PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(expr, &ss);
+  return ss.str();
+}
+}  // namespace memgraph::expr

src/expr/exceptions.hpp

@@ -32,4 +32,21 @@ class ExpressionRuntimeException : public utils::BasicException {
   using utils::BasicException::BasicException;
 };
 
+class RedeclareVariableError : public SemanticException {
+ public:
+  explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
+};
+
+class UnboundVariableError : public SemanticException {
+ public:
+  explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
+};
+
+class TypeMismatchError : public SemanticException {
+ public:
+  TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
+      : SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
+  }
+};
+
 }  // namespace memgraph::expr

src/expr/interpret/eval.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,22 +17,27 @@
 #include <map>
 #include <optional>
 #include <regex>
+#include <type_traits>
 #include <vector>
 
 #include "expr/ast.hpp"
 #include "expr/exceptions.hpp"
 #include "expr/interpret/frame.hpp"
 #include "expr/semantic/symbol_table.hpp"
+#include "functions/awesome_memgraph_functions.hpp"
 #include "utils/exceptions.hpp"
 
 namespace memgraph::expr {
 
+struct StorageTag {};
+struct QueryEngineTag {};
+
 template <typename TypedValue, typename EvaluationContext, typename DbAccessor, typename StorageView, typename LabelId,
-          typename PropertyValue, typename ConvFunction, typename Error>
+          typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
 class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
  public:
-  ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
-                      DbAccessor *dba, StorageView view)
+  ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
+                      StorageView view)
       : frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
 
   using ExpressionVisitor<TypedValue>::Visit;
@@ -42,12 +47,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   TypedValue Visit(NamedExpression &named_expression) override {
     const auto &symbol = symbol_table_->at(named_expression);
     auto value = named_expression.expression_->Accept(*this);
-    frame_->at(symbol) = value;
+    frame_->At(symbol) = value;
     return value;
   }
 
   TypedValue Visit(Identifier &ident) override {
-    return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
+    return TypedValue(frame_->At(symbol_table_->at(ident)), ctx_->memory);
   }
 
   // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
@@ -96,6 +101,28 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
 #undef BINARY_OPERATOR_VISITOR
 #undef UNARY_OPERATOR_VISITOR
 
+  void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
+    switch (shard_error) {
+      case Error::DELETED_OBJECT:
+        throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
+      case Error::NONEXISTENT_OBJECT:
+        throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
+      case Error::SERIALIZATION_ERROR:
+      case Error::VERTEX_HAS_EDGES:
+      case Error::PROPERTIES_DISABLED:
+      case Error::VERTEX_ALREADY_INSERTED:
+      case Error::OBJECT_NOT_FOUND:
+        throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
+      case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
+      case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
+      case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
+      case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
+      case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
+      case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
+        throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
+    }
+  }
+
   TypedValue Visit(AndOperator &op) override {
     auto value1 = op.expression1_->Accept(*this);
     if (value1.IsBool() && !value1.ValueBool()) {
@@ -377,6 +404,33 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     }
   }
 
+  template <typename VertexAccessor, typename TTag = Tag,
+            typename TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, bool>>
+  TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label, StorageTag /*tag*/) {
+    auto has_label = vertex.HasLabel(view_, GetLabel(label));
+    if (has_label.HasError() && has_label.GetError() == Error::NONEXISTENT_OBJECT) {
+      // This is a very nasty and temporary hack in order to make MERGE
+      // work. The old storage had the following logic when returning an
+      // `OLD` view: `return old ? old : new`. That means that if the
+      // `OLD` view didn't exist, it returned the NEW view. With this hack
+      // we simulate that behavior.
+      // TODO (mferencevic, teon.banek): Remove once MERGE is
+      // reimplemented.
+      has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
+    }
+    if (has_label.HasError()) {
+      HandleObjectAccessError(has_label.GetError().code, "labels");
+    }
+    return *has_label;
+  }
+
+  template <typename VertexAccessor, typename TTag = Tag,
+            typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
+  TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
+    auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
+    return vertex.HasLabel(label);
+  }
+
   TypedValue Visit(LabelsTest &labels_test) override {
     auto expression_result = labels_test.expression_->Accept(*this);
     switch (expression_result.type()) {
@@ -384,35 +438,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
         return TypedValue(ctx_->memory);
       case TypedValue::Type::Vertex: {
         const auto &vertex = expression_result.ValueVertex();
-        for (const auto &label : labels_test.labels_) {
-          auto has_label = vertex.HasLabel(view_, GetLabel(label));
-          if (has_label.HasError() && has_label.GetError() == Error::NONEXISTENT_OBJECT) {
-            // This is a very nasty and temporary hack in order to make MERGE
-            // work. The old storage had the following logic when returning an
-            // `OLD` view: `return old ? old : new`. That means that if the
-            // `OLD` view didn't exist, it returned the NEW view. With this hack
-            // we simulate that behavior.
-            // TODO (mferencevic, teon.banek): Remove once MERGE is
-            // reimplemented.
-            has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
-          }
-          if (has_label.HasError()) {
-            switch (has_label.GetError()) {
-              case Error::DELETED_OBJECT:
-                throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
-              case Error::NONEXISTENT_OBJECT:
-                throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
-              case Error::SERIALIZATION_ERROR:
-              case Error::VERTEX_HAS_EDGES:
-              case Error::PROPERTIES_DISABLED:
-                throw ExpressionRuntimeException("Unexpected error when accessing labels.");
-            }
-          }
-          if (!*has_label) {
-            return TypedValue(false, ctx_->memory);
-          }
+        if (std::ranges::all_of(labels_test.labels_, [&vertex, this](const auto label_test) {
+              return this->HasLabelImpl(vertex, label_test, Tag{});
+            })) {
+          return TypedValue(true, ctx_->memory);
         }
-        return TypedValue(true, ctx_->memory);
+        return TypedValue(false, ctx_->memory);
       }
       default:
         throw ExpressionRuntimeException("Only nodes have labels.");
@@ -439,7 +470,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   }
 
   TypedValue Visit(Aggregation &aggregation) override {
-    return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
+    return TypedValue(frame_->At(symbol_table_->at(aggregation)), ctx_->memory);
   }
 
   TypedValue Visit(Coalesce &coalesce) override {
@@ -460,7 +491,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   }
 
   TypedValue Visit(Function &function) override {
-    FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
+    functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
     // Stack allocate evaluated arguments when there's a small number of them.
     if (function.arguments_.size() <= 8) {
       TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
@@ -497,8 +528,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
     auto accumulator = reduce.initializer_->Accept(*this);
     for (const auto &element : list) {
-      frame_->at(accumulator_symbol) = accumulator;
-      frame_->at(element_symbol) = element;
+      frame_->At(accumulator_symbol) = accumulator;
+      frame_->At(element_symbol) = element;
       accumulator = reduce.expression_->Accept(*this);
     }
     return accumulator;
@@ -520,7 +551,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       if (element.IsNull()) {
         result.emplace_back();
       } else {
-        frame_->at(element_symbol) = element;
+        frame_->At(element_symbol) = element;
         result.emplace_back(extract.expression_->Accept(*this));
       }
     }
@@ -540,7 +571,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     bool has_null_elements = false;
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = all.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
@@ -577,7 +608,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     bool has_value = false;
     bool predicate_satisfied = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = single.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
@@ -614,7 +645,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &symbol = symbol_table_->at(*any.identifier_);
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = any.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
@@ -646,7 +677,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
     const auto &symbol = symbol_table_->at(*none.identifier_);
     bool has_value = false;
     for (const auto &element : list) {
-      frame_->at(symbol) = element;
+      frame_->At(symbol) = element;
       auto result = none.where_->expression_->Accept(*this);
       if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
         throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
@@ -695,7 +726,24 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
   }
 
  private:
-  template <class TRecordAccessor>
+  template <class TRecordAccessor, class TTag = Tag,
+            class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
+  TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
+    auto maybe_prop = record_accessor.GetProperty(prop.name);
+    // Handler non existent property
+    return conv_(maybe_prop, dba_);
+  }
+
+  template <class TRecordAccessor, class TTag = Tag,
+            class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
+  TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
+    auto maybe_prop = record_accessor.GetProperty(std::string(name));
+    // Handler non existent property
+    return conv_(maybe_prop, dba_);
+  }
+
+  template <class TRecordAccessor, class TTag = Tag,
+            class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
   TypedValue GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
     auto maybe_prop = record_accessor.GetProperty(view_, ctx_->properties[prop.ix]);
     if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
@@ -708,21 +756,13 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
     }
     if (maybe_prop.HasError()) {
-      switch (maybe_prop.GetError()) {
-        case Error::DELETED_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
-        case Error::NONEXISTENT_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
-        case Error::SERIALIZATION_ERROR:
-        case Error::VERTEX_HAS_EDGES:
-        case Error::PROPERTIES_DISABLED:
-          throw ExpressionRuntimeException("Unexpected error when getting a property.");
-      }
+      HandleObjectAccessError(maybe_prop.GetError().code, "property");
     }
-    return conv_(*maybe_prop);
+    return conv_(*maybe_prop, ctx_->memory);
   }
 
-  template <class TRecordAccessor>
+  template <class TRecordAccessor, class TTag = Tag,
+            class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
   TypedValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
     auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
     if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
@@ -735,29 +775,20 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
       maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
     }
     if (maybe_prop.HasError()) {
-      switch (maybe_prop.GetError()) {
-        case Error::DELETED_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
-        case Error::NONEXISTENT_OBJECT:
-          throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
-        case Error::SERIALIZATION_ERROR:
-        case Error::VERTEX_HAS_EDGES:
-        case Error::PROPERTIES_DISABLED:
-          throw ExpressionRuntimeException("Unexpected error when getting a property.");
-      }
+      HandleObjectAccessError(maybe_prop.GetError().code, "property");
     }
-    return conv_(*maybe_prop);
+    return conv_(*maybe_prop, ctx_->memory);
   }
 
   LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
 
-  Frame<TypedValue> *frame_;
+  Frame *frame_;
   const SymbolTable *symbol_table_;
   const EvaluationContext *ctx_;
   DbAccessor *dba_;
   // which switching approach should be used when evaluating
   StorageView view_;
-  ConvFunction conv_;
+  ConvFunctor conv_;
 };
 
 /// A helper function for evaluating an expression that's an int.

src/expr/interpret/frame.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,26 +20,43 @@
 
 namespace memgraph::expr {
 
-template <typename TypedValue>
 class Frame {
  public:
   /// Create a Frame of given size backed by a utils::NewDeleteResource()
-  explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
+  explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
 
-  Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
+  Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
 
   TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
   const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
 
-  TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
-  const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
+  TypedValue &At(const Symbol &symbol) { return elems_.at(symbol.position()); }
+  const TypedValue &At(const Symbol &symbol) const { return elems_.at(symbol.position()); }
 
-  auto &elems() { return elems_; }
+  uint64_t Id() const { return id_; }
+  void SetId(const uint64_t id) { id_ = id; }
+
+  const utils::pmr::vector<TypedValue> &Elems() const { return elems_; }
 
   utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
 
  private:
+  uint64_t id_{0U};
   utils::pmr::vector<TypedValue> elems_;
 };
 
+class FrameWithValidity final : public Frame {
+ public:
+  explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
+
+  FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
+
+  bool IsValid() const noexcept { return is_valid_; }
+  void MakeValid() noexcept { is_valid_ = true; }
+  void MakeInvalid() noexcept { is_valid_ = false; }
+
+ private:
+  bool is_valid_;
+};
+
 }  // namespace memgraph::expr

src/expr/typed_value.hpp

@@ -21,6 +21,7 @@
 #include <utility>
 #include <vector>
 
+#include "expr/typed_value_exception.hpp"
 #include "utils/algorithm.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/fnv.hpp"
@@ -32,16 +33,6 @@
 
 namespace memgraph::expr {
 
-/**
- * An exception raised by the TypedValue system. Typically when
- * trying to perform operations (such as addition) on TypedValues
- * of incompatible Types.
- */
-class TypedValueException : public utils::BasicException {
- public:
-  using utils::BasicException::BasicException;
-};
-
 // TODO: Neo4j does overflow checking. Should we also implement it?
 /**
  * Stores a query runtime value and its type.
@@ -116,13 +107,15 @@ class TypedValueT {
           return hash;
         }
         case TypedValueT::Type::Vertex:
+          return 34;
         case TypedValueT::Type::Edge:
-          return 0;
+          return 35;
         case TypedValueT::Type::Path: {
-          const auto &vertices = value.ValuePath().vertices();
-          const auto &edges = value.ValuePath().edges();
-          return utils::FnvCollection<decltype(vertices), TVertexAccessor>{}(vertices) ^
-                 utils::FnvCollection<decltype(edges), TEdgeAccessor>{}(edges);
+          // const auto &vertices = value.ValuePath().vertices();
+          // const auto &edges = value.ValuePath().edges();
+          // return utils::FnvCollection<decltype(vertices), TVertexAccessor>{}(vertices) ^
+          //        utils::FnvCollection<decltype(edges), TEdgeAccessor>{}(edges);
+          return 36;
         }
         case TypedValueT::Type::Date:
           return utils::DateHash{}(value.ValueDate());

src/expr/typed_value_exception.hpp

@@ -13,11 +13,14 @@
 
 #include "utils/exceptions.hpp"
 
-namespace memgraph::storage::v3::durability {
-
-/// Exception used to handle errors during recovery.
-class RecoveryFailure : public utils::BasicException {
+namespace memgraph::expr {
+/**
+ * An exception raised by the TypedValue system. Typically when
+ * trying to perform operations (such as addition) on TypedValues
+ * of incompatible Types.
+ */
+class TypedValueException : public utils::BasicException {
+ public:
   using utils::BasicException::BasicException;
 };
-
-}  // namespace memgraph::storage::v3::durability
+}  // namespace memgraph::expr

src/functions/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(mg-functions INTERFACE)

src/glue/v2/communication.cpp

@@ -15,9 +15,17 @@
 #include <string>
 #include <vector>
 
+#include "common/errors.hpp"
+#include "coordinator/shard_map.hpp"
+#include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
+#include "query/v2/requests.hpp"
 #include "storage/v3/edge_accessor.hpp"
+#include "storage/v3/id_types.hpp"
 #include "storage/v3/shard.hpp"
 #include "storage/v3/vertex_accessor.hpp"
+#include "storage/v3/view.hpp"
+#include "utils/exceptions.hpp"
 #include "utils/temporal.hpp"
 
 using memgraph::communication::bolt::Value;
@@ -63,127 +71,139 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
   }
 }
 
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
-                                                              const storage::v3::Shard &db, storage::v3::View view) {
-  return ToBoltVertex(vertex.impl_, db, view);
-}
-
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
-                                                          const storage::v3::Shard &db, storage::v3::View view) {
-  return ToBoltEdge(edge.impl_, db, view);
-}
-
-storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
-                                       storage::v3::View view) {
-  switch (value.type()) {
-    case query::v2::TypedValue::Type::Null:
-      return Value();
-    case query::v2::TypedValue::Type::Bool:
-      return Value(value.ValueBool());
-    case query::v2::TypedValue::Type::Int:
-      return Value(value.ValueInt());
-    case query::v2::TypedValue::Type::Double:
-      return Value(value.ValueDouble());
-    case query::v2::TypedValue::Type::String:
-      return Value(std::string(value.ValueString()));
-    case query::v2::TypedValue::Type::List: {
-      std::vector<Value> values;
-      values.reserve(value.ValueList().size());
-      for (const auto &v : value.ValueList()) {
-        auto maybe_value = ToBoltValue(v, db, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
-        values.emplace_back(std::move(*maybe_value));
-      }
-      return Value(std::move(values));
-    }
-    case query::v2::TypedValue::Type::Map: {
-      std::map<std::string, Value> map;
-      for (const auto &kv : value.ValueMap()) {
-        auto maybe_value = ToBoltValue(kv.second, db, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
-        map.emplace(kv.first, std::move(*maybe_value));
-      }
-      return Value(std::move(map));
-    }
-    case query::v2::TypedValue::Type::Vertex: {
-      auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
-      if (maybe_vertex.HasError()) return maybe_vertex.GetError();
-      return Value(std::move(*maybe_vertex));
-    }
-    case query::v2::TypedValue::Type::Edge: {
-      auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
-      if (maybe_edge.HasError()) return maybe_edge.GetError();
-      return Value(std::move(*maybe_edge));
-    }
-    case query::v2::TypedValue::Type::Path: {
-      auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
-      if (maybe_path.HasError()) return maybe_path.GetError();
-      return Value(std::move(*maybe_path));
-    }
-    case query::v2::TypedValue::Type::Date:
-      return Value(value.ValueDate());
-    case query::v2::TypedValue::Type::LocalTime:
-      return Value(value.ValueLocalTime());
-    case query::v2::TypedValue::Type::LocalDateTime:
-      return Value(value.ValueLocalDateTime());
-    case query::v2::TypedValue::Type::Duration:
-      return Value(value.ValueDuration());
-  }
-}
-
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
-                                                              const storage::v3::Shard &db, storage::v3::View view) {
-  // TODO(jbajic) Fix bolt communication
+communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
+                                         const query::v2::RequestRouterInterface *request_router,
+                                         storage::v3::View /*view*/) {
   auto id = communication::bolt::Id::FromUint(0);
-  auto maybe_labels = vertex.Labels(view);
-  if (maybe_labels.HasError()) return maybe_labels.GetError();
-  std::vector<std::string> labels;
-  labels.reserve(maybe_labels->size());
-  for (const auto &label : *maybe_labels) {
-    labels.push_back(db.LabelToName(label));
+
+  auto labels = vertex.Labels();
+  std::vector<std::string> new_labels;
+  new_labels.reserve(labels.size());
+  for (const auto &label : labels) {
+    new_labels.push_back(request_router->LabelToName(label.id));
   }
-  auto maybe_properties = vertex.Properties(view);
-  if (maybe_properties.HasError()) return maybe_properties.GetError();
-  std::map<std::string, Value> properties;
-  for (const auto &prop : *maybe_properties) {
-    properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
+
+  auto properties = vertex.Properties();
+  std::map<std::string, Value> new_properties;
+  for (const auto &[prop, property_value] : properties) {
+    new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
   }
-  return communication::bolt::Vertex{id, labels, properties};
+  return communication::bolt::Vertex{id, new_labels, new_properties};
 }
 
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
-                                                          const storage::v3::Shard &db, storage::v3::View view) {
+communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
+                                     const query::v2::RequestRouterInterface *request_router,
+                                     storage::v3::View /*view*/) {
   // TODO(jbajic) Fix bolt communication
   auto id = communication::bolt::Id::FromUint(0);
   auto from = communication::bolt::Id::FromUint(0);
   auto to = communication::bolt::Id::FromUint(0);
-  const auto &type = db.EdgeTypeToName(edge.EdgeType());
-  auto maybe_properties = edge.Properties(view);
-  if (maybe_properties.HasError()) return maybe_properties.GetError();
-  std::map<std::string, Value> properties;
-  for (const auto &prop : *maybe_properties) {
-    properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
+  const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
+
+  auto properties = edge.Properties();
+  std::map<std::string, Value> new_properties;
+  for (const auto &[prop, property_value] : properties) {
+    new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
   }
-  return communication::bolt::Edge{id, from, to, type, properties};
+  return communication::bolt::Edge{id, from, to, type, new_properties};
 }
 
-storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
-                                                          storage::v3::View view) {
-  std::vector<communication::bolt::Vertex> vertices;
-  vertices.reserve(path.vertices().size());
-  for (const auto &v : path.vertices()) {
-    auto maybe_vertex = ToBoltVertex(v, db, view);
-    if (maybe_vertex.HasError()) return maybe_vertex.GetError();
-    vertices.emplace_back(std::move(*maybe_vertex));
+communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
+                                     const query::v2::RequestRouterInterface * /*request_router*/,
+                                     storage::v3::View /*view*/) {
+  // TODO(jbajic) Fix bolt communication
+  MG_ASSERT(false, "Path is unimplemented!");
+  return {};
+}
+
+Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
+                  storage::v3::View view) {
+  switch (value.type()) {
+    case query::v2::TypedValue::Type::Null:
+      return {};
+    case query::v2::TypedValue::Type::Bool:
+      return {value.ValueBool()};
+    case query::v2::TypedValue::Type::Int:
+      return {value.ValueInt()};
+    case query::v2::TypedValue::Type::Double:
+      return {value.ValueDouble()};
+    case query::v2::TypedValue::Type::String:
+      return {std::string(value.ValueString())};
+    case query::v2::TypedValue::Type::List: {
+      std::vector<Value> values;
+      values.reserve(value.ValueList().size());
+      for (const auto &v : value.ValueList()) {
+        auto value = ToBoltValue(v, request_router, view);
+        values.emplace_back(std::move(value));
+      }
+      return {std::move(values)};
+    }
+    case query::v2::TypedValue::Type::Map: {
+      std::map<std::string, Value> map;
+      for (const auto &kv : value.ValueMap()) {
+        auto value = ToBoltValue(kv.second, request_router, view);
+        map.emplace(kv.first, std::move(value));
+      }
+      return {std::move(map)};
+    }
+    case query::v2::TypedValue::Type::Vertex: {
+      auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
+      return {std::move(vertex)};
+    }
+    case query::v2::TypedValue::Type::Edge: {
+      auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
+      return {std::move(edge)};
+    }
+    case query::v2::TypedValue::Type::Path: {
+      auto path = ToBoltPath(value.ValuePath(), request_router, view);
+      return {std::move(path)};
+    }
+    case query::v2::TypedValue::Type::Date:
+      return {value.ValueDate()};
+    case query::v2::TypedValue::Type::LocalTime:
+      return {value.ValueLocalTime()};
+    case query::v2::TypedValue::Type::LocalDateTime:
+      return {value.ValueLocalDateTime()};
+    case query::v2::TypedValue::Type::Duration:
+      return {value.ValueDuration()};
   }
-  std::vector<communication::bolt::Edge> edges;
-  edges.reserve(path.edges().size());
-  for (const auto &e : path.edges()) {
-    auto maybe_edge = ToBoltEdge(e, db, view);
-    if (maybe_edge.HasError()) return maybe_edge.GetError();
-    edges.emplace_back(std::move(*maybe_edge));
+}
+
+Value ToBoltValue(msgs::Value value) {
+  switch (value.type) {
+    case msgs::Value::Type::Null:
+      return {};
+    case msgs::Value::Type::Bool:
+      return {value.bool_v};
+    case msgs::Value::Type::Int64:
+      return {value.int_v};
+    case msgs::Value::Type::Double:
+      return {value.double_v};
+    case msgs::Value::Type::String:
+      return {std::string(value.string_v)};
+    case msgs::Value::Type::List: {
+      std::vector<Value> values;
+      values.reserve(value.list_v.size());
+      for (const auto &v : value.list_v) {
+        auto maybe_value = ToBoltValue(v);
+        values.emplace_back(std::move(maybe_value));
+      }
+      return Value{std::move(values)};
+    }
+    case msgs::Value::Type::Map: {
+      std::map<std::string, Value> map;
+      for (const auto &kv : value.map_v) {
+        auto maybe_value = ToBoltValue(kv.second);
+        map.emplace(kv.first, std::move(maybe_value));
+      }
+      return Value{std::move(map)};
+    }
+    case msgs::Value::Type::Vertex:
+    case msgs::Value::Type::Edge: {
+      throw utils::BasicException("Vertex and Edge not supported!");
+    }
+      // TODO Value to Date types not supported
   }
-  return communication::bolt::Path(vertices, edges);
 }
 
 storage::v3::PropertyValue ToPropertyValue(const Value &value) {

src/glue/v2/communication.hpp

@@ -13,10 +13,14 @@
 #pragma once
 
 #include "communication/bolt/v1/value.hpp"
+#include "coordinator/shard_map.hpp"
 #include "query/v2/bindings/typed_value.hpp"
+#include "query/v2/request_router.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
+#include "storage/v3/shard.hpp"
 #include "storage/v3/view.hpp"
+#include "utils/result.hpp"
 
 namespace memgraph::storage::v3 {
 class EdgeAccessor;
@@ -28,36 +32,37 @@ namespace memgraph::glue::v2 {
 
 /// @param storage::v3::VertexAccessor for converting to
 ///        communication::bolt::Vertex.
-/// @param storage::v3::Shard for getting label and property names.
+/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
 /// @param storage::v3::View for deciding which vertex attributes are visible.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
-                                                              const storage::v3::Shard &db, storage::v3::View view);
+communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
+                                         const query::v2::RequestRouterInterface *request_router,
+                                         storage::v3::View view);
 
 /// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
-/// @param storage::v3::Shard for getting edge type and property names.
+/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
 /// @param storage::v3::View for deciding which edge attributes are visible.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
-                                                          const storage::v3::Shard &db, storage::v3::View view);
+communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
+                                     const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
 
 /// @param query::v2::Path for converting to communication::bolt::Path.
-/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
+/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
 /// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
-                                                          storage::v3::View view);
+communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
+                                     const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
 
 /// @param query::v2::TypedValue for converting to communication::bolt::Value.
-/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
+/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
 /// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
 ///
 /// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
-                                                            const storage::v3::Shard &db, storage::v3::View view);
+communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
+                                       const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
 
 query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
 
@@ -65,4 +70,9 @@ communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
 
 storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
 
+communication::bolt::Value ToBoltValue(msgs::Value value);
+
+communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
+                                       storage::v3::View view);
+
 }  // namespace memgraph::glue::v2

src/io/address.hpp

@@ -15,11 +15,38 @@
 
 #include <fmt/format.h>
 #include <boost/asio/ip/tcp.hpp>
+#include <boost/functional/hash.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_generators.hpp>
 #include <boost/uuid/uuid_io.hpp>
 
+#include "utils/logging.hpp"
+
 namespace memgraph::io {
+
+struct PartialAddress {
+  boost::asio::ip::address ip;
+  uint16_t port;
+
+  friend bool operator==(const PartialAddress &lhs, const PartialAddress &rhs) = default;
+
+  /// unique_id is most dominant for ordering, then ip, then port
+  friend bool operator<(const PartialAddress &lhs, const PartialAddress &rhs) {
+    if (lhs.ip != rhs.ip) {
+      return lhs.ip < rhs.ip;
+    }
+
+    return lhs.port < rhs.port;
+  }
+
+  std::string ToString() const { return fmt::format("PartialAddress {{ ip: {}, port: {} }}", ip.to_string(), port); }
+
+  friend std::ostream &operator<<(std::ostream &in, const PartialAddress &partial_address) {
+    in << partial_address.ToString();
+    return in;
+  }
+};
+
 struct Address {
   // It's important for all participants to have a
   // unique identifier - IP and port alone are not
@@ -33,18 +60,39 @@ struct Address {
   uint16_t last_known_port;
 
   static Address TestAddress(uint16_t port) {
+    MG_ASSERT(port <= 255);
+
     return Address{
-        .unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
+        .unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
         .last_known_port = port,
     };
   }
 
+  // NB: don't use this in test code because it is non-deterministic
   static Address UniqueLocalAddress() {
     return Address{
         .unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
     };
   }
 
+  /// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
+  Address ForkLocalCoordinator() {
+    return Address{
+        .unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+        .last_known_ip = last_known_ip,
+        .last_known_port = last_known_port,
+    };
+  }
+
+  /// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
+  Address ForkLocalShardManager() {
+    return Address{
+        .unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+        .last_known_ip = last_known_ip,
+        .last_known_port = last_known_port,
+    };
+  }
+
   /// Returns a new ID with the same IP and port but a unique UUID.
   Address ForkUniqueAddress() {
     return Address{
@@ -54,6 +102,13 @@ struct Address {
     };
   }
 
+  PartialAddress ToPartialAddress() const {
+    return PartialAddress{
+        .ip = last_known_ip,
+        .port = last_known_port,
+    };
+  }
+
   friend bool operator==(const Address &lhs, const Address &rhs) = default;
 
   /// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
@@ -73,5 +128,36 @@ struct Address {
     return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
                        boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
   }
+
+  friend std::ostream &operator<<(std::ostream &in, const Address &address) {
+    in << address.ToString();
+    return in;
+  }
 };
+
 };  // namespace memgraph::io
+
+namespace std {
+template <>
+struct hash<memgraph::io::PartialAddress> {
+  size_t operator()(const memgraph::io::PartialAddress &pa) const {
+    using boost::hash_combine;
+    using boost::hash_value;
+
+    // Start with a hash value of 0    .
+    std::size_t seed = 0;
+
+    if (pa.ip.is_v4()) {
+      auto h = std::hash<boost::asio::ip::address_v4>()(pa.ip.to_v4());
+      hash_combine(seed, h);
+    } else {
+      auto h = std::hash<boost::asio::ip::address_v6>()(pa.ip.to_v6());
+      hash_combine(seed, h);
+    }
+    hash_combine(seed, hash_value(pa.port));
+
+    // Return the result.
+    return seed;
+  }
+};
+}  // namespace std

src/io/future.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -35,10 +35,13 @@ class Shared {
   std::optional<T> item_;
   bool consumed_ = false;
   bool waiting_ = false;
-  std::function<bool()> simulator_notifier_ = nullptr;
+  bool filled_ = false;
+  std::function<bool()> wait_notifier_ = nullptr;
+  std::function<void()> fill_notifier_ = nullptr;
 
  public:
-  explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
+  explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
+      : wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
   Shared() = default;
   Shared(Shared &&) = delete;
   Shared &operator=(Shared &&) = delete;
@@ -64,8 +67,7 @@ class Shared {
     waiting_ = true;
 
     while (!item_) {
-      bool simulator_progressed = false;
-      if (simulator_notifier_) [[unlikely]] {
+      if (wait_notifier_) [[unlikely]] {
         // We can't hold our own lock while notifying
         // the simulator because notifying the simulator
         // involves acquiring the simulator's mutex
@@ -77,7 +79,7 @@ class Shared {
         // so we have to get out of its way to avoid
         // a cyclical deadlock.
         lock.unlock();
-        simulator_progressed = std::invoke(simulator_notifier_);
+        std::invoke(wait_notifier_);
         lock.lock();
         if (item_) {
           // item may have been filled while we
@@ -85,8 +87,7 @@ class Shared {
           // the simulator of our waiting_ status.
           break;
         }
-      }
-      if (!simulator_progressed) [[likely]] {
+      } else {
         cv_.wait(lock);
       }
       MG_ASSERT(!consumed_, "Future consumed twice!");
@@ -99,7 +100,7 @@ class Shared {
 
   bool IsReady() const {
     std::unique_lock<std::mutex> lock(mu_);
-    return item_;
+    return item_.has_value();
   }
 
   std::optional<T> TryGet() {
@@ -117,11 +118,19 @@ class Shared {
       std::unique_lock<std::mutex> lock(mu_);
 
       MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
-      MG_ASSERT(!item_, "Promise filled twice!");
+      MG_ASSERT(!filled_, "Promise filled twice!");
 
-      item_ = item;
+      item_ = std::move(item);
+      filled_ = true;
     }  // lock released before condition variable notification
 
+    if (fill_notifier_) {
+      spdlog::trace("calling fill notifier");
+      std::invoke(fill_notifier_);
+    } else {
+      spdlog::trace("not calling fill notifier");
+    }
+
     cv_.notify_all();
   }
 
@@ -151,7 +160,9 @@ class Future {
   Future &operator=(Future &&old) noexcept {
     MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
     shared_ = std::move(old.shared_);
+    consumed_or_moved_ = old.consumed_or_moved_;
     old.consumed_or_moved_ = true;
+    return *this;
   }
 
   Future(const Future &) = delete;
@@ -198,7 +209,7 @@ class Future {
 template <typename T>
 class Promise {
   std::shared_ptr<details::Shared<T>> shared_;
-  bool filled_or_moved_ = false;
+  bool filled_or_moved_{false};
 
  public:
   explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
@@ -218,12 +229,13 @@ class Promise {
   Promise(const Promise &) = delete;
   Promise &operator=(const Promise &) = delete;
 
+  // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Branch)
   ~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
 
   // Fill the expected item into the Future.
   void Fill(T item) {
     MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
-    shared_->Fill(item);
+    shared_->Fill(std::move(item));
     filled_or_moved_ = true;
   }
 
@@ -250,8 +262,9 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
 }
 
 template <typename T>
-std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
-  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
+std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
+                                                                    std::function<void()> fill_notifier) {
+  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
 
   Future<T> future = Future<T>(shared);
   Promise<T> promise = Promise<T>(shared);

src/io/local_transport/local_system.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,7 +25,7 @@ class LocalSystem {
 
  public:
   Io<LocalTransport> Register(Address address) {
-    LocalTransport local_transport(local_transport_handle_, address);
+    LocalTransport local_transport(local_transport_handle_);
     return Io{local_transport, address};
   }
 

src/io/local_transport/local_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,33 +25,26 @@ namespace memgraph::io::local_transport {
 
 class LocalTransport {
   std::shared_ptr<LocalTransportHandle> local_transport_handle_;
-  const Address address_;
 
  public:
-  LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle, Address address)
-      : local_transport_handle_(std::move(local_transport_handle)), address_(address) {}
+  explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
+      : local_transport_handle_(std::move(local_transport_handle)) {}
 
-  template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> Request(Address to_address, RequestId request_id, RequestT request, Duration timeout) {
-    auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
-
-    Address from_address = address_;
-
-    local_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
-                                           std::move(promise));
-
-    return std::move(future);
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
+                                    std::function<void()> fill_notifier, Duration timeout) {
+    return local_transport_handle_->template SubmitRequest<ResponseT, RequestT>(
+        to_address, from_address, std::move(request), timeout, fill_notifier);
   }
 
   template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Duration timeout) {
-    Address from_address = address_;
-    return local_transport_handle_->template Receive<Ms...>(timeout);
+  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
+    return local_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
-    return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
+    return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
   }
 
   Time Now() const { return local_transport_handle_->Now(); }
@@ -63,5 +56,7 @@ class LocalTransport {
     std::random_device rng;
     return distrib(rng);
   }
+
+  LatencyHistogramSummaries ResponseLatencies() { return local_transport_handle_->ResponseLatencies(); }
 };
 };  // namespace memgraph::io::local_transport

src/io/local_transport/local_transport_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,6 +19,7 @@
 
 #include "io/errors.hpp"
 #include "io/message_conversion.hpp"
+#include "io/message_histogram_collector.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
 
@@ -28,6 +29,8 @@ class LocalTransportHandle {
   mutable std::mutex mu_{};
   mutable std::condition_variable cv_;
   bool should_shut_down_ = false;
+  MessageHistogramCollector histograms_;
+  RequestId request_id_counter_ = 0;
 
   // the responses to requests that are being waited on
   std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
@@ -36,6 +39,13 @@ class LocalTransportHandle {
   std::vector<OpaqueMessage> can_receive_;
 
  public:
+  ~LocalTransportHandle() {
+    for (auto &&[pk, promise] : promises_) {
+      std::move(promise.promise).TimeOut();
+    }
+    promises_.clear();
+  }
+
   void ShutDown() {
     std::unique_lock<std::mutex> lock(mu_);
     should_shut_down_ = true;
@@ -47,17 +57,24 @@ class LocalTransportHandle {
     return should_shut_down_;
   }
 
+  LatencyHistogramSummaries ResponseLatencies() {
+    std::unique_lock<std::mutex> lock(mu_);
+    return histograms_.ResponseLatencies();
+  }
+
   static Time Now() {
     auto nano_time = std::chrono::system_clock::now();
     return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
   }
 
   template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Duration timeout) {
+  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
     std::unique_lock lock(mu_);
 
     Time before = Now();
 
+    spdlog::info("can_receive_ size: {}", can_receive_.size());
+
     while (can_receive_.empty()) {
       Time now = Now();
 
@@ -87,25 +104,33 @@ class LocalTransportHandle {
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
-    std::any message_any(std::forward<M>(message));
-    OpaqueMessage opaque_message{
-        .from_address = from_address, .request_id = request_id, .message = std::move(message_any)};
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
+    auto type_info = TypeInfoFor(message);
 
-    PromiseKey promise_key{.requester_address = to_address,
-                           .request_id = opaque_message.request_id,
-                           .replier_address = opaque_message.from_address};
+    std::any message_any(std::move(message));
+    OpaqueMessage opaque_message{.to_address = to_address,
+                                 .from_address = from_address,
+                                 .request_id = request_id,
+                                 .message = std::move(message_any),
+                                 .type_info = type_info};
+
+    PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
 
     {
       std::unique_lock<std::mutex> lock(mu_);
 
       if (promises_.contains(promise_key)) {
+        spdlog::info("using message to fill promise");
         // complete waiting promise if it's there
         DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
         promises_.erase(promise_key);
 
-        dop.promise.Fill(std::move(opaque_message));
+        Duration response_latency = Now() - dop.requested_at;
+
+        dop.promise.Fill(std::move(opaque_message), response_latency);
+        histograms_.Measure(type_info, response_latency);
       } else {
+        spdlog::info("placing message in can_receive_");
         can_receive_.emplace_back(std::move(opaque_message));
       }
     }  // lock dropped
@@ -113,26 +138,39 @@ class LocalTransportHandle {
     cv_.notify_all();
   }
 
-  template <Message RequestT, Message ResponseT>
-  void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
-                     Duration timeout, ResponsePromise<ResponseT> promise) {
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
+                                          Duration timeout, std::function<void()> fill_notifier) {
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
+        // set null notifier for when the Future::Wait is called
+        nullptr,
+        // set notifier for when Promise::Fill is called
+        std::move(fill_notifier));
+
     const bool port_matches = to_address.last_known_port == from_address.last_known_port;
     const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
 
     MG_ASSERT(port_matches && ip_matches);
-    const Time deadline = Now() + timeout;
 
+    const auto now = Now();
+    const Time deadline = now + timeout;
+
+    RequestId request_id = 0;
     {
       std::unique_lock<std::mutex> lock(mu_);
 
-      PromiseKey promise_key{
-          .requester_address = from_address, .request_id = request_id, .replier_address = to_address};
+      request_id = ++request_id_counter_;
+      PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
       OpaquePromise opaque_promise(std::move(promise).ToUnique());
-      DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
+      DeadlineAndOpaquePromise dop{.requested_at = now, .deadline = deadline, .promise = std::move(opaque_promise)};
+
+      MG_ASSERT(!promises_.contains(promise_key));
       promises_.emplace(std::move(promise_key), std::move(dop));
     }  // lock dropped
 
-    Send(to_address, from_address, request_id, std::forward<RequestT>(request));
+    Send<RequestT>(to_address, from_address, request_id, std::move(request));
+
+    return std::move(future);
   }
 };
 

src/io/message_conversion.hpp

@@ -11,20 +11,17 @@
 
 #pragma once
 
+#include <boost/core/demangle.hpp>
+
+#include "io/time.hpp"
 #include "io/transport.hpp"
+#include "utils/type_info_ref.hpp"
 
 namespace memgraph::io {
 
-using memgraph::io::Duration;
-using memgraph::io::Message;
-using memgraph::io::Time;
-
 struct PromiseKey {
   Address requester_address;
   uint64_t request_id;
-  // TODO(tyler) possibly remove replier_address from promise key
-  // once we want to support DSR.
-  Address replier_address;
 
  public:
   friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
@@ -32,11 +29,7 @@ struct PromiseKey {
       return lhs.requester_address < rhs.requester_address;
     }
 
-    if (lhs.request_id != rhs.request_id) {
-      return lhs.request_id < rhs.request_id;
-    }
-
-    return lhs.replier_address < rhs.replier_address;
+    return lhs.request_id < rhs.request_id;
   }
 };
 
@@ -45,6 +38,8 @@ struct OpaqueMessage {
   Address from_address;
   uint64_t request_id;
   std::any message;
+  utils::TypeInfoRef type_info;
+  Time deliverable_at;
 
   /// Recursively tries to match a specific type from the outer
   /// variant's parameter pack against the type of the std::any,
@@ -92,6 +87,10 @@ struct OpaqueMessage {
       };
     }
 
+    std::string demangled_name = "\"" + boost::core::demangle(message.type().name()) + "\"";
+    spdlog::error("failed to cast message of type {} to expected request type (probably in Receive argument types)",
+                  demangled_name);
+
     return std::nullopt;
   }
 };
@@ -100,7 +99,7 @@ class OpaquePromiseTraitBase {
  public:
   virtual const std::type_info *TypeInfo() const = 0;
   virtual bool IsAwaited(void *ptr) const = 0;
-  virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
+  virtual void Fill(void *ptr, OpaqueMessage &&, Duration) const = 0;
   virtual void TimeOut(void *ptr) const = 0;
 
   virtual ~OpaquePromiseTraitBase() = default;
@@ -118,12 +117,13 @@ class OpaquePromiseTrait : public OpaquePromiseTraitBase {
 
   bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
 
-  void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
+  void Fill(void *ptr, OpaqueMessage &&opaque_message, Duration response_latency) const override {
     T message = std::any_cast<T>(std::move(opaque_message.message));
     auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
                                                  .request_id = opaque_message.request_id,
                                                  .to_address = opaque_message.to_address,
-                                                 .from_address = opaque_message.from_address};
+                                                 .from_address = opaque_message.from_address,
+                                                 .response_latency = response_latency};
     auto promise = static_cast<ResponsePromise<T> *>(ptr);
     auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
     unique_promise->Fill(std::move(response_envelope));
@@ -187,9 +187,9 @@ class OpaquePromise {
     ptr_ = nullptr;
   }
 
-  void Fill(OpaqueMessage &&opaque_message) {
+  void Fill(OpaqueMessage &&opaque_message, Duration response_latency) {
     MG_ASSERT(ptr_ != nullptr);
-    trait_->Fill(ptr_, std::move(opaque_message));
+    trait_->Fill(ptr_, std::move(opaque_message), response_latency);
     ptr_ = nullptr;
   }
 
@@ -199,8 +199,40 @@ class OpaquePromise {
 };
 
 struct DeadlineAndOpaquePromise {
+  Time requested_at;
   Time deadline;
   OpaquePromise promise;
 };
 
+template <class From>
+std::type_info const &TypeInfoForVariant(From const &from) {
+  return std::visit([](auto &&x) -> decltype(auto) { return typeid(x); }, from);
+}
+
+template <class T>
+utils::TypeInfoRef TypeInfoFor(const T & /* t */) {
+  return typeid(T);
+}
+
+template <typename From, typename Return, typename Head, typename... Rest>
+std::optional<Return> ConvertVariantInner(From &&a) {
+  if (typeid(Head) == TypeInfoForVariant(a)) {
+    Head concrete = std::get<Head>(std::forward<From>(a));
+    return concrete;
+  }
+
+  if constexpr (sizeof...(Rest) > 0) {
+    return ConvertVariantInner<From, Return, Rest...>(std::forward<From>(a));
+  } else {
+    return std::nullopt;
+  }
+}
+
+/// This function converts a variant to another variant holding a subset OR superset of
+/// possible types.
+template <class From, class... Ms>
+requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> ConvertVariant(From &&from) {
+  return ConvertVariantInner<From, std::variant<Ms...>, Ms...>(std::forward<From>(from));
+}
+
 }  // namespace memgraph::io

src/io/message_histogram_collector.hpp

@@ -0,0 +1,132 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <chrono>
+#include <cmath>
+#include <compare>
+#include <unordered_map>
+
+#include <boost/core/demangle.hpp>
+
+#include "io/time.hpp"
+#include "utils/histogram.hpp"
+#include "utils/logging.hpp"
+#include "utils/print_helpers.hpp"
+#include "utils/type_info_ref.hpp"
+
+namespace memgraph::io {
+
+struct LatencyHistogramSummary {
+  uint64_t count;
+  Duration p0;
+  Duration p50;
+  Duration p75;
+  Duration p90;
+  Duration p95;
+  Duration p975;
+  Duration p99;
+  Duration p999;
+  Duration p9999;
+  Duration p100;
+  Duration sum;
+
+  friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
+
+  friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
+    in << "{ \"count\": " << histo.count;
+    in << ", \"p0\": " << histo.p0.count();
+    in << ", \"p50\": " << histo.p50.count();
+    in << ", \"p75\": " << histo.p75.count();
+    in << ", \"p90\": " << histo.p90.count();
+    in << ", \"p95\": " << histo.p95.count();
+    in << ", \"p975\": " << histo.p975.count();
+    in << ", \"p99\": " << histo.p99.count();
+    in << ", \"p999\": " << histo.p999.count();
+    in << ", \"p9999\": " << histo.p9999.count();
+    in << ", \"p100\": " << histo.p100.count();
+    in << ", \"sum\": " << histo.sum.count();
+    in << " }";
+
+    return in;
+  }
+};
+
+struct LatencyHistogramSummaries {
+  std::unordered_map<std::string, LatencyHistogramSummary> latencies;
+
+  std::string SummaryTable() {
+    std::string output;
+
+    const auto row = [&output](const auto &c1, const auto &c2, const auto &c3, const auto &c4, const auto &c5,
+                               const auto &c6, const auto &c7) {
+      output +=
+          fmt::format("{: >50} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8}\n", c1, c2, c3, c4, c5, c6, c7);
+    };
+    row("name", "count", "min (μs)", "med (μs)", "p99 (μs)", "max (μs)", "sum (ms)");
+
+    for (const auto &[name, histo] : latencies) {
+      row(name, histo.count, histo.p0.count(), histo.p50.count(), histo.p99.count(), histo.p100.count(),
+          histo.sum.count() / 1000);
+    }
+
+    output += "\n";
+    return output;
+  }
+
+  friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
+
+  friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
+    using memgraph::utils::print_helpers::operator<<;
+    in << histo.latencies;
+    return in;
+  }
+};
+
+class MessageHistogramCollector {
+  std::unordered_map<utils::TypeInfoRef, utils::Histogram, utils::TypeInfoHasher, utils::TypeInfoEqualTo> histograms_;
+
+ public:
+  void Measure(const std::type_info &type_info, const Duration &duration) {
+    auto &histo = histograms_[type_info];
+    histo.Measure(duration.count());
+  }
+
+  LatencyHistogramSummaries ResponseLatencies() {
+    std::unordered_map<std::string, LatencyHistogramSummary> ret{};
+
+    for (const auto &[type_id, histo] : histograms_) {
+      std::string demangled_name = "\"" + boost::core::demangle(type_id.get().name()) + "\"";
+
+      LatencyHistogramSummary latency_histogram_summary{
+          .count = histo.Count(),
+          .p0 = Duration(static_cast<int64_t>(histo.Percentile(0.0))),
+          .p50 = Duration(static_cast<int64_t>(histo.Percentile(50.0))),
+          .p75 = Duration(static_cast<int64_t>(histo.Percentile(75.0))),
+          .p90 = Duration(static_cast<int64_t>(histo.Percentile(90.0))),
+          .p95 = Duration(static_cast<int64_t>(histo.Percentile(95.0))),
+          .p975 = Duration(static_cast<int64_t>(histo.Percentile(97.5))),
+          .p99 = Duration(static_cast<int64_t>(histo.Percentile(99.0))),
+          .p999 = Duration(static_cast<int64_t>(histo.Percentile(99.9))),
+          .p9999 = Duration(static_cast<int64_t>(histo.Percentile(99.99))),
+          .p100 = Duration(static_cast<int64_t>(histo.Percentile(100.0))),
+          .sum = Duration(histo.Sum()),
+      };
+
+      ret.emplace(demangled_name, latency_histogram_summary);
+    }
+
+    return LatencyHistogramSummaries{.latencies = ret};
+  }
+};
+
+}  // namespace memgraph::io

src/io/messages.hpp

@@ -0,0 +1,46 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <variant>
+
+#include <coordinator/coordinator.hpp>
+#include <io/rsm/raft.hpp>
+#include "query/v2/requests.hpp"
+#include "utils/concepts.hpp"
+
+namespace memgraph::io::messages {
+
+using memgraph::coordinator::CoordinatorReadRequests;
+using memgraph::coordinator::CoordinatorWriteRequests;
+using memgraph::coordinator::CoordinatorWriteResponses;
+using StorageReadRequest = msgs::ReadRequests;
+using StorageWriteRequest = msgs::WriteRequests;
+
+using memgraph::io::rsm::AppendRequest;
+using memgraph::io::rsm::AppendResponse;
+using memgraph::io::rsm::ReadRequest;
+using memgraph::io::rsm::VoteRequest;
+using memgraph::io::rsm::VoteResponse;
+using memgraph::io::rsm::WriteRequest;
+using memgraph::io::rsm::WriteResponse;
+
+using CoordinatorMessages =
+    std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
+                 WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>;
+
+using ShardMessages = std::variant<ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, AppendResponse,
+                                   WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>;
+
+using ShardManagerMessages = std::variant<WriteResponse<CoordinatorWriteResponses>>;
+
+}  // namespace memgraph::io::messages

src/io/notifier.hpp

@@ -0,0 +1,93 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <optional>
+#include <vector>
+
+namespace memgraph::io {
+
+class ReadinessToken {
+  size_t id_;
+
+ public:
+  explicit ReadinessToken(size_t id) : id_(id) {}
+  size_t GetId() const { return id_; }
+};
+
+class Inner {
+  std::condition_variable cv_;
+  std::mutex mu_;
+  std::vector<ReadinessToken> ready_;
+  std::optional<std::function<bool()>> tick_simulator_;
+
+ public:
+  void Notify(ReadinessToken readiness_token) {
+    {
+      std::unique_lock<std::mutex> lock(mu_);
+      ready_.emplace_back(readiness_token);
+    }  // mutex dropped
+
+    cv_.notify_all();
+  }
+
+  ReadinessToken Await() {
+    std::unique_lock<std::mutex> lock(mu_);
+
+    while (ready_.empty()) {
+      if (tick_simulator_) [[unlikely]] {
+        // This avoids a deadlock in a similar way that
+        // Future::Wait will release its mutex while
+        // interacting with the simulator, due to
+        // the fact that the simulator may cause
+        // notifications that we are interested in.
+        lock.unlock();
+        std::invoke(tick_simulator_.value());
+        lock.lock();
+      } else {
+        cv_.wait(lock);
+      }
+    }
+
+    ReadinessToken ret = ready_.back();
+    ready_.pop_back();
+    return ret;
+  }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
+    std::unique_lock<std::mutex> lock(mu_);
+    tick_simulator_ = tick_simulator;
+  }
+};
+
+class Notifier {
+  std::shared_ptr<Inner> inner_;
+
+ public:
+  Notifier() : inner_(std::make_shared<Inner>()) {}
+  Notifier(const Notifier &) = default;
+  Notifier &operator=(const Notifier &) = default;
+  Notifier(Notifier &&old) = default;
+  Notifier &operator=(Notifier &&old) = default;
+  ~Notifier() = default;
+
+  void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
+
+  ReadinessToken Await() const { return inner_->Await(); }
+
+  void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
+};
+
+}  // namespace memgraph::io

src/io/rsm/raft.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,9 +19,12 @@
 #include <map>
 #include <set>
 #include <thread>
-#include <unordered_map>
+#include <type_traits>
 #include <vector>
 
+#include <boost/core/demangle.hpp>
+
+#include "io/message_conversion.hpp"
 #include "io/simulator/simulator.hpp"
 #include "io/transport.hpp"
 #include "utils/concepts.hpp"
@@ -53,7 +56,6 @@ static constexpr size_t kMaximumAppendBatchSize = 1024;
 using Term = uint64_t;
 using LogIndex = uint64_t;
 using LogSize = uint64_t;
-using RequestId = uint64_t;
 
 template <typename WriteOperation>
 struct WriteRequest {
@@ -89,6 +91,46 @@ struct ReadResponse {
   std::optional<Address> retry_leader;
 };
 
+template <class... ReadReturn>
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
+  return TypeInfoForVariant(response.read_return);
+}
+
+template <class ReadReturn>
+utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
+  return typeid(ReadReturn);
+}
+
+template <class ReadOperation>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
+  return typeid(ReadOperation);
+}
+
+template <class... ReadOperations>
+utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
+  return TypeInfoForVariant(request.operation);
+}
+
+template <class... WriteReturn>
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
+  return TypeInfoForVariant(response.write_return);
+}
+
+template <class WriteReturn>
+utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
+  return typeid(WriteReturn);
+}
+
+template <class WriteOperation>
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
+  return typeid(WriteOperation);
+}
+
+template <class... WriteOperations>
+utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
+  return TypeInfoForVariant(request.operation);
+}
+
 /// AppendRequest is a raft-level message that the Leader
 /// periodically broadcasts to all Follower peers. This
 /// serves three main roles:
@@ -150,7 +192,7 @@ struct PendingClientRequest {
 
 struct Leader {
   std::map<Address, FollowerTracker> followers;
-  std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
+  std::map<LogIndex, PendingClientRequest> pending_client_requests;
   Time last_broadcast = Time::min();
 
   std::string static ToString() { return "\tLeader   \t"; }
@@ -205,7 +247,7 @@ to a CAS operation.
 template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
           typename ReadResponseValue>
 concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
-  { state.Read(r) } -> std::same_as<ReadResponseValue>;
+  { state.Read(std::move(r)) } -> std::same_as<ReadResponseValue>;
   { state.Apply(w) } -> std::same_as<WriteResponseValue>;
 };
 
@@ -230,22 +272,60 @@ class Raft {
   Io<IoImpl> io_;
   std::vector<Address> peers_;
   ReplicatedState replicated_state_;
+  Time next_cron_;
 
  public:
   Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
       : io_(std::forward<Io<IoImpl>>(io)),
         peers_(peers),
-        replicated_state_(std::forward<ReplicatedState>(replicated_state)) {}
+        replicated_state_(std::forward<ReplicatedState>(replicated_state)) {
+    if (peers.empty()) {
+      role_ = Leader{};
+    }
+  }
+
+  /// Periodic protocol maintenance. Returns the time that Cron should be called again
+  /// in the future.
+  Time Cron() {
+    // dispatch periodic logic based on our role to a specific Cron method.
+    std::optional<Role> new_role = std::visit([&](auto &role) { return Cron(role); }, role_);
+
+    if (new_role) {
+      role_ = std::move(new_role).value();
+    }
+    const Duration random_cron_interval = RandomTimeout(kMinimumCronInterval, kMaximumCronInterval);
+
+    return io_.Now() + random_cron_interval;
+  }
+
+  /// Returns the Address for our underlying Io implementation
+  Address GetAddress() { return io_.GetAddress(); }
+
+  using ReceiveVariant = std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
+                                      WriteRequest<WriteOperation>, VoteRequest, VoteResponse>;
+
+  void Handle(ReceiveVariant &&message_variant, RequestId request_id, Address from_address) {
+    // dispatch the message to a handler based on our role,
+    // which can be specified in the Handle first argument,
+    // or it can be `auto` if it's a handler for several roles
+    // or messages.
+    std::optional<Role> new_role = std::visit(
+        [&](auto &&msg, auto &role) mutable {
+          return Handle(role, std::forward<decltype(msg)>(msg), request_id, from_address);
+        },
+        std::forward<ReceiveVariant>(message_variant), role_);
+
+    // TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
+    if (new_role) {
+      role_ = std::move(new_role).value();
+    }
+  }
 
   void Run() {
-    Time last_cron = io_.Now();
-
     while (!io_.ShouldShutDown()) {
       const auto now = io_.Now();
-      const Duration random_cron_interval = RandomTimeout(kMinimumCronInterval, kMaximumCronInterval);
-      if (now - last_cron > random_cron_interval) {
-        Cron();
-        last_cron = now;
+      if (now >= next_cron_) {
+        next_cron_ = Cron();
       }
 
       const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
@@ -323,7 +403,7 @@ class Raft {
         const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
         leader.pending_client_requests.erase(apply_index);
 
-        const WriteResponse<WriteResponseValue> resp{
+        WriteResponse<WriteResponseValue> resp{
             .success = true,
             .write_return = std::move(write_return),
             .raft_index = apply_index,
@@ -449,16 +529,6 @@ class Raft {
   /// been received.
   /////////////////////////////////////////////////////////////
 
-  /// Periodic protocol maintenance.
-  void Cron() {
-    // dispatch periodic logic based on our role to a specific Cron method.
-    std::optional<Role> new_role = std::visit([&](auto &role) { return Cron(role); }, role_);
-
-    if (new_role) {
-      role_ = std::move(new_role).value();
-    }
-  }
-
   // Raft paper - 5.2
   // Candidates keep sending Vote to peers until:
   // 1. receiving Append with a higher term (become Follower)
@@ -485,7 +555,7 @@ class Raft {
       for (const auto &peer : peers_) {
         // request_id not necessary to set because it's not a Future-backed Request.
         static constexpr auto request_id = 0;
-        io_.template Send<VoteRequest>(peer, request_id, request);
+        io_.template Send(peer, request_id, VoteRequest{request});
         outstanding_votes.insert(peer);
       }
 
@@ -521,7 +591,7 @@ class Raft {
     const Time now = io_.Now();
     const Duration broadcast_timeout = RandomTimeout(kMinimumBroadcastTimeout, kMaximumBroadcastTimeout);
 
-    if (now - leader.last_broadcast > broadcast_timeout) {
+    if (now > leader.last_broadcast + broadcast_timeout) {
       BroadcastAppendEntries(leader.followers);
       leader.last_broadcast = now;
     }
@@ -541,26 +611,6 @@ class Raft {
   /// message that has been received.
   /////////////////////////////////////////////////////////////
 
-  using ReceiveVariant = std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
-                                      WriteRequest<WriteOperation>, VoteRequest, VoteResponse>;
-
-  void Handle(ReceiveVariant &&message_variant, RequestId request_id, Address from_address) {
-    // dispatch the message to a handler based on our role,
-    // which can be specified in the Handle first argument,
-    // or it can be `auto` if it's a handler for several roles
-    // or messages.
-    std::optional<Role> new_role = std::visit(
-        [&](auto &&msg, auto &role) mutable {
-          return Handle(role, std::forward<decltype(msg)>(msg), request_id, from_address);
-        },
-        std::forward<ReceiveVariant>(message_variant), role_);
-
-    // TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
-    if (new_role) {
-      role_ = std::move(new_role).value();
-    }
-  }
-
   // all roles can receive Vote and possibly become a follower
   template <AllRoles ALL>
   std::optional<Role> Handle(ALL & /* variable */, VoteRequest &&req, RequestId request_id, Address from_address) {
@@ -575,13 +625,12 @@ class Raft {
       MG_ASSERT(std::max(req.term, state_.term) == req.term);
     }
 
-    const VoteResponse res{
-        .term = std::max(req.term, state_.term),
-        .committed_log_size = state_.committed_log_size,
-        .vote_granted = new_leader,
-    };
-
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id,
+             VoteResponse{
+                 .term = std::max(req.term, state_.term),
+                 .committed_log_size = state_.committed_log_size,
+                 .vote_granted = new_leader,
+             });
 
     if (new_leader) {
       // become a follower
@@ -643,7 +692,7 @@ class Raft {
 
       return Leader{
           .followers = std::move(followers),
-          .pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
+          .pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
       };
     }
 
@@ -669,6 +718,10 @@ class Raft {
         .log_size = state_.log.size(),
     };
 
+    static_assert(std::is_trivially_copyable_v<AppendResponse>,
+                  "This function copies this message, therefore it is important to be trivially copyable. Otherwise it "
+                  "should be moved");
+
     if constexpr (std::is_same<ALL, Leader>()) {
       MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
     }
@@ -687,7 +740,7 @@ class Raft {
       // become follower of this leader, reply with our log status
       state_.term = req.term;
 
-      io_.Send(from_address, request_id, res);
+      io_.Send(from_address, request_id, AppendResponse{res});
 
       Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
       return Follower{
@@ -698,7 +751,7 @@ class Raft {
 
     if (req.term < state_.term) {
       // nack this request from an old leader
-      io_.Send(from_address, request_id, res);
+      io_.Send(from_address, request_id, AppendResponse{res});
 
       return std::nullopt;
     }
@@ -759,7 +812,7 @@ class Raft {
 
     Log("returning log_size of ", res.log_size);
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, AppendResponse{res});
 
     return std::nullopt;
   }
@@ -807,18 +860,20 @@ class Raft {
   // Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
   std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
                              Address from_address) {
-    Log("handling ReadOperation");
-    ReadOperation read_operation = req.operation;
+    auto type_info = TypeInfoFor(req);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    Log("handling ReadOperation<" + demangled_name + ">");
+    ReadOperation &read_operation = req.operation;
 
-    ReadResponseValue read_return = replicated_state_.Read(read_operation);
+    ReadResponseValue read_return = replicated_state_.Read(std::move(read_operation));
 
-    const ReadResponse<ReadResponseValue> resp{
+    ReadResponse<ReadResponseValue> resp{
         .success = true,
         .read_return = std::move(read_return),
         .retry_leader = std::nullopt,
     };
 
-    io_.Send(from_address, request_id, resp);
+    io_.Send(from_address, request_id, std::move(resp));
 
     return std::nullopt;
   }
@@ -827,11 +882,11 @@ class Raft {
   std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
                              RequestId request_id, Address from_address) {
     Log("received ReadOperation - not redirecting because no Leader is known");
-    const ReadResponse<ReadResponseValue> res{
+    ReadResponse<ReadResponseValue> res{
         .success = false,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     Cron();
 
@@ -843,12 +898,12 @@ class Raft {
                              Address from_address) {
     Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
 
-    const ReadResponse<ReadResponseValue> res{
+    ReadResponse<ReadResponseValue> res{
         .success = false,
         .retry_leader = follower.leader_address,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     return std::nullopt;
   }
@@ -862,12 +917,12 @@ class Raft {
                              Address from_address) {
     Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
 
-    const WriteResponse<WriteResponseValue> res{
+    WriteResponse<WriteResponseValue> res{
         .success = false,
         .retry_leader = follower.leader_address,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     return std::nullopt;
   }
@@ -876,11 +931,11 @@ class Raft {
                              RequestId request_id, Address from_address) {
     Log("received WriteRequest - not redirecting because no Leader is known");
 
-    const WriteResponse<WriteResponseValue> res{
+    WriteResponse<WriteResponseValue> res{
         .success = false,
     };
 
-    io_.Send(from_address, request_id, res);
+    io_.Send(from_address, request_id, std::move(res));
 
     Cron();
 
@@ -890,7 +945,9 @@ class Raft {
   // only leaders actually handle replication requests from clients
   std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
                              Address from_address) {
-    Log("handling WriteRequest");
+    auto type_info = TypeInfoFor(req);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    Log("handling WriteRequest<" + demangled_name + ">");
 
     // we are the leader. add item to log and send Append to peers
     MG_ASSERT(state_.term >= LastLogTerm());
@@ -906,7 +963,11 @@ class Raft {
 
     leader.pending_client_requests.emplace(log_index, pcr);
 
-    BroadcastAppendEntries(leader.followers);
+    if (peers_.empty()) {
+      BumpCommitIndexAndReplyToClients(leader);
+    } else {
+      BroadcastAppendEntries(leader.followers);
+    }
 
     return std::nullopt;
   }

src/io/rsm/rsm_client.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,9 +13,13 @@
 
 #include <iostream>
 #include <optional>
+#include <type_traits>
+#include <unordered_map>
 #include <vector>
 
 #include "io/address.hpp"
+#include "io/errors.hpp"
+#include "io/notifier.hpp"
 #include "io/rsm/raft.hpp"
 #include "utils/result.hpp"
 
@@ -34,6 +38,14 @@ using memgraph::io::rsm::WriteRequest;
 using memgraph::io::rsm::WriteResponse;
 using memgraph::utils::BasicResult;
 
+template <typename RequestT, typename ResponseT>
+struct AsyncRequest {
+  Time start_time;
+  RequestT request;
+  Notifier notifier;
+  ResponseFuture<ResponseT> future;
+};
+
 template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
           typename ReadResponseT>
 class RsmClient {
@@ -43,21 +55,28 @@ class RsmClient {
   Address leader_;
   ServerPool server_addrs_;
 
+  /// State for single async read/write operations. In the future this could become a map
+  /// of async operations that can be accessed via an ID etc...
+  std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
+  std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
+
+  void SelectRandomLeader() {
+    std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
+    size_t addr_index = io_.Rand(addr_distrib);
+    leader_ = server_addrs_[addr_index];
+
+    spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
+  }
+
   template <typename ResponseT>
   void PossiblyRedirectLeader(const ResponseT &response) {
     if (response.retry_leader) {
       MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
       leader_ = response.retry_leader.value();
       spdlog::debug("client redirected to leader server {}", leader_.ToString());
-    } else if (!response.success) {
-      std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
-      size_t addr_index = io_.Rand(addr_distrib);
-      leader_ = server_addrs_[addr_index];
-
-      spdlog::debug(
-          "client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
-          "a RSM Candidate) trying a random one at index {} with address {}",
-          addr_index, leader_.ToString());
+    }
+    if (!response.success) {
+      SelectRandomLeader();
     }
   }
 
@@ -65,71 +84,173 @@ class RsmClient {
   RsmClient(Io<IoImpl> io, Address leader, ServerPool server_addrs)
       : io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
 
+  RsmClient(const RsmClient &) = delete;
+  RsmClient &operator=(const RsmClient &) = delete;
+  RsmClient(RsmClient &&) noexcept = default;
+  RsmClient &operator=(RsmClient &&) noexcept = default;
+
   RsmClient() = delete;
+  ~RsmClient() = default;
 
   BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
-    WriteRequest<WriteRequestT> client_req;
-    client_req.operation = req;
-
-    const Duration overall_timeout = io_.GetDefaultTimeout();
-    const Time before = io_.Now();
-
-    do {
-      spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
-      ResponseFuture<WriteResponse<WriteResponseT>> response_future =
-          io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
-      ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
-
-      if (response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        return response_result.GetError();
-      }
-
-      ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
-      WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
-
-      if (write_response.success) {
-        return std::move(write_response.write_return);
-      }
-
-      PossiblyRedirectLeader(write_response);
-    } while (io_.Now() < before + overall_timeout);
-
-    return TimedOut{};
+    Notifier notifier;
+    const ReadinessToken readiness_token{0};
+    SendAsyncWriteRequest(std::move(req), notifier, readiness_token);
+    auto poll_result = AwaitAsyncWriteRequest(readiness_token);
+    while (!poll_result) {
+      poll_result = AwaitAsyncWriteRequest(readiness_token);
+    }
+    return poll_result.value();
   }
 
   BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
-    ReadRequest<ReadRequestT> read_req;
-    read_req.operation = req;
+    Notifier notifier;
+    const ReadinessToken readiness_token{0};
+    SendAsyncReadRequest(std::move(req), notifier, readiness_token);
+    auto poll_result = AwaitAsyncReadRequest(readiness_token);
+    while (!poll_result) {
+      poll_result = AwaitAsyncReadRequest(readiness_token);
+    }
+    return poll_result.value();
+  }
+
+  /// AsyncRead methods
+  void SendAsyncReadRequest(ReadRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
+    ReadRequest<ReadRequestT> read_req = {.operation = req};
+
+    AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
+        .start_time = io_.Now(),
+        .request = std::move(req),
+        .notifier = notifier,
+        .future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
+            leader_, std::move(read_req), notifier, readiness_token),
+    };
+
+    async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
+  }
+
+  void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
+
+    ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
+
+    async_request.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
+        leader_, std::move(read_req), async_request.notifier, readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
+
+    if (!async_request.future.IsReady()) {
+      return std::nullopt;
+    }
+
+    return AwaitAsyncReadRequest(readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_reads_.at(readiness_token.GetId());
+    ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
 
     const Duration overall_timeout = io_.GetDefaultTimeout();
-    const Time before = io_.Now();
+    const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
+    const bool result_has_error = get_response_result.HasError();
 
-    do {
-      spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
-
-      ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
-          io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
-
-      // receive response
-      ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
-
-      if (get_response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        return get_response_result.GetError();
-      }
+    if (result_has_error && past_time_out) {
+      // TODO static assert the exact type of error.
+      spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
+      async_reads_.erase(readiness_token.GetId());
+      return TimedOut{};
+    }
 
+    if (!result_has_error) {
       ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
       ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
 
+      PossiblyRedirectLeader(read_get_response);
+
       if (read_get_response.success) {
+        async_reads_.erase(readiness_token.GetId());
+        spdlog::debug("returning read_return for RSM request");
         return std::move(read_get_response.read_return);
       }
+    } else {
+      SelectRandomLeader();
+    }
 
-      PossiblyRedirectLeader(read_get_response);
-    } while (io_.Now() < before + overall_timeout);
+    ResendAsyncReadRequest(readiness_token);
 
-    return TimedOut{};
+    return std::nullopt;
+  }
+
+  /// AsyncWrite methods
+  void SendAsyncWriteRequest(WriteRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
+    WriteRequest<WriteRequestT> write_req = {.operation = req};
+
+    AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
+        .start_time = io_.Now(),
+        .request = std::move(req),
+        .notifier = notifier,
+        .future = io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
+            leader_, std::move(write_req), notifier, readiness_token),
+    };
+
+    async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
+  }
+
+  void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
+
+    WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
+
+    async_request.future =
+        io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
+            leader_, std::move(write_req), async_request.notifier, readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
+
+    if (!async_request.future.IsReady()) {
+      return std::nullopt;
+    }
+
+    return AwaitAsyncWriteRequest(readiness_token);
+  }
+
+  std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
+    auto &async_request = async_writes_.at(readiness_token.GetId());
+    ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
+
+    const Duration overall_timeout = io_.GetDefaultTimeout();
+    const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
+    const bool result_has_error = get_response_result.HasError();
+
+    if (result_has_error && past_time_out) {
+      // TODO static assert the exact type of error.
+      spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
+      async_writes_.erase(readiness_token.GetId());
+      return TimedOut{};
+    }
+
+    if (!result_has_error) {
+      ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
+          std::move(get_response_result.GetValue());
+      WriteResponse<WriteResponseT> &&write_get_response = std::move(get_response_envelope.message);
+
+      PossiblyRedirectLeader(write_get_response);
+
+      if (write_get_response.success) {
+        async_writes_.erase(readiness_token.GetId());
+        return std::move(write_get_response.write_return);
+      }
+    } else {
+      SelectRandomLeader();
+    }
+
+    ResendAsyncWriteRequest(readiness_token);
+
+    return std::nullopt;
   }
 };
 

src/io/simulator/simulator.hpp

@@ -29,6 +29,8 @@ class Simulator {
   explicit Simulator(SimulatorConfig config)
       : rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
 
+  ~Simulator() { ShutDown(); }
+
   void ShutDown() { simulator_handle_->ShutDown(); }
 
   Io<SimulatorTransport> RegisterNew() {
@@ -39,7 +41,7 @@ class Simulator {
   Io<SimulatorTransport> Register(Address address) {
     std::uniform_int_distribution<uint64_t> seed_distrib;
     uint64_t seed = seed_distrib(rng_);
-    return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
+    return Io{SimulatorTransport(simulator_handle_, address, seed), address};
   }
 
   void IncrementServerCountAndWaitForQuiescentState(Address address) {
@@ -47,5 +49,14 @@ class Simulator {
   }
 
   SimulatorStats Stats() { return simulator_handle_->Stats(); }
+
+  std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
+
+  std::function<bool()> GetSimulatorTickClosure() {
+    std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
+      return handle_copy->MaybeTickSimulator();
+    };
+    return tick_closure;
+  }
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_config.hpp

@@ -26,5 +26,6 @@ struct SimulatorConfig {
   uint64_t rng_seed = 0;
   Time start_time = Time::min();
   Time abort_time = Time::max();
+  Duration message_delay = std::chrono::microseconds(100);
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_handle.cpp

@@ -16,15 +16,19 @@
 #include "io/simulator/simulator_stats.hpp"
 #include "io/time.hpp"
 #include "io/transport.hpp"
+#include "utils/exceptions.hpp"
 
 namespace memgraph::io::simulator {
 
-using memgraph::io::Duration;
-using memgraph::io::Time;
-
 void SimulatorHandle::ShutDown() {
   std::unique_lock<std::mutex> lock(mu_);
   should_shut_down_ = true;
+  for (auto it = promises_.begin(); it != promises_.end();) {
+    auto &[promise_key, dop] = *it;
+    std::move(dop).promise.TimeOut();
+    it = promises_.erase(it);
+  }
+  can_receive_.clear();
   cv_.notify_all();
 }
 
@@ -33,57 +37,99 @@ bool SimulatorHandle::ShouldShutDown() const {
   return should_shut_down_;
 }
 
+LatencyHistogramSummaries SimulatorHandle::ResponseLatencies() {
+  std::unique_lock<std::mutex> lock(mu_);
+  return histograms_.ResponseLatencies();
+}
+
 void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
   std::unique_lock<std::mutex> lock(mu_);
   server_addresses_.insert(address);
 
   while (true) {
-    const size_t blocked_servers = blocked_on_receive_;
+    const size_t blocked_servers = blocked_on_receive_.size();
 
     const bool all_servers_blocked = blocked_servers == server_addresses_.size();
 
     if (all_servers_blocked) {
+      spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
+                    server_addresses_.size());
       return;
     }
 
+    spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
+                  server_addresses_.size());
     cv_.wait(lock);
   }
 }
 
+bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
+  // NB: never sort based on the request ID etc...
+  // This should only be used from std::stable_sort
+  // because by comparing on the from_address alone,
+  // we expect the sender ordering to remain
+  // deterministic.
+  const auto &[addr_1, om_1] = lhs;
+  const auto &[addr_2, om_2] = rhs;
+  return om_1.from_address < om_2.from_address;
+}
+
 bool SimulatorHandle::MaybeTickSimulator() {
   std::unique_lock<std::mutex> lock(mu_);
 
-  const size_t blocked_servers = blocked_on_receive_;
+  const size_t blocked_servers = blocked_on_receive_.size();
 
-  if (blocked_servers < server_addresses_.size()) {
+  if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
     // we only need to advance the simulator when all
     // servers have reached a quiescent state, blocked
     // on their own futures or receive methods.
     return false;
   }
 
+  // We allow the simulator to progress the state of the system only
+  // after all servers are blocked on receive.
+  spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
   stats_.simulator_ticks++;
-
+  blocked_on_receive_.clear();
   cv_.notify_all();
 
-  TimeoutPromisesPastDeadline();
+  bool timed_anything_out = TimeoutPromisesPastDeadline();
+
+  if (timed_anything_out) {
+    spdlog::trace("simulator progressing: timed out a request");
+  }
+
+  const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
+
+  // We don't always want to advance the clock with every message that we deliver because
+  // when we advance it for every message, it causes timeouts to occur for any "request path"
+  // over a certain length. Alternatively, we don't want to simply deliver multiple messages
+  // in a single simulator tick because that would reduce the amount of concurrent message
+  // mixing that may naturally occur in production. This approach is to mod the random clock
+  // advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
+  // by only advancing the clock by an even amount etc...) and only advancing the clock close to
+  // half of the time.
+  if (clock_advance.count() % 97 > 49) {
+    spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
+    cluster_wide_time_microseconds_ += clock_advance;
+    stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
+  }
+
+  if (cluster_wide_time_microseconds_ >= config_.abort_time) {
+    spdlog::error(
+        "Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
+        "in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
+        config_.rng_seed);
+    throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
+  }
 
   if (in_flight_.empty()) {
-    // return early here because there are no messages to schedule
-
-    // We tick the clock forward when all servers are blocked but
-    // there are no in-flight messages to schedule delivery of.
-    std::poisson_distribution<> time_distrib(50);
-    Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
-    cluster_wide_time_microseconds_ += clock_advance;
-
-    MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
-              "Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
-              "in an expected amount of time.");
     return true;
   }
 
-  if (config_.scramble_messages) {
+  std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
+
+  if (config_.scramble_messages && in_flight_.size() > 1) {
     // scramble messages
     std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
     const size_t swap_index = swap_distrib(rng_);
@@ -93,17 +139,14 @@ bool SimulatorHandle::MaybeTickSimulator() {
   auto [to_address, opaque_message] = std::move(in_flight_.back());
   in_flight_.pop_back();
 
-  std::uniform_int_distribution<int> drop_distrib(0, 99);
-  const int drop_threshold = drop_distrib(rng_);
+  const int drop_threshold = drop_distrib_(rng_);
   const bool should_drop = drop_threshold < config_.drop_percent;
 
   if (should_drop) {
     stats_.dropped_messages++;
   }
 
-  PromiseKey promise_key{.requester_address = to_address,
-                         .request_id = opaque_message.request_id,
-                         .replier_address = opaque_message.from_address};
+  PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
 
   if (promises_.contains(promise_key)) {
     // complete waiting promise if it's there
@@ -115,16 +158,25 @@ bool SimulatorHandle::MaybeTickSimulator() {
     if (should_drop || normal_timeout) {
       stats_.timed_out_requests++;
       dop.promise.TimeOut();
+      spdlog::trace("simulator timing out request ");
     } else {
       stats_.total_responses++;
-      dop.promise.Fill(std::move(opaque_message));
+      Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
+      auto type_info = opaque_message.type_info;
+      dop.promise.Fill(std::move(opaque_message), response_latency);
+      histograms_.Measure(type_info, response_latency);
+      spdlog::trace("simulator replying to request");
     }
   } else if (should_drop) {
     // don't add it anywhere, let it drop
+    spdlog::trace("simulator silently dropping request");
   } else {
     // add to can_receive_ if not
-    const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
-    om_vec->second.emplace_back(std::move(opaque_message));
+    spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
+                  opaque_message.to_address.last_known_port);
+    const auto &[om_vec, inserted] =
+        can_receive_.try_emplace(to_address.ToPartialAddress(), std::deque<OpaqueMessage>());
+    om_vec->second.emplace_front(std::move(opaque_message));
   }
 
   return true;

src/io/simulator/simulator_handle.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,9 +22,12 @@
 #include <variant>
 #include <vector>
 
+#include <boost/core/demangle.hpp>
+
 #include "io/address.hpp"
 #include "io/errors.hpp"
 #include "io/message_conversion.hpp"
+#include "io/message_histogram_collector.hpp"
 #include "io/simulator/simulator_config.hpp"
 #include "io/simulator/simulator_stats.hpp"
 #include "io/time.hpp"
@@ -43,35 +46,53 @@ class SimulatorHandle {
   std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
 
   // messages that are sent to servers that may later receive them
-  std::map<Address, std::vector<OpaqueMessage>> can_receive_;
+  std::map<PartialAddress, std::deque<OpaqueMessage>> can_receive_;
 
   Time cluster_wide_time_microseconds_;
   bool should_shut_down_ = false;
   SimulatorStats stats_;
-  size_t blocked_on_receive_ = 0;
+  std::set<Address> blocked_on_receive_;
   std::set<Address> server_addresses_;
   std::mt19937 rng_;
+  std::uniform_int_distribution<int> time_distrib_{0, 30000};
+  std::uniform_int_distribution<int> drop_distrib_{0, 99};
   SimulatorConfig config_;
+  MessageHistogramCollector histograms_;
+  RequestId request_id_counter_{0};
 
-  void TimeoutPromisesPastDeadline() {
+  bool TimeoutPromisesPastDeadline() {
+    bool timed_anything_out = false;
     const Time now = cluster_wide_time_microseconds_;
-
-    for (auto &[promise_key, dop] : promises_) {
-      if (dop.deadline < now) {
-        spdlog::debug("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
-                      promise_key.replier_address.ToString());
+    for (auto it = promises_.begin(); it != promises_.end();) {
+      auto &[promise_key, dop] = *it;
+      if (dop.deadline < now && config_.perform_timeouts) {
+        spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
         std::move(dop).promise.TimeOut();
-        promises_.erase(promise_key);
+        it = promises_.erase(it);
 
         stats_.timed_out_requests++;
+        timed_anything_out = true;
+      } else {
+        ++it;
       }
     }
+    return timed_anything_out;
   }
 
  public:
   explicit SimulatorHandle(SimulatorConfig config)
       : cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
 
+  LatencyHistogramSummaries ResponseLatencies();
+
+  ~SimulatorHandle() {
+    for (auto it = promises_.begin(); it != promises_.end();) {
+      auto &[promise_key, dop] = *it;
+      std::move(dop).promise.TimeOut();
+      it = promises_.erase(it);
+    }
+  }
+
   void IncrementServerCountAndWaitForQuiescentState(Address address);
 
   /// This method causes most of the interesting simulation logic to happen, wrt network behavior.
@@ -84,80 +105,122 @@ class SimulatorHandle {
 
   bool ShouldShutDown() const;
 
-  template <Message Request, Message Response>
-  void SubmitRequest(Address to_address, Address from_address, RequestId request_id, Request &&request,
-                     Duration timeout, ResponsePromise<Response> &&promise) {
-    std::unique_lock<std::mutex> lock(mu_);
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
+                                          Duration timeout, std::function<bool()> &&maybe_tick_simulator,
+                                          std::function<void()> &&fill_notifier) {
+    auto type_info = TypeInfoFor(request);
+    std::string demangled_name = boost::core::demangle(type_info.get().name());
+    spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
 
-    const Time deadline = cluster_wide_time_microseconds_ + timeout;
+    auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
+        // set notifier for when the Future::Wait is called
+        std::move(maybe_tick_simulator),
+        // set notifier for when Promise::Fill is called
+        std::move(fill_notifier));
 
-    std::any message(request);
-    OpaqueMessage om{.to_address = to_address,
-                     .from_address = from_address,
-                     .request_id = request_id,
-                     .message = std::move(message)};
-    in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
+    {
+      std::unique_lock<std::mutex> lock(mu_);
 
-    PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
-    OpaquePromise opaque_promise(std::move(promise).ToUnique());
-    DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
-    promises_.emplace(std::move(promise_key), std::move(dop));
+      RequestId request_id = ++request_id_counter_;
 
-    stats_.total_messages++;
-    stats_.total_requests++;
+      const Time deadline = cluster_wide_time_microseconds_ + timeout;
+
+      std::any message(std::move(request));
+      OpaqueMessage om{.to_address = to_address,
+                       .from_address = from_address,
+                       .request_id = request_id,
+                       .message = std::move(message),
+                       .type_info = type_info,
+                       .deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
+      in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
+
+      PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
+      OpaquePromise opaque_promise(std::move(promise).ToUnique());
+      DeadlineAndOpaquePromise dop{
+          .requested_at = cluster_wide_time_microseconds_,
+          .deadline = deadline,
+          .promise = std::move(opaque_promise),
+      };
+
+      MG_ASSERT(!promises_.contains(promise_key));
+
+      promises_.emplace(std::move(promise_key), std::move(dop));
+
+      stats_.total_messages++;
+      stats_.total_requests++;
+    }  // lock dropped here
 
     cv_.notify_all();
+
+    return std::move(future);
   }
 
   template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
     std::unique_lock<std::mutex> lock(mu_);
 
-    blocked_on_receive_ += 1;
-
     const Time deadline = cluster_wide_time_microseconds_ + timeout;
 
+    auto partial_address = receiver.ToPartialAddress();
+
     while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
-      if (can_receive_.contains(receiver)) {
-        std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
-        if (!can_rx.empty()) {
+      if (can_receive_.contains(partial_address)) {
+        std::deque<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
+
+        bool contains_items = !can_rx.empty();
+        bool can_receive = contains_items && can_rx.back().deliverable_at <= cluster_wide_time_microseconds_;
+
+        if (can_receive) {
           OpaqueMessage message = std::move(can_rx.back());
           can_rx.pop_back();
 
           // TODO(tyler) search for item in can_receive_ that matches the desired types, rather
           // than asserting that the last item in can_rx matches.
           auto m_opt = std::move(message).Take<Ms...>();
-
-          blocked_on_receive_ -= 1;
+          MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
 
           return std::move(m_opt).value();
         }
+        if (contains_items) {
+          auto count = can_rx.back().deliverable_at.time_since_epoch().count();
+          auto now_count = cluster_wide_time_microseconds_.time_since_epoch().count();
+          spdlog::trace("can't receive message yet due to artificial latency. deliverable_at: {}, now: {}", count,
+                        now_count);
+        }
       }
 
-      lock.unlock();
-      bool made_progress = MaybeTickSimulator();
-      lock.lock();
-      if (!should_shut_down_ && !made_progress) {
+      if (!should_shut_down_) {
+        if (!blocked_on_receive_.contains(receiver)) {
+          blocked_on_receive_.emplace(receiver);
+          spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
+          cv_.notify_all();
+        }
         cv_.wait(lock);
       }
     }
-
-    blocked_on_receive_ -= 1;
+    spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
 
     return TimedOut{};
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, RequestId request_id, M message) {
-    std::unique_lock<std::mutex> lock(mu_);
-    std::any message_any(std::move(message));
-    OpaqueMessage om{.to_address = to_address,
-                     .from_address = from_address,
-                     .request_id = request_id,
-                     .message = std::move(message_any)};
-    in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
+  void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
+    spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
+    auto type_info = TypeInfoFor(message);
+    {
+      std::unique_lock<std::mutex> lock(mu_);
+      std::any message_any(std::move(message));
+      OpaqueMessage om{.to_address = to_address,
+                       .from_address = from_address,
+                       .request_id = request_id,
+                       .message = std::move(message_any),
+                       .type_info = type_info,
+                       .deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
+      in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
 
-    stats_.total_messages++;
+      stats_.total_messages++;
+    }  // lock dropped before cv notification
 
     cv_.notify_all();
   }

src/io/simulator/simulator_stats.hpp

@@ -13,6 +13,10 @@
 
 #include <cstdint>
 
+#include <fmt/format.h>
+
+#include "io/time.hpp"
+
 namespace memgraph::io::simulator {
 struct SimulatorStats {
   uint64_t total_messages = 0;
@@ -21,5 +25,22 @@ struct SimulatorStats {
   uint64_t total_requests = 0;
   uint64_t total_responses = 0;
   uint64_t simulator_ticks = 0;
+  Duration elapsed_time;
+
+  friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
+
+  friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
+    auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
+
+    std::string formated = fmt::format(
+        "SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
+        "total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
+        stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
+        stats.total_responses, stats.simulator_ticks, elapsed_ms);
+
+    in << formated;
+
+    return in;
+  }
 };
 };  // namespace memgraph::io::simulator

src/io/simulator/simulator_transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,6 +15,7 @@
 #include <utility>
 
 #include "io/address.hpp"
+#include "io/notifier.hpp"
 #include "io/simulator/simulator_handle.hpp"
 #include "io/time.hpp"
 
@@ -25,32 +26,32 @@ using memgraph::io::Time;
 
 class SimulatorTransport {
   std::shared_ptr<SimulatorHandle> simulator_handle_;
-  const Address address_;
+  Address address_;
   std::mt19937 rng_;
 
  public:
   SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
       : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
 
-  template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> Request(Address address, uint64_t request_id, RequestT request, Duration timeout) {
-    std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
-    auto [future, promise] =
-        memgraph::io::FuturePromisePairWithNotifier<ResponseResult<ResponseT>>(maybe_tick_simulator);
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
+                                    std::function<void()> notification, Duration timeout) {
+    std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
+      return handle_copy->MaybeTickSimulator();
+    };
 
-    simulator_handle_->SubmitRequest(address, address_, request_id, std::move(request), timeout, std::move(promise));
-
-    return std::move(future);
+    return simulator_handle_->template SubmitRequest<ResponseT, RequestT>(
+        to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
   }
 
   template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Duration timeout) {
-    return simulator_handle_->template Receive<Ms...>(address_, timeout);
+  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
+    return simulator_handle_->template Receive<Ms...>(receiver_address, timeout);
   }
 
   template <Message M>
-  void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
-    return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
+  void Send(Address to_address, Address from_address, uint64_t request_id, RValueRef<M> message) {
+    return simulator_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
   }
 
   Time Now() const { return simulator_handle_->Now(); }
@@ -61,5 +62,7 @@ class SimulatorTransport {
   Return Rand(D distrib) {
     return distrib(rng_);
   }
+
+  LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }
 };
 };  // namespace memgraph::io::simulator

src/io/transport.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,7 +19,10 @@
 #include "io/address.hpp"
 #include "io/errors.hpp"
 #include "io/future.hpp"
+#include "io/message_histogram_collector.hpp"
+#include "io/notifier.hpp"
 #include "io/time.hpp"
+#include "utils/concepts.hpp"
 #include "utils/result.hpp"
 
 namespace memgraph::io {
@@ -30,7 +33,15 @@ using memgraph::utils::BasicResult;
 // reasonable constraints around message types over time,
 // as we adapt things to use Thrift-generated message types.
 template <typename T>
-concept Message = std::same_as<T, std::decay_t<T>>;
+concept Message = std::movable<T> && std::copyable<T>;
+
+template <utils::Object T>
+struct RValueRefEnforcer {
+  using Type = T &&;
+};
+
+template <typename T>
+using RValueRef = typename RValueRefEnforcer<T>::Type;
 
 using RequestId = uint64_t;
 
@@ -40,6 +51,7 @@ struct ResponseEnvelope {
   RequestId request_id;
   Address to_address;
   Address from_address;
+  Duration response_latency;
 };
 
 template <Message M>
@@ -66,8 +78,7 @@ template <typename I>
 class Io {
   I implementation_;
   Address address_;
-  RequestId request_id_counter_ = 0;
-  Duration default_timeout_ = std::chrono::microseconds{50000};
+  Duration default_timeout_ = std::chrono::microseconds{100000};
 
  public:
   Io(I io, Address address) : implementation_(io), address_(address) {}
@@ -80,26 +91,51 @@ class Io {
   Duration GetDefaultTimeout() { return default_timeout_; }
 
   /// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
-  template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
-    const RequestId request_id = ++request_id_counter_;
-    return implementation_.template Request<RequestT, ResponseT>(address, request_id, request, timeout);
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> RequestWithTimeout(Address address, RValueRef<RequestT> request, Duration timeout) {
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<ResponseT, RequestT>(address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
   }
 
   /// Issue a request that times out after the default timeout. This tends
   /// to be used by clients.
-  template <Message RequestT, Message ResponseT>
-  ResponseFuture<ResponseT> Request(Address address, RequestT request) {
-    const RequestId request_id = ++request_id_counter_;
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> Request(Address to_address, RValueRef<RequestT> request) {
     const Duration timeout = default_timeout_;
-    return implementation_.template Request<RequestT, ResponseT>(address, request_id, std::move(request), timeout);
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = nullptr;
+    return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RValueRef<RequestT> request, Notifier notifier,
+                                                    ReadinessToken readiness_token) {
+    const Duration timeout = default_timeout_;
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
+                                                                 fill_notifier, timeout);
+  }
+
+  /// Issue a request that will notify a Notifier when it is filled or times out.
+  template <Message ResponseT, Message RequestT>
+  ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT &&request, Notifier notifier,
+                                                              ReadinessToken readiness_token, Duration timeout) {
+    const Address from_address = address_;
+    std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
+    return implementation_.template Request<ResponseT>(to_address, from_address, std::forward<RequestT>(request),
+                                                       fill_notifier, timeout);
   }
 
   /// Wait for an explicit number of microseconds for a request of one of the
   /// provided types to arrive. This tends to be used by servers.
   template <Message... Ms>
   RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
-    return implementation_.template Receive<Ms...>(timeout);
+    return implementation_.template Receive<Ms...>(address_, timeout);
   }
 
   /// Wait the default number of microseconds for a request of one of the
@@ -107,16 +143,16 @@ class Io {
   template <Message... Ms>
   requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
     const Duration timeout = default_timeout_;
-    return implementation_.template Receive<Ms...>(timeout);
+    return implementation_.template Receive<Ms...>(address_, timeout);
   }
 
   /// Send a message in a best-effort fashion. This is used for messaging where
   /// responses are not necessarily expected, and for servers to respond to requests.
   /// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
   template <Message M>
-  void Send(Address to_address, RequestId request_id, M message) {
+  void Send(Address to_address, RequestId request_id, M &&message) {
     Address from_address = address_;
-    return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
+    return implementation_.template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
   }
 
   /// The current system time. This time source should be preferred over any other,
@@ -134,5 +170,17 @@ class Io {
   }
 
   Address GetAddress() { return address_; }
+  void SetAddress(Address address) { address_ = address; }
+
+  Io<I> ForkLocal(boost::uuids::uuid uuid) {
+    Address new_address{
+        .unique_id = uuid,
+        .last_known_ip = address_.last_known_ip,
+        .last_known_port = address_.last_known_port,
+    };
+    return Io(implementation_, new_address);
+  }
+
+  LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
 };
 };  // namespace memgraph::io

src/machine_manager/machine_config.hpp

@@ -0,0 +1,48 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <algorithm>
+#include <thread>
+
+#include <boost/asio/ip/tcp.hpp>
+
+#include "io/address.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/schemas.hpp"
+
+namespace memgraph::machine_manager {
+
+using memgraph::io::Address;
+using memgraph::storage::v3::SchemaProperty;
+using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
+
+struct InitialLabelSpace {
+  std::string label_name;
+  std::vector<SchemaProperty> schema;
+  size_t replication_factor;
+  std::vector<CompoundKey> split_points;
+};
+
+struct MachineConfig {
+  std::vector<InitialLabelSpace> initial_label_spaces;
+  std::vector<Address> coordinator_addresses;
+  bool is_storage;
+  bool is_coordinator;
+  bool is_query_engine;
+  boost::asio::ip::address listen_ip;
+  uint16_t listen_port;
+  size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
+  bool sync_message_handling = false;
+};
+
+}  // namespace memgraph::machine_manager

src/machine_manager/machine_manager.hpp

@@ -0,0 +1,233 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "coordinator/coordinator_rsm.hpp"
+#include "coordinator/coordinator_worker.hpp"
+#include "io/message_conversion.hpp"
+#include "io/messages.hpp"
+#include "io/rsm/rsm_client.hpp"
+#include "io/time.hpp"
+#include "machine_manager/machine_config.hpp"
+#include "storage/v3/shard_manager.hpp"
+
+namespace memgraph::machine_manager {
+
+using coordinator::Coordinator;
+using coordinator::CoordinatorReadRequests;
+using coordinator::CoordinatorReadResponses;
+using coordinator::CoordinatorRsm;
+using coordinator::CoordinatorWriteRequests;
+using coordinator::CoordinatorWriteResponses;
+using coordinator::coordinator_worker::CoordinatorWorker;
+using CoordinatorRouteMessage = coordinator::coordinator_worker::RouteMessage;
+using CoordinatorQueue = coordinator::coordinator_worker::Queue;
+using io::ConvertVariant;
+using io::Duration;
+using io::RequestId;
+using io::Time;
+using io::messages::CoordinatorMessages;
+using io::messages::ShardManagerMessages;
+using io::messages::ShardMessages;
+using io::messages::StorageReadRequest;
+using io::messages::StorageWriteRequest;
+using io::rsm::AppendRequest;
+using io::rsm::AppendResponse;
+using io::rsm::ReadRequest;
+using io::rsm::VoteRequest;
+using io::rsm::VoteResponse;
+using io::rsm::WriteRequest;
+using io::rsm::WriteResponse;
+using storage::v3::ShardManager;
+
+/// The MachineManager is responsible for:
+/// * starting the entire system and ensuring that high-level
+///   operational requirements continue to be met
+/// * acting as a machine's single caller of Io::Receive
+/// * routing incoming messages from the Io interface to the
+///   appropriate Coordinator or to the StorageManager
+///   (it's not necessary to route anything in this layer
+///   to the query engine because the query engine only
+///   communicates using higher-level Futures that will
+///   be filled immediately when the response comes in
+///   at the lower-level transport layer.
+///
+/// Every storage engine has exactly one RsmEngine.
+template <typename IoImpl>
+class MachineManager {
+  io::Io<IoImpl> io_;
+  MachineConfig config_;
+  Address coordinator_address_;
+  CoordinatorQueue coordinator_queue_;
+  std::jthread coordinator_handle_;
+  ShardManager<IoImpl> shard_manager_;
+  Time next_cron_ = Time::min();
+
+ public:
+  // TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress
+  // which is only true for single-machine config.
+  MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
+      : io_(io),
+        config_(config),
+        coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
+        shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
+                       coordinator_address_} {
+    auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
+    CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
+    coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
+  }
+
+  MachineManager(MachineManager &&) noexcept = default;
+  MachineManager &operator=(MachineManager &&) noexcept = default;
+  MachineManager(const MachineManager &) = delete;
+  MachineManager &operator=(const MachineManager &) = delete;
+
+  ~MachineManager() {
+    if (coordinator_handle_.joinable()) {
+      coordinator_queue_.Push(coordinator::coordinator_worker::ShutDown{});
+      coordinator_handle_.join();
+    }
+  }
+
+  Address CoordinatorAddress() { return coordinator_address_; }
+
+  void Run() {
+    while (true) {
+      MaybeBlockOnSyncHandling();
+
+      if (io_.ShouldShutDown()) {
+        break;
+      }
+
+      const auto now = io_.Now();
+
+      uint64_t now_us = now.time_since_epoch().count();
+      uint64_t next_us = next_cron_.time_since_epoch().count();
+
+      if (now >= next_cron_) {
+        spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
+        next_cron_ = Cron();
+      } else {
+        spdlog::info("now {} < next_cron_ {}", now_us, next_us);
+      }
+
+      Duration receive_timeout = std::max(next_cron_, now) - now;
+
+      // Note: this parameter pack must be kept in-sync with the ReceiveWithTimeout parameter pack below
+      using AllMessages =
+          std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
+                       WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse,
+                       WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
+                       AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
+
+      spdlog::info("MM waiting on Receive on address {}", io_.GetAddress().ToString());
+
+      // Note: this parameter pack must be kept in-sync with the AllMessages parameter pack above
+      auto request_result = io_.template ReceiveWithTimeout<
+          ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
+          WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse, WriteResponse<CoordinatorWriteResponses>,
+          ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>(
+          receive_timeout);
+
+      if (request_result.HasError()) {
+        // time to do Cron
+        continue;
+      }
+
+      auto &&request_envelope = std::move(request_result.GetValue());
+
+      spdlog::info("MM got message to {}", request_envelope.to_address.ToString());
+
+      // If message is for the coordinator, cast it to subset and pass it to the coordinator
+      bool to_coordinator = coordinator_address_ == request_envelope.to_address;
+      if (to_coordinator) {
+        std::optional<CoordinatorMessages> conversion_attempt =
+            ConvertVariant<AllMessages, ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>,
+                           AppendResponse, WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>(
+                std::move(request_envelope.message));
+
+        MG_ASSERT(conversion_attempt.has_value(), "coordinator message conversion failed");
+
+        spdlog::info("got coordinator message");
+
+        CoordinatorMessages &&cm = std::move(conversion_attempt.value());
+
+        CoordinatorRouteMessage route_message{
+            .message = std::move(cm),
+            .request_id = request_envelope.request_id,
+            .to = request_envelope.to_address,
+            .from = request_envelope.from_address,
+        };
+        coordinator_queue_.Push(std::move(route_message));
+        continue;
+      }
+
+      bool to_sm = shard_manager_.GetAddress() == request_envelope.to_address;
+      spdlog::info("smm: {}", shard_manager_.GetAddress().ToString());
+      if (to_sm) {
+        std::optional<ShardManagerMessages> conversion_attempt =
+            ConvertVariant<AllMessages, WriteResponse<CoordinatorWriteResponses>>(std::move(request_envelope.message));
+
+        MG_ASSERT(conversion_attempt.has_value(), "shard manager message conversion failed");
+
+        spdlog::info("got shard manager message");
+
+        ShardManagerMessages &&smm = std::move(conversion_attempt.value());
+        shard_manager_.Receive(std::forward<ShardManagerMessages>(smm), request_envelope.request_id,
+                               request_envelope.from_address);
+        continue;
+      }
+
+      // treat this as a message to a specific shard rsm and cast it accordingly
+
+      std::optional<ShardMessages> conversion_attempt =
+          ConvertVariant<AllMessages, ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>,
+                         AppendResponse, WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>(
+              std::move(request_envelope.message));
+
+      MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {} - incorrect message type",
+                request_envelope.to_address.ToString());
+
+      spdlog::info("got shard rsm message");
+
+      ShardMessages &&sm = std::move(conversion_attempt.value());
+      shard_manager_.Route(std::forward<ShardMessages>(sm), request_envelope.request_id, request_envelope.to_address,
+                           request_envelope.from_address);
+    }
+  }
+
+ private:
+  // This method exists for controlling concurrency
+  // during deterministic simulation testing.
+  void MaybeBlockOnSyncHandling() {
+    if (!config_.sync_message_handling) {
+      return;
+    }
+
+    // block on coordinator
+    coordinator_queue_.BlockOnQuiescence();
+
+    // block on shards
+    shard_manager_.BlockOnQuiescence();
+  }
+
+  Time Cron() {
+    spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
+    coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
+    MaybeBlockOnSyncHandling();
+    Time ret = shard_manager_.Cron();
+    MaybeBlockOnSyncHandling();
+    return ret;
+  }
+};
+
+}  // namespace memgraph::machine_manager

src/parser/CMakeLists.txt

@@ -23,7 +23,7 @@ add_custom_command(
     COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
     COMMAND
         java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-            -Dlanguage=Cpp -visitor -package antlropencypher
+            -Dlanguage=Cpp -visitor -package antlropencypher::v2
             -o ${opencypher_generated}
             ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"

src/parser/opencypher/parser.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,10 +14,10 @@
 #include <string>
 
 #include "antlr4-runtime.h"
-#include "utils/exceptions.hpp"
 #include "parser/opencypher/generated/MemgraphCypher.h"
 #include "parser/opencypher/generated/MemgraphCypherLexer.h"
 #include "utils/concepts.hpp"
+#include "utils/exceptions.hpp"
 
 namespace memgraph::frontend::opencypher {
 
@@ -32,11 +32,9 @@ class SyntaxException : public utils::BasicException {
  * This thing must me a class since parser.cypher() returns pointer and there is
  * no way for us to get ownership over the object.
  */
-enum class ParserOpTag : uint8_t {
-  CYPHER, EXPRESSION
-};
+enum class ParserOpTag : uint8_t { CYPHER, EXPRESSION };
 
-template<ParserOpTag Tag = ParserOpTag::CYPHER>
+template <ParserOpTag Tag = ParserOpTag::CYPHER>
 class Parser {
  public:
   /**
@@ -46,10 +44,9 @@ class Parser {
   Parser(const std::string query) : query_(std::move(query)) {
     parser_.removeErrorListeners();
     parser_.addErrorListener(&error_listener_);
-    if constexpr(Tag == ParserOpTag::CYPHER) {
+    if constexpr (Tag == ParserOpTag::CYPHER) {
       tree_ = parser_.cypher();
-    }
-    else {
+    } else {
       tree_ = parser_.expression();
     }
     if (parser_.getNumberOfSyntaxErrors()) {
@@ -75,11 +72,11 @@ class Parser {
   FirstMessageErrorListener error_listener_;
   std::string query_;
   antlr4::ANTLRInputStream input_{query_};
-  antlropencypher::MemgraphCypherLexer lexer_{&input_};
+  antlropencypher::v2::MemgraphCypherLexer lexer_{&input_};
   antlr4::CommonTokenStream tokens_{&lexer_};
 
   // generate ast
-  antlropencypher::MemgraphCypher parser_{&tokens_};
+  antlropencypher::v2::MemgraphCypher parser_{&tokens_};
   antlr4::tree::ParseTree *tree_ = nullptr;
 };
 }  // namespace memgraph::frontend::opencypher

src/query/CMakeLists.txt

@@ -48,18 +48,20 @@ add_dependencies(mg-query generate_lcp_query)
 target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(mg-query dl cppitertools Boost::headers)
 target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
+
 if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
     set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
 endif()
+
 if("${MG_PYTHON_VERSION}" STREQUAL "")
     find_package(Python3 3.5 REQUIRED COMPONENTS Development)
 else()
     find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
 endif()
+
 target_link_libraries(mg-query Python3::Python)
 
 # Generate Antlr openCypher parser
-
 set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
 set(opencypher_generated ${opencypher_frontend}/generated)
 set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
@@ -82,15 +84,15 @@ add_custom_command(
     OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
     COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
     COMMAND
-        java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-            -Dlanguage=Cpp -visitor -package antlropencypher
-            -o ${opencypher_generated}
-            ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+    java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
+    -Dlanguage=Cpp -visitor -package antlropencypher
+    -o ${opencypher_generated}
+    ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     DEPENDS
-        ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
-        ${opencypher_frontend}/grammar/CypherLexer.g4
-        ${opencypher_frontend}/grammar/Cypher.g4)
+    ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+    ${opencypher_frontend}/grammar/CypherLexer.g4
+    ${opencypher_frontend}/grammar/Cypher.g4)
 
 add_custom_target(generate_opencypher_parser
     DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})

src/query/v2/CMakeLists.txt

@@ -9,10 +9,8 @@ set(mg_query_v2_sources
     ${lcp_query_v2_cpp_files}
     common.cpp
     cypher_query_interpreter.cpp
-    dump.cpp
     frontend/semantic/required_privileges.cpp
     frontend/stripped.cpp
-    interpret/awesome_memgraph_functions.cpp
     interpreter.cpp
     metadata.cpp
     plan/operator.cpp
@@ -23,17 +21,10 @@ set(mg_query_v2_sources
     plan/rewrite/index_lookup.cpp
     plan/rule_based_planner.cpp
     plan/variable_start_planner.cpp
-    procedure/mg_procedure_impl.cpp
-    procedure/mg_procedure_helpers.cpp
-    procedure/module.cpp
-    procedure/py_module.cpp
     serialization/property_value.cpp
-    stream/streams.cpp
-    stream/sources.cpp
-    stream/common.cpp
-    trigger.cpp
-    trigger_context.cpp
-    bindings/typed_value.cpp)
+    bindings/typed_value.cpp
+    accessors.cpp
+    multiframe.cpp)
 
 find_package(Boost REQUIRED)
 
@@ -42,8 +33,8 @@ add_dependencies(mg-query-v2 generate_lcp_query_v2)
 target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
 target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
-target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory)
-target_link_libraries(mg-query-v2 mg-expr)
+target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
+target_link_libraries(mg-query-v2 mg-expr mg-functions)
 
 if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
     set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")

src/query/v2/accessors.cpp

@@ -0,0 +1,103 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
+#include "query/v2/requests.hpp"
+#include "storage/v3/id_types.hpp"
+
+namespace memgraph::query::v2::accessors {
+EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
+    : edge(std::move(edge)), request_router_(request_router) {}
+
+EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
+
+const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
+
+Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
+  auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
+  if (!maybe_prop) {
+    return {};
+  }
+  const auto prop_id = *maybe_prop;
+  auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
+  return it->second;
+}
+
+const Edge &EdgeAccessor::GetEdge() const { return edge; }
+
+bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
+
+size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
+
+VertexAccessor EdgeAccessor::To() const {
+  return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
+}
+
+VertexAccessor EdgeAccessor::From() const {
+  return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
+}
+
+VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
+                               const RequestRouterInterface *request_router)
+    : vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
+
+VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
+                               const RequestRouterInterface *request_router)
+    : vertex(std::move(v)), request_router_(request_router) {
+  properties.reserve(props.size());
+  for (auto &[id, value] : props) {
+    properties.emplace_back(std::make_pair(id, std::move(value)));
+  }
+}
+
+VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
+                               const RequestRouterInterface *request_router)
+    : vertex(std::move(v)), request_router_(request_router) {
+  properties.reserve(props.size());
+  for (const auto &[id, value] : props) {
+    properties.emplace_back(std::make_pair(id, value));
+  }
+}
+
+Label VertexAccessor::PrimaryLabel() const { return vertex.id.first; }
+
+const msgs::VertexId &VertexAccessor::Id() const { return vertex.id; }
+
+std::vector<Label> VertexAccessor::Labels() const { return vertex.labels; }
+
+bool VertexAccessor::HasLabel(Label &label) const {
+  return std::find_if(vertex.labels.begin(), vertex.labels.end(),
+                      [label](const auto &l) { return l.id == label.id; }) != vertex.labels.end();
+}
+
+const std::vector<std::pair<PropertyId, Value>> &VertexAccessor::Properties() const { return properties; }
+
+Value VertexAccessor::GetProperty(PropertyId prop_id) const {
+  auto it = std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; });
+  if (it == properties.end()) {
+    return {};
+  }
+  return it->second;
+}
+
+// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
+Value VertexAccessor::GetProperty(const std::string &prop_name) const {
+  auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
+  if (!maybe_prop) {
+    return {};
+  }
+  return GetProperty(*maybe_prop);
+}
+
+msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }
+
+}  // namespace memgraph::query::v2::accessors

src/query/v2/accessors.hpp

@@ -0,0 +1,164 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <map>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "query/exceptions.hpp"
+#include "query/v2/requests.hpp"
+#include "storage/v3/view.hpp"
+#include "utils/bound.hpp"
+#include "utils/exceptions.hpp"
+#include "utils/memory.hpp"
+#include "utils/memory_tracker.hpp"
+
+namespace memgraph::query::v2 {
+class RequestRouterInterface;
+}  // namespace memgraph::query::v2
+
+namespace memgraph::query::v2::accessors {
+
+using Value = msgs::Value;
+using Edge = msgs::Edge;
+using Vertex = msgs::Vertex;
+using Label = msgs::Label;
+using PropertyId = msgs::PropertyId;
+using EdgeTypeId = msgs::EdgeTypeId;
+
+class VertexAccessor;
+
+class EdgeAccessor final {
+ public:
+  explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
+
+  [[nodiscard]] EdgeTypeId EdgeType() const;
+
+  [[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
+
+  [[nodiscard]] Value GetProperty(const std::string &prop_name) const;
+
+  [[nodiscard]] const Edge &GetEdge() const;
+
+  [[nodiscard]] bool IsCycle() const;
+
+  [[nodiscard]] size_t CypherId() const;
+
+  [[nodiscard]] VertexAccessor To() const;
+  [[nodiscard]] VertexAccessor From() const;
+
+  friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return lhs.edge == rhs.edge; }
+
+  friend bool operator!=(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return !(lhs == rhs); }
+
+ private:
+  Edge edge;
+  const RequestRouterInterface *request_router_;
+};
+
+class VertexAccessor final {
+ public:
+  using PropertyId = msgs::PropertyId;
+  using Label = msgs::Label;
+  using VertexId = msgs::VertexId;
+  VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
+                 const RequestRouterInterface *request_router);
+
+  VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
+  VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
+
+  [[nodiscard]] Label PrimaryLabel() const;
+
+  [[nodiscard]] const msgs::VertexId &Id() const;
+
+  [[nodiscard]] std::vector<Label> Labels() const;
+
+  [[nodiscard]] bool HasLabel(Label &label) const;
+
+  [[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
+
+  [[nodiscard]] Value GetProperty(PropertyId prop_id) const;
+  [[nodiscard]] Value GetProperty(const std::string &prop_name) const;
+
+  [[nodiscard]] msgs::Vertex GetVertex() const;
+
+  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
+  [[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
+
+  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
+  [[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
+
+  friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
+    return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
+  }
+
+  friend bool operator!=(const VertexAccessor &lhs, const VertexAccessor &rhs) { return !(lhs == rhs); }
+
+ private:
+  Vertex vertex;
+  std::vector<std::pair<PropertyId, Value>> properties;
+  const RequestRouterInterface *request_router_;
+};
+
+// Highly mocked interface. Won't work if used.
+class Path {
+ public:
+  // Empty for now
+  explicit Path(const VertexAccessor & /*vertex*/, utils::MemoryResource *memory = utils::NewDeleteResource())
+      : mem(memory) {}
+
+  template <typename... TOthers>
+  explicit Path(const VertexAccessor &vertex, const TOthers &...others) {}
+
+  template <typename... TOthers>
+  Path(std::allocator_arg_t /*unused*/, utils::MemoryResource *memory, const VertexAccessor &vertex,
+       const TOthers &...others) {}
+
+  Path(const Path & /*other*/) {}
+
+  Path(const Path & /*other*/, utils::MemoryResource *memory) : mem(memory) {}
+
+  Path(Path && /*other*/) noexcept {}
+
+  Path(Path && /*other*/, utils::MemoryResource *memory) : mem(memory) {}
+  Path &operator=(const Path &path) {
+    if (this == &path) {
+      return *this;
+    }
+    return *this;
+  }
+
+  Path &operator=(Path &&path) noexcept {
+    if (this == &path) {
+      return *this;
+    }
+    return *this;
+  }
+
+  ~Path() {}
+
+  friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
+  utils::MemoryResource *GetMemoryResource() { return mem; }
+
+  auto &vertices() { return vertices_; }
+  auto &edges() { return edges_; }
+  const auto &vertices() const { return vertices_; }
+  const auto &edges() const { return edges_; }
+
+ private:
+  std::vector<VertexAccessor> vertices_;
+  std::vector<EdgeAccessor> edges_;
+  utils::MemoryResource *mem = utils::NewDeleteResource();
+};
+}  // namespace memgraph::query::v2::accessors

src/query/v2/bindings/bindings.hpp

@@ -11,5 +11,9 @@
 
 #pragma once
 
+#ifdef MG_AST_INCLUDE_PATH
+#error You are probably trying to include files from expr from both the storage and query engines! You will have a rough time kid!
+#endif
+
 #define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"  // NOLINT(cppcoreguidelines-macro-usage)
 #define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2       // NOLINT(cppcoreguidelines-macro-usage)

src/query/v2/bindings/eval.hpp

@@ -16,19 +16,32 @@
 #include "expr/interpret/eval.hpp"
 #include "query/v2/bindings/typed_value.hpp"
 #include "query/v2/context.hpp"
+#include "query/v2/conversions.hpp"
 #include "query/v2/db_accessor.hpp"
-#include "storage/v3/id_types.hpp"
-#include "storage/v3/property_store.hpp"
-#include "storage/v3/view.hpp"
+#include "query/v2/requests.hpp"
 #include "storage/v3/conversions.hpp"
+#include "storage/v3/property_value.hpp"
+#include "storage/v3/result.hpp"
+#include "storage/v3/view.hpp"
 
 namespace memgraph::query::v2 {
 
-inline const auto lam = [](const auto &val) { return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val); };
+class RequestRouterInterface;
 
-using ExpressionEvaluator =
-    memgraph::expr::ExpressionEvaluator<TypedValue, EvaluationContext, DbAccessor, storage::v3::View,
-                                        storage::v3::LabelId, storage::v3::PropertyStore, decltype(lam),
-                                        memgraph::storage::v3::Error>;
+namespace detail {
+class Callable {
+ public:
+  auto operator()(const storage::v3::PropertyValue &val) const {
+    return storage::v3::PropertyToTypedValue<TypedValue>(val);
+  };
+  auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
+    return ValueToTypedValue(val, request_router);
+  };
+};
+
+}  // namespace detail
+using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
+                                                      storage::v3::View, storage::v3::LabelId, msgs::Value,
+                                                      detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
 
 }  // namespace memgraph::query::v2

src/query/v2/bindings/frame.hpp

@@ -13,9 +13,10 @@
 
 #include "query/v2/bindings/bindings.hpp"
 
-#include "query/v2/bindings/typed_value.hpp"
 #include "expr/interpret/frame.hpp"
+#include "query/v2/bindings/typed_value.hpp"
 
 namespace memgraph::query::v2 {
-using Frame = memgraph::expr::Frame<TypedValue>;
-} // namespace memgraph::query::v2
+using Frame = memgraph::expr::Frame;
+using FrameWithValidity = memgraph::expr::FrameWithValidity;
+}  // namespace memgraph::query::v2

src/query/v2/bindings/typed_value.cpp

@@ -10,10 +10,10 @@
 // licenses/APL.txt.
 
 #include "expr/typed_value.hpp"
-#include "query/v2/db_accessor.hpp"
+#include "query/v2/accessors.hpp"
 #include "query/v2/path.hpp"
 
 namespace memgraph::expr {
 namespace v2 = memgraph::query::v2;
-template class TypedValueT<v2::VertexAccessor, v2::EdgeAccessor, v2::Path>;
+template class TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor, v2::accessors::Path>;
 }  // namespace memgraph::expr

src/query/v2/bindings/typed_value.hpp

@@ -14,13 +14,14 @@
 #include "query/v2/bindings/bindings.hpp"
 
 #include "expr/typed_value.hpp"
-#include "query/v2/db_accessor.hpp"
-#include "query/v2/path.hpp"
+#include "query/v2/accessors.hpp"
 
 namespace memgraph::expr {
 namespace v2 = memgraph::query::v2;
-extern template class memgraph::expr::TypedValueT<v2::VertexAccessor, v2::EdgeAccessor, v2::Path>;
+extern template class memgraph::expr::TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor,
+                                                  v2::accessors::Path>;
 }  // namespace memgraph::expr
 namespace memgraph::query::v2 {
-using TypedValue = memgraph::expr::TypedValueT<VertexAccessor, EdgeAccessor, Path>;
+using TypedValue =
+    memgraph::expr::TypedValueT<v2::accessors::VertexAccessor, v2::accessors::EdgeAccessor, v2::accessors::Path>;
 }  // namespace memgraph::query::v2

src/query/v2/common.hpp

@@ -20,7 +20,6 @@
 
 #include "query/v2/bindings/symbol.hpp"
 #include "query/v2/bindings/typed_value.hpp"
-#include "query/v2/db_accessor.hpp"
 #include "query/v2/exceptions.hpp"
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/path.hpp"
@@ -28,7 +27,6 @@
 #include "storage/v3/id_types.hpp"
 #include "storage/v3/property_value.hpp"
 #include "storage/v3/result.hpp"
-#include "storage/v3/schema_validator.hpp"
 #include "storage/v3/view.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
@@ -82,102 +80,5 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
     throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
 }
 
-template <typename T>
-concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
-                                           const storage::v3::PropertyValue new_value) {
-  { accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
-};
-
-template <typename T>
-concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
-                                                      const storage::v3::PropertyValue new_value) {
-  {
-    accessor.SetPropertyAndValidate(key, new_value)
-    } -> std::same_as<storage::v3::ResultSchema<storage::v3::PropertyValue>>;
-};
-
-template <typename TRecordAccessor>
-concept RecordAccessor =
-    AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
-
-inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
-  switch (schema_violation.status) {
-    case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
-      throw SchemaViolationException(
-          fmt::format("Primary key {} not defined on label :{}",
-                      storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type),
-                      dba.LabelToName(schema_violation.label)));
-    }
-    case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: {
-      throw SchemaViolationException(
-          fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label)));
-    }
-    case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: {
-      throw SchemaViolationException(
-          fmt::format("Wrong type of property {} in schema :{}, should be of type {}",
-                      *schema_violation.violated_property_value, dba.LabelToName(schema_violation.label),
-                      storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type)));
-    }
-    case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: {
-      throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported",
-                                                 *schema_violation.violated_property_value,
-                                                 dba.LabelToName(schema_violation.label)));
-    }
-    case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: {
-      throw SchemaViolationException(fmt::format(
-          "Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value,
-          dba.LabelToName(schema_violation.label)));
-    }
-    case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
-      throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}",
-                                                 dba.LabelToName(schema_violation.label)));
-    }
-  }
-}
-
-inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
-  switch (error) {
-    case storage::v3::Error::SERIALIZATION_ERROR:
-      throw TransactionSerializationException();
-    case storage::v3::Error::DELETED_OBJECT:
-      throw QueryRuntimeException("Trying to set properties on a deleted object.");
-    case storage::v3::Error::PROPERTIES_DISABLED:
-      throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
-    case storage::v3::Error::VERTEX_HAS_EDGES:
-    case storage::v3::Error::NONEXISTENT_OBJECT:
-      throw QueryRuntimeException("Unexpected error when setting a property.");
-  }
-}
-
-/// Set a property `value` mapped with given `key` on a `record`.
-///
-/// @throw QueryRuntimeException if value cannot be set as a property value
-template <RecordAccessor T>
-storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
-                                           const TypedValue &value) {
-  try {
-    if constexpr (std::is_same_v<T, VertexAccessor>) {
-      const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::TypedToPropertyValue(value));
-      if (maybe_old_value.HasError()) {
-        std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); },
-                                     [&dba](const storage::v3::SchemaViolation &schema_violation) {
-                                       HandleSchemaViolation(schema_violation, dba);
-                                     }},
-                   maybe_old_value.GetError());
-      }
-      return std::move(*maybe_old_value);
-    } else {
-      // No validation on edge properties
-      const auto maybe_old_value = record->SetProperty(key, storage::v3::TypedToPropertyValue(value));
-      if (maybe_old_value.HasError()) {
-        HandleErrorOnPropertyUpdate(maybe_old_value.GetError());
-      }
-      return std::move(*maybe_old_value);
-    }
-  } catch (const expr::TypedValueException &) {
-    throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
-  }
-}
-
 int64_t QueryTimestamp();
 }  // namespace memgraph::query::v2

src/query/v2/context.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,16 +13,35 @@
 
 #include <type_traits>
 
+#include "io/local_transport/local_transport.hpp"
 #include "query/v2/bindings/symbol_table.hpp"
 #include "query/v2/common.hpp"
 #include "query/v2/metadata.hpp"
 #include "query/v2/parameters.hpp"
 #include "query/v2/plan/profile.hpp"
-#include "query/v2/trigger.hpp"
+//#include "query/v2/trigger.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/async_timer.hpp"
 
 namespace memgraph::query::v2 {
 
+// Used to store range of ids that are available
+// Used for edge id assignment
+class IdAllocator {
+ public:
+  IdAllocator() = default;
+  IdAllocator(uint64_t low, uint64_t high) : current_edge_id_{low}, max_id_{high} {};
+
+  uint64_t AllocateId() {
+    MG_ASSERT(current_edge_id_ < max_id_, "Current Edge Id went above max id");
+    return current_edge_id_++;
+  }
+
+ private:
+  uint64_t current_edge_id_;
+  uint64_t max_id_;
+};
+
 struct EvaluationContext {
   /// Memory for allocations during evaluation of a *single* Pull call.
   ///
@@ -42,20 +61,27 @@ struct EvaluationContext {
 };
 
 inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
-                                                              DbAccessor *dba) {
+                                                              RequestRouterInterface *request_router) {
   std::vector<storage::v3::PropertyId> properties;
+  // TODO Fix by using reference
   properties.reserve(property_names.size());
-  for (const auto &name : property_names) {
-    properties.push_back(dba->NameToProperty(name));
+  if (request_router != nullptr) {
+    for (const auto &name : property_names) {
+      properties.push_back(request_router->NameToProperty(name));
+    }
   }
   return properties;
 }
 
-inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
+inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
+                                                       RequestRouterInterface *request_router) {
   std::vector<storage::v3::LabelId> labels;
   labels.reserve(label_names.size());
-  for (const auto &name : label_names) {
-    labels.push_back(dba->NameToLabel(name));
+  // TODO Fix by using reference
+  if (request_router != nullptr) {
+    for (const auto &name : label_names) {
+      labels.push_back(request_router->NameToLabel(name));
+    }
   }
   return labels;
 }
@@ -70,8 +96,9 @@ struct ExecutionContext {
   plan::ProfilingStats stats;
   plan::ProfilingStats *stats_root{nullptr};
   ExecutionStats execution_stats;
-  TriggerContextCollector *trigger_context_collector{nullptr};
   utils::AsyncTimer timer;
+  RequestRouterInterface *request_router{nullptr};
+  IdAllocator *edge_ids_alloc;
 };
 
 static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");

src/query/v2/conversions.hpp

@@ -0,0 +1,107 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+#include "bindings/typed_value.hpp"
+#include "query/v2/accessors.hpp"
+#include "query/v2/request_router.hpp"
+#include "query/v2/requests.hpp"
+
+namespace memgraph::query::v2 {
+
+inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
+  using Value = msgs::Value;
+  switch (value.type) {
+    case Value::Type::Null:
+      return {};
+    case Value::Type::Bool:
+      return TypedValue(value.bool_v);
+    case Value::Type::Int64:
+      return TypedValue(value.int_v);
+    case Value::Type::Double:
+      return TypedValue(value.double_v);
+    case Value::Type::String:
+      return TypedValue(value.string_v);
+    case Value::Type::List: {
+      const auto &lst = value.list_v;
+      std::vector<TypedValue> dst;
+      dst.reserve(lst.size());
+      for (const auto &elem : lst) {
+        dst.push_back(ValueToTypedValue(elem, request_router));
+      }
+      return TypedValue(std::move(dst));
+    }
+    case Value::Type::Map: {
+      const auto &value_map = value.map_v;
+      std::map<std::string, TypedValue> dst;
+      for (const auto &[key, val] : value_map) {
+        dst[key] = ValueToTypedValue(val, request_router);
+      }
+      return TypedValue(std::move(dst));
+    }
+    case Value::Type::Vertex:
+      return TypedValue(accessors::VertexAccessor(
+          value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
+    case Value::Type::Edge:
+      return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
+  }
+  throw std::runtime_error("Incorrect type in conversion");
+}
+
+inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
+  return ValueToTypedValue(value, request_router);
+};
+
+inline msgs::Value TypedValueToValue(const TypedValue &value) {
+  using Value = msgs::Value;
+  switch (value.type()) {
+    case TypedValue::Type::Null:
+      return {};
+    case TypedValue::Type::Bool:
+      return Value(value.ValueBool());
+    case TypedValue::Type::Int:
+      return Value(value.ValueInt());
+    case TypedValue::Type::Double:
+      return Value(value.ValueDouble());
+    case TypedValue::Type::String:
+      return Value(std::string(value.ValueString()));
+    case TypedValue::Type::List: {
+      const auto &lst = value.ValueList();
+      std::vector<Value> dst;
+      dst.reserve(lst.size());
+      for (const auto &elem : lst) {
+        dst.push_back(TypedValueToValue(elem));
+      }
+      return Value(std::move(dst));
+    }
+    case TypedValue::Type::Map: {
+      const auto &value_map = value.ValueMap();
+      std::map<std::string, Value> dst;
+      for (const auto &[key, val] : value_map) {
+        dst[std::string(key)] = TypedValueToValue(val);
+      }
+      return Value(std::move(dst));
+    }
+    case TypedValue::Type::Vertex:
+      return Value(value.ValueVertex().GetVertex());
+    case TypedValue::Type::Edge:
+      return Value(value.ValueEdge().GetEdge());
+    case TypedValue::Type::Path:
+    case TypedValue::Type::LocalTime:
+    case TypedValue::Type::LocalDateTime:
+    case TypedValue::Type::Date:
+    case TypedValue::Type::Duration:
+      break;
+  }
+  throw std::runtime_error("Incorrect type in conversion");
+}
+
+}  // namespace memgraph::query::v2

src/query/v2/cypher_query_interpreter.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,11 +11,12 @@
 
 #include "query/v2/cypher_query_interpreter.hpp"
 #include "query/v2/bindings/symbol_generator.hpp"
+#include "query/v2/request_router.hpp"
 
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
+DEFINE_HIDDEN_bool(query_v2_cost_planner, true, "Use the cost-estimating query planner.");
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
+DEFINE_VALIDATED_int32(query_v2_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
                        FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
 
 namespace memgraph::query::v2 {
@@ -117,19 +118,19 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
 }
 
 std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
-                                             DbAccessor *db_accessor,
+                                             RequestRouterInterface *request_router,
                                              const std::vector<Identifier *> &predefined_identifiers) {
-  auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
+  auto vertex_counts = plan::MakeVertexCountCache(request_router);
   auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
   auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
-  auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
+  auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_v2_cost_planner);
   return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
                                                  std::move(symbol_table));
 }
 
 std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
                                               const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
-                                              DbAccessor *db_accessor,
+                                              RequestRouterInterface *request_router,
                                               const std::vector<Identifier *> &predefined_identifiers) {
   std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
   if (plan_cache) {
@@ -145,7 +146,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
   }
 
   auto plan = std::make_shared<CachedPlan>(
-      MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
+      MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
   if (plan_cache_access) {
     plan_cache_access->insert({hash, plan});
   }

src/query/v2/cypher_query_interpreter.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,9 +22,9 @@
 #include "utils/timer.hpp"
 
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_bool(query_cost_planner);
+DECLARE_bool(query_v2_cost_planner);
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_int32(query_plan_cache_ttl);
+DECLARE_int32(query_v2_plan_cache_ttl);
 
 namespace memgraph::query::v2 {
 
@@ -58,7 +58,7 @@ class CachedPlan {
 
   bool IsExpired() const {
     // NOLINTNEXTLINE (modernize-use-nullptr)
-    return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
+    return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_v2_plan_cache_ttl);
   };
 
  private:
@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
 };
 
 std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
-                                             DbAccessor *db_accessor,
+                                             RequestRouterInterface *request_router,
                                              const std::vector<Identifier *> &predefined_identifiers);
 
 /**
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
  */
 std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
                                               const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
-                                              DbAccessor *db_accessor,
+                                              RequestRouterInterface *request_router,
                                               const std::vector<Identifier *> &predefined_identifiers = {});
 
 }  // namespace memgraph::query::v2

src/query/v2/db_accessor.hpp

@@ -36,7 +36,7 @@
 // This cannot be avoided by simple include orderings so we
 // simply undefine those macros as we're sure that libkrb5
 // won't and can't be used anywhere in the query engine.
-#include "storage/v3/storage.hpp"
+#include "storage/v3/shard.hpp"
 #include "utils/logging.hpp"
 #include "utils/result.hpp"
 
@@ -64,24 +64,11 @@ class EdgeAccessor final {
 
   auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
 
-  storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
-                                                              storage::v3::PropertyId key) const {
+  storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
+                                                                   storage::v3::PropertyId key) const {
     return impl_.GetProperty(key, view);
   }
 
-  storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
-                                                              const storage::v3::PropertyValue &value) {
-    return impl_.SetProperty(key, value);
-  }
-
-  storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
-    return SetProperty(key, storage::v3::PropertyValue());
-  }
-
-  storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
-    return impl_.ClearProperties();
-  }
-
   VertexAccessor To() const;
 
   VertexAccessor From() const;
@@ -113,51 +100,19 @@ class VertexAccessor final {
 
   auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
 
-  storage::v3::ResultSchema<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabelAndValidate(label); }
-
-  storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
-    return impl_.AddLabelAndValidate(label);
-  }
-
-  storage::v3::ResultSchema<bool> RemoveLabel(storage::v3::LabelId label) {
-    return impl_.RemoveLabelAndValidate(label);
-  }
-
-  storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
-    return impl_.RemoveLabelAndValidate(label);
-  }
-
-  storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
+  storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
     return impl_.HasLabel(label, view);
   }
 
   auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
 
-  storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
-                                                              storage::v3::PropertyId key) const {
+  storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
+                                                                   storage::v3::PropertyId key) const {
     return impl_.GetProperty(key, view);
   }
 
-  storage::v3::ResultSchema<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
-                                                                    const storage::v3::PropertyValue &value) {
-    return impl_.SetPropertyAndValidate(key, value);
-  }
-
-  storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
-      storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
-    return impl_.SetPropertyAndValidate(key, value);
-  }
-
-  storage::v3::ResultSchema<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
-    return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
-  }
-
-  storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
-    return impl_.ClearProperties();
-  }
-
   auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
     auto maybe_edges = impl_.InEdges(view, edge_types);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -167,7 +122,7 @@ class VertexAccessor final {
 
   auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
                const VertexAccessor &dest) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
     const auto dest_id = dest.impl_.Id(view).GetValue();
     auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
@@ -175,7 +130,7 @@ class VertexAccessor final {
   }
 
   auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
     auto maybe_edges = impl_.OutEdges(view, edge_types);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -185,16 +140,16 @@ class VertexAccessor final {
 
   auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
                 const VertexAccessor &dest) const
-      -> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
+      -> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
     const auto dest_id = dest.impl_.Id(view).GetValue();
     auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
     if (maybe_edges.HasError()) return maybe_edges.GetError();
     return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
   }
 
-  storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
+  storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
 
-  storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
+  storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
 
   // TODO(jbajic) Fix Remove Gid
   static int64_t CypherId() { return 1; }
@@ -219,203 +174,7 @@ inline VertexAccessor EdgeAccessor::From() const { return *static_cast<VertexAcc
 inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
 
 class DbAccessor final {
-  storage::v3::Shard::Accessor *accessor_;
-
-  class VerticesIterable final {
-    storage::v3::VerticesIterable iterable_;
-
-   public:
-    class Iterator final {
-      storage::v3::VerticesIterable::Iterator it_;
-
-     public:
-      explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {}
-
-      VertexAccessor operator*() const { return VertexAccessor(*it_); }
-
-      Iterator &operator++() {
-        ++it_;
-        return *this;
-      }
-
-      bool operator==(const Iterator &other) const { return it_ == other.it_; }
-
-      bool operator!=(const Iterator &other) const { return !(other == *this); }
-    };
-
-    explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
-
-    Iterator begin() { return Iterator(iterable_.begin()); }
-
-    Iterator end() { return Iterator(iterable_.end()); }
-  };
-
  public:
-  explicit DbAccessor(storage::v3::Shard::Accessor *accessor) : accessor_(accessor) {}
-
-  // TODO(jbajic) Fix Remove Gid
-  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  std::optional<VertexAccessor> FindVertex(uint64_t /*unused*/) { return std::nullopt; }
-
-  std::optional<VertexAccessor> FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) {
-    auto maybe_vertex = accessor_->FindVertex(primary_key, view);
-    if (maybe_vertex) return VertexAccessor(*maybe_vertex);
-    return std::nullopt;
-  }
-
-  void FinalizeTransaction() { accessor_->FinalizeTransaction(); }
-
-  VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); }
-
-  VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) {
-    return VerticesIterable(accessor_->Vertices(label, view));
-  }
-
-  VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) {
-    return VerticesIterable(accessor_->Vertices(label, property, view));
-  }
-
-  VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
-                            const storage::v3::PropertyValue &value) {
-    return VerticesIterable(accessor_->Vertices(label, property, value, view));
-  }
-
-  VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
-                            const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
-                            const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) {
-    return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
-  }
-
-  storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
-      const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
-      const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
-    auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties);
-    if (maybe_vertex_acc.HasError()) {
-      return {std::move(maybe_vertex_acc.GetError())};
-    }
-    return VertexAccessor{maybe_vertex_acc.GetValue()};
-  }
-
-  storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
-                                               const storage::v3::EdgeTypeId &edge_type) {
-    static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
-    auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(),
-                                            to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
-    if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
-    return EdgeAccessor(*maybe_edge);
-  }
-
-  storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
-    auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid());
-    if (res.HasError()) {
-      return res.GetError();
-    }
-
-    const auto &value = res.GetValue();
-    if (!value) {
-      return std::optional<EdgeAccessor>{};
-    }
-
-    return std::make_optional<EdgeAccessor>(*value);
-  }
-
-  storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
-      VertexAccessor *vertex_accessor) {
-    using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
-
-    auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
-    if (res.HasError()) {
-      return res.GetError();
-    }
-
-    const auto &value = res.GetValue();
-    if (!value) {
-      return std::optional<ReturnType>{};
-    }
-
-    const auto &[vertex, edges] = *value;
-
-    std::vector<EdgeAccessor> deleted_edges;
-    deleted_edges.reserve(edges.size());
-    std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
-                   [](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
-
-    return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
-  }
-
-  storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
-    auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
-    if (res.HasError()) {
-      return res.GetError();
-    }
-
-    const auto &value = res.GetValue();
-    if (!value) {
-      return std::optional<VertexAccessor>{};
-    }
-
-    return {std::make_optional<VertexAccessor>(*value)};
-  }
-
-  // TODO(jbajic) Query engine should have a map of labels, properties and edge
-  // types
-  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  storage::v3::PropertyId NameToProperty(const std::string_view /*name*/) {
-    return storage::v3::PropertyId::FromUint(0);
-  }
-
-  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  storage::v3::LabelId NameToLabel(const std::string_view /*name*/) { return storage::v3::LabelId::FromUint(0); }
-
-  // NOLINTNEXTLINE(readability-convert-member-functions-to-static)
-  storage::v3::EdgeTypeId NameToEdgeType(const std::string_view /*name*/) {
-    return storage::v3::EdgeTypeId::FromUint(0);
-  }
-
-  const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }
-
-  const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); }
-
-  const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
-
-  void AdvanceCommand() { accessor_->AdvanceCommand(); }
-
-  utils::BasicResult<storage::v3::ConstraintViolation, void> Commit() { return accessor_->Commit(); }
-
-  void Abort() { accessor_->Abort(); }
-
-  bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); }
-
-  bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const {
-    return accessor_->LabelPropertyIndexExists(label, prop);
-  }
-
-  int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
-
-  int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
-
-  int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const {
-    return accessor_->ApproximateVertexCount(label, property);
-  }
-
-  int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
-                        const storage::v3::PropertyValue &value) const {
-    return accessor_->ApproximateVertexCount(label, property, value);
-  }
-
-  int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
-                        const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
-                        const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) const {
-    return accessor_->ApproximateVertexCount(label, property, lower, upper);
-  }
-
-  storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
-
-  storage::v3::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); }
-
-  const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); }
-
-  storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); }
 };
 
 }  // namespace memgraph::query::v2

src/query/v2/dump.cpp

@@ -1,541 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "query/v2/dump.hpp"
-
-#include <iomanip>
-#include <limits>
-#include <map>
-#include <optional>
-#include <ostream>
-#include <utility>
-#include <vector>
-
-#include <fmt/format.h>
-
-#include "query/v2/bindings/typed_value.hpp"
-#include "query/v2/db_accessor.hpp"
-#include "query/v2/exceptions.hpp"
-#include "query/v2/stream.hpp"
-#include "storage/v3/property_value.hpp"
-#include "storage/v3/storage.hpp"
-#include "utils/algorithm.hpp"
-#include "utils/logging.hpp"
-#include "utils/string.hpp"
-#include "utils/temporal.hpp"
-
-namespace memgraph::query::v2 {
-
-namespace {
-
-// Property that is used to make a difference among vertices. It is added to
-// property set of vertices to match edges and removed after the entire graph
-// is built.
-const char *kInternalPropertyId = "__mg_id__";
-
-// Label that is attached to each vertex and is used for easier creation of
-// index on internal property id.
-const char *kInternalVertexLabel = "__mg_vertex__";
-
-/// A helper function that escapes label, edge type and property names.
-std::string EscapeName(const std::string_view value) {
-  std::string out;
-  out.reserve(value.size() + 2);
-  out.append(1, '`');
-  for (auto c : value) {
-    if (c == '`') {
-      out.append("``");
-    } else {
-      out.append(1, c);
-    }
-  }
-  out.append(1, '`');
-  return out;
-}
-
-void DumpPreciseDouble(std::ostream *os, double value) {
-  // A temporary stream is used to keep precision of the original output
-  // stream unchanged.
-  std::ostringstream temp_oss;
-  temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10) << value;
-  *os << temp_oss.str();
-}
-
-namespace {
-void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) {
-  utils::Date date(value.microseconds);
-  os << "DATE(\"" << date << "\")";
-}
-
-void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) {
-  utils::LocalTime lt(value.microseconds);
-  os << "LOCALTIME(\"" << lt << "\")";
-}
-
-void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) {
-  utils::LocalDateTime ldt(value.microseconds);
-  os << "LOCALDATETIME(\"" << ldt << "\")";
-}
-
-void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) {
-  utils::Duration dur(value.microseconds);
-  os << "DURATION(\"" << dur << "\")";
-}
-
-void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) {
-  switch (value.type) {
-    case storage::v3::TemporalType::Date: {
-      DumpDate(os, value);
-      return;
-    }
-    case storage::v3::TemporalType::LocalTime: {
-      DumpLocalTime(os, value);
-      return;
-    }
-    case storage::v3::TemporalType::LocalDateTime: {
-      DumpLocalDateTime(os, value);
-      return;
-    }
-    case storage::v3::TemporalType::Duration: {
-      DumpDuration(os, value);
-      return;
-    }
-  }
-}
-}  // namespace
-
-void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) {
-  switch (value.type()) {
-    case storage::v3::PropertyValue::Type::Null:
-      *os << "Null";
-      return;
-    case storage::v3::PropertyValue::Type::Bool:
-      *os << (value.ValueBool() ? "true" : "false");
-      return;
-    case storage::v3::PropertyValue::Type::String:
-      *os << utils::Escape(value.ValueString());
-      return;
-    case storage::v3::PropertyValue::Type::Int:
-      *os << value.ValueInt();
-      return;
-    case storage::v3::PropertyValue::Type::Double:
-      DumpPreciseDouble(os, value.ValueDouble());
-      return;
-    case storage::v3::PropertyValue::Type::List: {
-      *os << "[";
-      const auto &list = value.ValueList();
-      utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); });
-      *os << "]";
-      return;
-    }
-    case storage::v3::PropertyValue::Type::Map: {
-      *os << "{";
-      const auto &map = value.ValueMap();
-      utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) {
-        os << EscapeName(kv.first) << ": ";
-        DumpPropertyValue(&os, kv.second);
-      });
-      *os << "}";
-      return;
-    }
-    case storage::v3::PropertyValue::Type::TemporalData: {
-      DumpTemporalData(*os, value.ValueTemporalData());
-      return;
-    }
-  }
-}
-
-void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba,
-                    const std::map<storage::v3::PropertyId, storage::v3::PropertyValue> &store,
-                    std::optional<int64_t> property_id = std::nullopt) {
-  *os << "{";
-  if (property_id) {
-    *os << kInternalPropertyId << ": " << *property_id;
-    if (store.size() > 0) *os << ", ";
-  }
-  utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
-    os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
-    DumpPropertyValue(&os, kv.second);
-  });
-  *os << "}";
-}
-
-void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) {
-  *os << "CREATE (";
-  *os << ":" << kInternalVertexLabel;
-  auto maybe_labels = vertex.Labels(storage::v3::View::OLD);
-  if (maybe_labels.HasError()) {
-    switch (maybe_labels.GetError()) {
-      case storage::v3::Error::DELETED_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node.");
-      case storage::v3::Error::NONEXISTENT_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
-      case storage::v3::Error::SERIALIZATION_ERROR:
-      case storage::v3::Error::VERTEX_HAS_EDGES:
-      case storage::v3::Error::PROPERTIES_DISABLED:
-        throw query::v2::QueryRuntimeException("Unexpected error when getting labels.");
-    }
-  }
-  for (const auto &label : *maybe_labels) {
-    *os << ":" << EscapeName(dba->LabelToName(label));
-  }
-  *os << " ";
-  auto maybe_props = vertex.Properties(storage::v3::View::OLD);
-  if (maybe_props.HasError()) {
-    switch (maybe_props.GetError()) {
-      case storage::v3::Error::DELETED_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
-      case storage::v3::Error::NONEXISTENT_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist.");
-      case storage::v3::Error::SERIALIZATION_ERROR:
-      case storage::v3::Error::VERTEX_HAS_EDGES:
-      case storage::v3::Error::PROPERTIES_DISABLED:
-        throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
-    }
-  }
-  DumpProperties(os, dba, *maybe_props, vertex.CypherId());
-  *os << ");";
-}
-
-void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) {
-  *os << "MATCH ";
-  *os << "(u:" << kInternalVertexLabel << "), ";
-  *os << "(v:" << kInternalVertexLabel << ")";
-  *os << " WHERE ";
-  *os << "u." << kInternalPropertyId << " = " << edge.From().CypherId();
-  *os << " AND ";
-  *os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " ";
-  *os << "CREATE (u)-[";
-  *os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType()));
-  auto maybe_props = edge.Properties(storage::v3::View::OLD);
-  if (maybe_props.HasError()) {
-    switch (maybe_props.GetError()) {
-      case storage::v3::Error::DELETED_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
-      case storage::v3::Error::NONEXISTENT_OBJECT:
-        throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist.");
-      case storage::v3::Error::SERIALIZATION_ERROR:
-      case storage::v3::Error::VERTEX_HAS_EDGES:
-      case storage::v3::Error::PROPERTIES_DISABLED:
-        throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
-    }
-  }
-  if (maybe_props->size() > 0) {
-    *os << " ";
-    DumpProperties(os, dba, *maybe_props);
-  }
-  *os << "]->(v);";
-}
-
-void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) {
-  *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";";
-}
-
-void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
-                            storage::v3::PropertyId property) {
-  *os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property))
-      << ");";
-}
-
-void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
-                             storage::v3::PropertyId property) {
-  *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u."
-      << EscapeName(dba->PropertyToName(property)) << ");";
-}
-
-void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
-                          const std::set<storage::v3::PropertyId> &properties) {
-  *os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT ";
-  utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) {
-    stream << "u." << EscapeName(dba->PropertyToName(property));
-  });
-  *os << " IS UNIQUE;";
-}
-
-}  // namespace
-
-PullPlanDump::PullPlanDump(DbAccessor *dba)
-    : dba_(dba),
-      vertices_iterable_(dba->Vertices(storage::v3::View::OLD)),
-      pull_chunks_{// Dump all label indices
-                   CreateLabelIndicesPullChunk(),
-                   // Dump all label property indices
-                   CreateLabelPropertyIndicesPullChunk(),
-                   // Dump all existence constraints
-                   CreateExistenceConstraintsPullChunk(),
-                   // Dump all unique constraints
-                   CreateUniqueConstraintsPullChunk(),
-                   // Create internal index for faster edge creation
-                   CreateInternalIndexPullChunk(),
-                   // Dump all vertices
-                   CreateVertexPullChunk(),
-                   // Dump all edges
-                   CreateEdgePullChunk(),
-                   // Drop the internal index
-                   CreateDropInternalIndexPullChunk(),
-                   // Internal index cleanup
-                   CreateInternalIndexCleanupPullChunk()} {}
-
-bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
-  // Iterate all functions that stream some results.
-  // Each function should return number of results it streamed after it
-  // finishes. If the function did not finish streaming all the results,
-  // std::nullopt should be returned because n results have already been sent.
-  while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) {
-    const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n);
-
-    if (!maybe_streamed_count) {
-      // n wasn't large enough to stream all the results from the current chunk
-      break;
-    }
-
-    if (n) {
-      // chunk finished streaming its results
-      // subtract number of results streamed in current pull
-      // so we know how many results we need to stream from future
-      // chunks.
-      *n -= *maybe_streamed_count;
-    }
-
-    ++current_chunk_index_;
-  }
-  return current_chunk_index_ == pull_chunks_.size();
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
-  // Dump all label indices
-  return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the construction of indices vectors
-    if (!indices_info_) {
-      indices_info_.emplace(dba_->ListAllIndices());
-    }
-    const auto &label = indices_info_->label;
-
-    size_t local_counter = 0;
-    while (global_index < label.size() && (!n || local_counter < *n)) {
-      std::ostringstream os;
-      DumpLabelIndex(&os, dba_, label[global_index]);
-      stream->Result({TypedValue(os.str())});
-
-      ++global_index;
-      ++local_counter;
-    }
-
-    if (global_index == label.size()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
-  return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the construction of indices vectors
-    if (!indices_info_) {
-      indices_info_.emplace(dba_->ListAllIndices());
-    }
-    const auto &label_property = indices_info_->label_property;
-
-    size_t local_counter = 0;
-    while (global_index < label_property.size() && (!n || local_counter < *n)) {
-      std::ostringstream os;
-      const auto &label_property_index = label_property[global_index];
-      DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second);
-      stream->Result({TypedValue(os.str())});
-
-      ++global_index;
-      ++local_counter;
-    }
-
-    if (global_index == label_property.size()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateExistenceConstraintsPullChunk() {
-  return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the construction of constraint vectors
-    if (!constraints_info_) {
-      constraints_info_.emplace(dba_->ListAllConstraints());
-    }
-
-    const auto &existence = constraints_info_->existence;
-    size_t local_counter = 0;
-    while (global_index < existence.size() && (!n || local_counter < *n)) {
-      const auto &constraint = existence[global_index];
-      std::ostringstream os;
-      DumpExistenceConstraint(&os, dba_, constraint.first, constraint.second);
-      stream->Result({TypedValue(os.str())});
-
-      ++global_index;
-      ++local_counter;
-    }
-
-    if (global_index == existence.size()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateUniqueConstraintsPullChunk() {
-  return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the construction of constraint vectors
-    if (!constraints_info_) {
-      constraints_info_.emplace(dba_->ListAllConstraints());
-    }
-
-    const auto &unique = constraints_info_->unique;
-    size_t local_counter = 0;
-    while (global_index < unique.size() && (!n || local_counter < *n)) {
-      const auto &constraint = unique[global_index];
-      std::ostringstream os;
-      DumpUniqueConstraint(&os, dba_, constraint.first, constraint.second);
-      stream->Result({TypedValue(os.str())});
-
-      ++global_index;
-      ++local_counter;
-    }
-
-    if (global_index == unique.size()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
-  return [this](AnyStream *stream, std::optional<int>) mutable -> std::optional<size_t> {
-    if (vertices_iterable_.begin() != vertices_iterable_.end()) {
-      std::ostringstream os;
-      os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
-      stream->Result({TypedValue(os.str())});
-      internal_index_created_ = true;
-      return 1;
-    }
-    return 0;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
-  return [this, maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
-             AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the call of begin() function
-    // If multiple begins are called before an iteration,
-    // one iteration will make the rest of iterators be in undefined
-    // states.
-    if (!maybe_current_iter) {
-      maybe_current_iter.emplace(vertices_iterable_.begin());
-    }
-
-    auto &current_iter{*maybe_current_iter};
-
-    size_t local_counter = 0;
-    while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) {
-      std::ostringstream os;
-      DumpVertex(&os, dba_, *current_iter);
-      stream->Result({TypedValue(os.str())});
-      ++local_counter;
-      ++current_iter;
-    }
-    if (current_iter == vertices_iterable_.end()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
-  return [this, maybe_current_vertex_iter = std::optional<VertexAccessorIterableIterator>{},
-          // we need to save the iterable which contains list of accessor so
-          // our saved iterator is valid in the next run
-          maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
-          maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
-             AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
-    // Delay the call of begin() function
-    // If multiple begins are called before an iteration,
-    // one iteration will make the rest of iterators be in undefined
-    // states.
-    if (!maybe_current_vertex_iter) {
-      maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
-    }
-
-    auto &current_vertex_iter{*maybe_current_vertex_iter};
-    size_t local_counter = 0U;
-    for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
-      const auto &vertex = *current_vertex_iter;
-      // If we have a saved iterable from a previous pull
-      // we need to use the same iterable
-      if (!maybe_edge_iterable) {
-        maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::v3::View::OLD));
-      }
-      auto &maybe_edges = *maybe_edge_iterable;
-      MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
-      auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin();
-      for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) {
-        std::ostringstream os;
-        DumpEdge(&os, dba_, *current_edge_iter);
-        stream->Result({TypedValue(os.str())});
-
-        ++local_counter;
-      }
-
-      if (current_edge_iter != maybe_edges->end()) {
-        maybe_current_edge_iter.emplace(current_edge_iter);
-        return std::nullopt;
-      }
-
-      maybe_current_edge_iter = std::nullopt;
-      maybe_edge_iterable = nullptr;
-    }
-
-    if (current_vertex_iter == vertices_iterable_.end()) {
-      return local_counter;
-    }
-
-    return std::nullopt;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() {
-  return [this](AnyStream *stream, std::optional<int>) {
-    if (internal_index_created_) {
-      std::ostringstream os;
-      os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
-      stream->Result({TypedValue(os.str())});
-      return 1;
-    }
-    return 0;
-  };
-}
-
-PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
-  return [this](AnyStream *stream, std::optional<int>) {
-    if (internal_index_created_) {
-      std::ostringstream os;
-      os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";";
-      stream->Result({TypedValue(os.str())});
-      return 1;
-    }
-    return 0;
-  };
-}
-
-void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); }
-
-}  // namespace memgraph::query::v2

src/query/v2/dump.hpp

@@ -1,66 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <ostream>
-
-#include "query/v2/db_accessor.hpp"
-#include "query/v2/stream.hpp"
-#include "storage/v3/storage.hpp"
-
-namespace memgraph::query::v2 {
-
-void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream);
-
-struct PullPlanDump {
-  explicit PullPlanDump(query::v2::DbAccessor *dba);
-
-  /// Pull the dump results lazily
-  /// @return true if all results were returned, false otherwise
-  bool Pull(AnyStream *stream, std::optional<int> n);
-
- private:
-  query::v2::DbAccessor *dba_ = nullptr;
-
-  std::optional<storage::v3::IndicesInfo> indices_info_ = std::nullopt;
-  std::optional<storage::v3::ConstraintsInfo> constraints_info_ = std::nullopt;
-
-  using VertexAccessorIterable = decltype(std::declval<query::v2::DbAccessor>().Vertices(storage::v3::View::OLD));
-  using VertexAccessorIterableIterator = decltype(std::declval<VertexAccessorIterable>().begin());
-
-  using EdgeAccessorIterable = decltype(std::declval<VertexAccessor>().OutEdges(storage::v3::View::OLD));
-  using EdgeAccessorIterableIterator = decltype(std::declval<EdgeAccessorIterable>().GetValue().begin());
-
-  VertexAccessorIterable vertices_iterable_;
-  bool internal_index_created_ = false;
-
-  size_t current_chunk_index_ = 0;
-
-  using PullChunk = std::function<std::optional<size_t>(AnyStream *stream, std::optional<int> n)>;
-  // We define every part of the dump query in a self contained function.
-  // Each functions is responsible of keeping track of its execution status.
-  // If a function did finish its execution, it should return number of results
-  // it streamed so we know how many rows should be pulled from the next
-  // function, otherwise std::nullopt is returned.
-  std::vector<PullChunk> pull_chunks_;
-
-  PullChunk CreateLabelIndicesPullChunk();
-  PullChunk CreateLabelPropertyIndicesPullChunk();
-  PullChunk CreateExistenceConstraintsPullChunk();
-  PullChunk CreateUniqueConstraintsPullChunk();
-  PullChunk CreateInternalIndexPullChunk();
-  PullChunk CreateVertexPullChunk();
-  PullChunk CreateEdgePullChunk();
-  PullChunk CreateDropInternalIndexPullChunk();
-  PullChunk CreateInternalIndexCleanupPullChunk();
-};
-}  // namespace memgraph::query::v2

src/query/v2/exceptions.hpp

@@ -224,12 +224,4 @@ class VersionInfoInMulticommandTxException : public QueryException {
       : QueryException("Version info query not allowed in multicommand transactions.") {}
 };
 
-/**
- * An exception for an illegal operation that violates schema
- */
-class SchemaViolationException : public QueryRuntimeException {
- public:
-  using QueryRuntimeException::QueryRuntimeException;
-};
-
 }  // namespace memgraph::query::v2

src/query/v2/frontend/ast/ast.lcp

@@ -18,12 +18,15 @@
 #include <vector>
 
 #include "query/v2/bindings/ast_visitor.hpp"
+#include "common/types.hpp"
 #include "query/v2/bindings/symbol.hpp"
-#include "query/v2/interpret/awesome_memgraph_functions.hpp"
+#include "functions/awesome_memgraph_functions.hpp"
 #include "query/v2/bindings/typed_value.hpp"
 #include "query/v2/db_accessor.hpp"
 #include "query/v2/path.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/typeinfo.hpp"
+#include "query/v2/conversions.hpp"
 
 cpp<#
 
@@ -835,13 +838,15 @@ cpp<#
               :slk-load (slk-load-ast-vector "Expression"))
    (function-name "std::string" :scope :public)
    (function "std::function<TypedValue(const TypedValue *, int64_t,
-                                       const FunctionContext &)>"
+                                       const functions::FunctionContext<RequestRouterInterface> &)>"
              :scope :public
              :dont-save t
              :clone :copy
              :slk-load (lambda (member)
                         #>cpp
-                        self->${member} = query::v2::NameToFunction(self->function_name_);
+                        self->${member} = functions::NameToFunction<TypedValue,
+                                            functions::FunctionContext<RequestRouterInterface>,
+                                            functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
                         cpp<#)))
   (:public
     #>cpp
@@ -864,7 +869,8 @@ cpp<#
              const std::vector<Expression *> &arguments)
         : arguments_(arguments),
           function_name_(function_name),
-          function_(NameToFunction(function_name_)) {
+          function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
+                                              functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
       if (!function_) {
         throw SemanticException("Function '{}' doesn't exist.", function_name);
       }

src/query/v2/frontend/semantic/required_privileges.cpp

@@ -11,7 +11,6 @@
 
 #include "query/v2/bindings/ast_visitor.hpp"
 #include "query/v2/frontend/ast/ast.hpp"
-#include "query/v2/procedure/module.hpp"
 #include "utils/memory.hpp"
 
 namespace memgraph::query::v2 {
@@ -86,12 +85,7 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
     AddPrivilege(AuthQuery::Privilege::CREATE);
     return false;
   }
-  bool PreVisit(CallProcedure &procedure) override {
-    const auto maybe_proc =
-        procedure::FindProcedure(procedure::gModuleRegistry, procedure.procedure_name_, utils::NewDeleteResource());
-    if (maybe_proc && maybe_proc->second->info.required_privilege) {
-      AddPrivilege(*maybe_proc->second->info.required_privilege);
-    }
+  bool PreVisit(CallProcedure & /*procedure*/) override {
     return false;
   }
   bool PreVisit(Delete & /*unused*/) override {

src/query/v2/interpret/awesome_memgraph_functions.hpp

@@ -16,12 +16,13 @@
 #include <unordered_map>
 
 #include "query/v2/bindings/typed_value.hpp"
+#include "query/v2/db_accessor.hpp"
 #include "storage/v3/view.hpp"
 #include "utils/memory.hpp"
 
 namespace memgraph::query::v2 {
 
-class DbAccessor;
+class RequestRouterInterface;
 
 namespace {
 const char kStartsWith[] = "STARTSWITH";
@@ -31,7 +32,9 @@ const char kId[] = "ID";
 }  // namespace
 
 struct FunctionContext {
-  DbAccessor *db_accessor;
+  // TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
+  // DbAccessor *db_accessor;
+  RequestRouterInterface *request_router;
   utils::MemoryResource *memory;
   int64_t timestamp;
   std::unordered_map<std::string, int64_t> *counters;

src/query/v2/interpreter.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,7 +12,11 @@
 #pragma once
 
 #include <gflags/gflags.h>
+#include <cstdint>
 
+#include "coordinator/coordinator.hpp"
+#include "coordinator/coordinator_client.hpp"
+#include "io/transport.hpp"
 #include "query/v2/auth_checker.hpp"
 #include "query/v2/bindings/cypher_main_visitor.hpp"
 #include "query/v2/bindings/typed_value.hpp"
@@ -26,9 +30,8 @@
 #include "query/v2/metadata.hpp"
 #include "query/v2/plan/operator.hpp"
 #include "query/v2/plan/read_write_type_checker.hpp"
+#include "query/v2/requests.hpp"
 #include "query/v2/stream.hpp"
-#include "query/v2/stream/streams.hpp"
-#include "query/v2/trigger.hpp"
 #include "storage/v3/isolation_level.hpp"
 #include "storage/v3/name_id_mapper.hpp"
 #include "utils/event_counter.hpp"
@@ -167,7 +170,9 @@ struct PreparedQuery {
  */
 struct InterpreterContext {
   explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
-                              const std::filesystem::path &data_directory);
+                              const std::filesystem::path &data_directory,
+                              std::unique_ptr<RequestRouterFactory> request_router_factory,
+                              coordinator::Address coordinator_addr);
 
   storage::v3::Shard *db;
 
@@ -180,28 +185,27 @@ struct InterpreterContext {
   utils::SkipList<QueryCacheEntry> ast_cache;
   utils::SkipList<PlanCacheEntry> plan_cache;
 
-  TriggerStore trigger_store;
-  utils::ThreadPool after_commit_trigger_pool{1};
-
   const InterpreterConfig config;
+  IdAllocator edge_ids_alloc;
 
-  query::v2::stream::Streams streams;
+  coordinator::Address coordinator_address;
+  std::unique_ptr<RequestRouterFactory> request_router_factory_;
 
   storage::v3::LabelId NameToLabelId(std::string_view label_name) {
-    return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
+    return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
   }
 
   storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
-    return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
+    return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
   }
 
   storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
-    return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
+    return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
   }
 
  private:
   // TODO Replace with local map of labels, properties and edge type ids
-  storage::v3::NameIdMapper query_id_mapper;
+  storage::v3::NameIdMapper query_id_mapper_;
 };
 
 /// Function that is used to tell all active interpreters that they should stop
@@ -290,11 +294,16 @@ class Interpreter final {
    */
   void Abort();
 
+  const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
+  void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
+    request_router_->InstallSimulatorTicker(tick_simulator);
+  }
+
  private:
   struct QueryExecution {
-    std::optional<PreparedQuery> prepared_query;
     utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
     utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
+    std::optional<PreparedQuery> prepared_query;
 
     std::map<std::string, TypedValue> summary;
     std::vector<Notification> notifications;
@@ -331,10 +340,10 @@ class Interpreter final {
 
   // This cannot be std::optional because we need to move this accessor later on into a lambda capture
   // which is assigned to std::function. std::function requires every object to be copyable, so we
-  // move this unique_ptr into a shrared_ptr.
+  // move this unique_ptr into a shared_ptr.
   std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
   std::optional<DbAccessor> execution_db_accessor_;
-  std::optional<TriggerContextCollector> trigger_context_collector_;
+  std::unique_ptr<RequestRouterInterface> request_router_;
   bool in_explicit_transaction_{false};
   bool expect_rollback_{false};
 

src/query/v2/multiframe.cpp

@@ -0,0 +1,153 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/v2/multiframe.hpp"
+
+#include <algorithm>
+#include <iterator>
+
+#include "query/v2/bindings/frame.hpp"
+#include "utils/pmr/vector.hpp"
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
+
+namespace memgraph::query::v2 {
+
+static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
+static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
+static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
+static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
+
+MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
+    : frames_(utils::pmr::vector<FrameWithValidity>(
+          number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
+  MG_ASSERT(number_of_frames > 0);
+}
+
+MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
+
+FrameWithValidity &MultiFrame::GetFirstFrame() {
+  MG_ASSERT(!frames_.empty());
+  return frames_.front();
+}
+
+void MultiFrame::MakeAllFramesInvalid() noexcept {
+  std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
+}
+
+bool MultiFrame::HasValidFrame() const noexcept {
+  return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
+}
+
+bool MultiFrame::HasInvalidFrame() const noexcept {
+  return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
+}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+void MultiFrame::DefragmentValidFrames() noexcept {
+  static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
+  static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
+  auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
+  auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
+  while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
+    std::swap(*first_invalid_frame, *following_first_valid);
+    first_invalid_frame++;
+    first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
+    following_first_valid++;
+    following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
+  }
+}
+
+ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
+
+ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
+
+ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
+
+InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
+
+ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
+  /*
+  From: https://en.cppreference.com/w/cpp/algorithm/find
+  Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
+  find_if searches for an element for which predicate p returns true
+  Return value
+    Iterator to the first element satisfying the condition or last if no such element is found.
+
+  -> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
+  */
+  auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
+                         [](const auto &frame) { return !frame.IsValid(); });
+  after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
+}
+
+ValidFramesReader::Iterator ValidFramesReader::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0]};
+  }
+  return end();
+}
+
+ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
+
+ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+ValidFramesModifier::Iterator ValidFramesModifier::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesModifier::Iterator ValidFramesModifier::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+// NOLINTNEXTLINE (bugprone-exception-escape)
+ValidFramesConsumer::~ValidFramesConsumer() noexcept {
+  // TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
+  // cost too much to store it
+  multiframe_->DefragmentValidFrames();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
+  if (multiframe_->frames_[0].IsValid()) {
+    return Iterator{&multiframe_->frames_[0], *this};
+  }
+  return end();
+}
+
+ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
+}
+
+InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
+  for (auto &frame : multiframe_->frames_) {
+    if (!frame.IsValid()) {
+      return Iterator{&frame};
+    }
+  }
+  return end();
+}
+
+InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
+  return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
+}
+
+}  // namespace memgraph::query::v2

src/query/v2/multiframe.hpp

@@ -0,0 +1,308 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <iterator>
+
+#include <gflags/gflags.h>
+
+#include "query/v2/bindings/frame.hpp"
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint64(default_multi_frame_size);
+
+namespace memgraph::query::v2 {
+
+class ValidFramesConsumer;
+class ValidFramesModifier;
+class ValidFramesReader;
+class InvalidFramesPopulator;
+
+class MultiFrame {
+ public:
+  friend class ValidFramesConsumer;
+  friend class ValidFramesModifier;
+  friend class ValidFramesReader;
+  friend class InvalidFramesPopulator;
+
+  MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
+  ~MultiFrame() = default;
+
+  // Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
+  MultiFrame(const MultiFrame &other);
+  MultiFrame(MultiFrame &&other) noexcept;
+  MultiFrame &operator=(const MultiFrame &other) = delete;
+  MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame nor its content with this implementation.
+   */
+  ValidFramesReader GetValidFramesReader();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can't modify the validity of the Frame with this implementation. One can modify its content.
+   */
+  ValidFramesModifier GetValidFramesModifier();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
+   * state in the MultiFrame.
+   * Iteration goes in a deterministic order.
+   * One can modify the validity of the Frame with this implementation.
+   * If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
+   * this is faster.
+   */
+  ValidFramesConsumer GetValidFramesConsumer();
+
+  /*
+   * Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
+   * invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
+   * the Frame with this implementation.
+   */
+  InvalidFramesPopulator GetInvalidFramesPopulator();
+
+  /**
+   * Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
+   * using the iterators instead.
+   * The Frame can be valid or invalid.
+   */
+  FrameWithValidity &GetFirstFrame();
+
+  void MakeAllFramesInvalid() noexcept;
+
+  bool HasValidFrame() const noexcept;
+  bool HasInvalidFrame() const noexcept;
+
+  inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
+
+ private:
+  void DefragmentValidFrames() noexcept;
+
+  utils::pmr::vector<FrameWithValidity> frames_;
+};
+
+class ValidFramesReader {
+ public:
+  explicit ValidFramesReader(MultiFrame &multiframe);
+
+  ~ValidFramesReader() = default;
+  ValidFramesReader(const ValidFramesReader &other) = delete;
+  ValidFramesReader(ValidFramesReader &&other) noexcept = default;
+  ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
+  ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = const Frame;
+    using pointer = value_type *;
+    using reference = const Frame &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ptr_++;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  FrameWithValidity *after_last_valid_frame_;
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesModifier {
+ public:
+  explicit ValidFramesModifier(MultiFrame &multiframe);
+
+  ~ValidFramesModifier() = default;
+  ValidFramesModifier(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
+  ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
+  ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = Frame;
+    using pointer = value_type *;
+    using reference = Frame &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    // Prefix increment
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesModifier *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class ValidFramesConsumer {
+ public:
+  explicit ValidFramesConsumer(MultiFrame &multiframe);
+
+  ~ValidFramesConsumer() noexcept;
+  ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
+  ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
+  ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
+        : ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      do {
+        ptr_++;
+      } while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
+
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++*this;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+    ValidFramesConsumer *iterator_wrapper_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+class InvalidFramesPopulator {
+ public:
+  explicit InvalidFramesPopulator(MultiFrame &multiframe);
+  ~InvalidFramesPopulator() = default;
+
+  InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
+  InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
+  InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
+
+  struct Iterator {
+    using iterator_category = std::forward_iterator_tag;
+    using difference_type = std::ptrdiff_t;
+    using value_type = FrameWithValidity;
+    using pointer = value_type *;
+    using reference = FrameWithValidity &;
+
+    Iterator() = default;
+    explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
+
+    reference operator*() const { return *ptr_; }
+    pointer operator->() { return ptr_; }
+
+    Iterator &operator++() {
+      ptr_->MakeValid();
+      ptr_++;
+      return *this;
+    }
+
+    // NOLINTNEXTLINE(cert-dcl21-cpp)
+    Iterator operator++(int) {
+      auto old = *this;
+      ++ptr_;
+      return old;
+    }
+
+    friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
+    friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
+
+   private:
+    FrameWithValidity *ptr_{nullptr};
+  };
+
+  Iterator begin();
+  Iterator end();
+
+ private:
+  MultiFrame *multiframe_;
+};
+
+}  // namespace memgraph::query::v2

src/query/v2/parameters.hpp

@@ -12,8 +12,8 @@
 #pragma once
 
 #include <algorithm>
+#include <unordered_map>
 #include <utility>
-#include <vector>
 
 #include "storage/v3/property_value.hpp"
 #include "utils/logging.hpp"
@@ -32,7 +32,7 @@ struct Parameters {
    * @param position Token position in query of value.
    * @param value
    */
-  void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace_back(position, value); }
+  void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace(position, value); }
 
   /**
    *  Returns the value found for the given token position.
@@ -41,23 +41,11 @@ struct Parameters {
    *  @return Value for the given token position.
    */
   const storage::v3::PropertyValue &AtTokenPosition(int position) const {
-    auto found = std::find_if(storage_.begin(), storage_.end(), [&](const auto &a) { return a.first == position; });
+    auto found = storage_.find(position);
     MG_ASSERT(found != storage_.end(), "Token position must be present in container");
     return found->second;
   }
 
-  /**
-   * Returns the position-th stripped value. Asserts that this
-   * container has at least (position + 1) elements.
-   *
-   * @param position Which stripped param is sought.
-   * @return Token position and value for sought param.
-   */
-  const std::pair<int, storage::v3::PropertyValue> &At(int position) const {
-    MG_ASSERT(position < static_cast<int>(storage_.size()), "Invalid position");
-    return storage_[position];
-  }
-
   /** Returns the number of arguments in this container */
   auto size() const { return storage_.size(); }
 
@@ -65,7 +53,7 @@ struct Parameters {
   auto end() const { return storage_.end(); }
 
  private:
-  std::vector<std::pair<int, storage::v3::PropertyValue>> storage_;
+  std::unordered_map<int, storage::v3::PropertyValue> storage_;
 };
 
 }  // namespace memgraph::query::v2

src/query/v2/path.hpp

@@ -14,6 +14,8 @@
 #include <functional>
 #include <utility>
 
+#include "query/db_accessor.hpp"
+#include "query/v2/accessors.hpp"
 #include "query/v2/db_accessor.hpp"
 #include "utils/logging.hpp"
 #include "utils/memory.hpp"
@@ -28,6 +30,8 @@ namespace memgraph::query::v2 {
  */
 class Path {
  public:
+  using VertexAccessor = accessors::VertexAccessor;
+  using EdgeAccessor = accessors::EdgeAccessor;
   /** Allocator type so that STL containers are aware that we need one */
   using allocator_type = utils::Allocator<char>;
 

src/query/v2/plan/cost_estimator.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -149,8 +149,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
     return true;
   }
 
-  // TODO: Cost estimate ScanAllById?
-
 // For the given op first increments the cardinality and then cost.
 #define POST_VISIT_CARD_FIRST(NAME)     \
   bool PostVisit(NAME &) override {     \

src/query/v2/plan/operator.lcp

@@ -22,18 +22,24 @@
 #include <variant>
 #include <vector>
 
+#include "expr/semantic/symbol.hpp"
 #include "query/v2/common.hpp"
 #include "query/v2/frontend/ast/ast.hpp"
-#include "expr/semantic/symbol.hpp"
-#include "query/v2//bindings/typed_value.hpp"
-#include "query/v2//bindings/frame.hpp"
-#include "query/v2//bindings/symbol_table.hpp"
+#include "query/v2/bindings/typed_value.hpp"
+#include "query/v2/bindings/frame.hpp"
+#include "query/v2/bindings/symbol_table.hpp"
+#include "query/v2/multiframe.hpp"
 #include "storage/v3/id_types.hpp"
 #include "utils/bound.hpp"
 #include "utils/fnv.hpp"
 #include "utils/memory.hpp"
 #include "utils/visitor.hpp"
 #include "utils/logging.hpp"
+#include "query/v2/accessors.hpp"
+
+using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
+using EdgeAccessor = memgraph::query::v2::accessors::EdgeAccessor;
+using Path = memgraph::query::v2::accessors::Path;
 cpp<#
 
 (lcp:namespace memgraph)
@@ -66,6 +72,24 @@ class Cursor {
   /// @throws QueryRuntimeException if something went wrong with execution
   virtual bool Pull(Frame &, ExecutionContext &) = 0;
 
+  /// Run an iteration of a @c LogicalOperator with MultiFrame.
+  ///
+  /// Since operators may be chained, the iteration may pull results from
+  /// multiple operators.
+  ///
+  /// @param MultiFrame May be read from or written to while performing the
+  ///     iteration.
+  /// @param ExecutionContext Used to get the position of symbols in frame and
+  ///     other information.
+  /// @return True if the operator was able to populate at least one Frame on the MultiFrame,
+  ///     thus if an operator returns true, that means there is at least one valid Frame in the
+  ///     MultiFrame.
+  ///
+  /// @throws QueryRuntimeException if something went wrong with execution
+  virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
+
+  virtual void PushDown(const MultiFrame&) { MG_ASSERT(false, "PushDownIsNotImplemented"); }
+
   /// Resets the Cursor to its initial state.
   virtual void Reset() = 0;
 
@@ -105,7 +129,7 @@ class ScanAllByLabel;
 class ScanAllByLabelPropertyRange;
 class ScanAllByLabelPropertyValue;
 class ScanAllByLabelProperty;
-class ScanAllById;
+class ScanByPrimaryKey;
 class Expand;
 class ExpandVariable;
 class ConstructNamedPath;
@@ -136,7 +160,7 @@ class Foreach;
 using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
     Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
     ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
-    ScanAllByLabelProperty, ScanAllById,
+    ScanAllByLabelProperty, ScanByPrimaryKey,
     Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
     SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
     EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
@@ -327,11 +351,14 @@ and false on every following Pull.")
    class OnceCursor : public Cursor {
     public:
      OnceCursor() {}
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
+     void PushDown(const MultiFrame&) override;
      bool Pull(Frame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
     private:
+     std::optional<MultiFrame> pushed_down_multi_frame_;
      bool did_pull_{false};
    };
    cpp<#)
@@ -836,19 +863,21 @@ given label and property.
   (:serialize (:slk))
   (:clone))
 
-
-
-(lcp:define-class scan-all-by-id (scan-all)
-  ((expression "Expression *" :scope :public
+(lcp:define-class scan-by-primary-key (scan-all)
+  ((label "::storage::v3::LabelId" :scope :public)
+    (primary-key "std::vector<Expression*>" :scope :public)
+    (expression "Expression *" :scope :public
                :slk-save #'slk-save-ast-pointer
                :slk-load (slk-load-ast-pointer "Expression")))
   (:documentation
-    "ScanAll producing a single node with ID equal to evaluated expression")
+    "ScanAll producing a single node with specified by the label and primary key")
   (:public
     #>cpp
-    ScanAllById() {}
-    ScanAllById(const std::shared_ptr<LogicalOperator> &input,
-                Symbol output_symbol, Expression *expression,
+    ScanByPrimaryKey() {}
+    ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
+                Symbol output_symbol,
+                storage::v3::LabelId label,
+                std::vector<query::v2::Expression*> primary_key,
                 storage::v3::View view = storage::v3::View::OLD);
 
     bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
@@ -1151,6 +1180,7 @@ a boolean value.")
     public:
      FilterCursor(const Filter &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1202,6 +1232,7 @@ RETURN clause) the Produce's pull succeeds exactly once.")
     public:
      ProduceCursor(const Produce &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1249,6 +1280,7 @@ Has a flag for using DETACH DELETE when deleting vertices.")
     public:
      DeleteCursor(const Delete &, utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1542,6 +1574,7 @@ edge lists).")
      EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
                                 utils::MemoryResource *);
      bool Pull(Frame &, ExecutionContext &) override;
+     bool PullMultiple(MultiFrame &, ExecutionContext &) override;
      void Shutdown() override;
      void Reset() override;
 
@@ -1938,27 +1971,6 @@ and returns true, once.")
      input_ = input;
    }
    cpp<#)
-  (:private
-   #>cpp
-   class OptionalCursor : public Cursor {
-    public:
-     OptionalCursor(const Optional &, utils::MemoryResource *);
-     bool Pull(Frame &, ExecutionContext &) override;
-     void Shutdown() override;
-     void Reset() override;
-
-    private:
-     const Optional &self_;
-     const UniqueCursorPtr input_cursor_;
-     const UniqueCursorPtr optional_cursor_;
-     // indicates if the next Pull from this cursor should
-     // perform a Pull from the input_cursor_
-     // this is true when:
-     //  - first pulling from this Cursor
-     //  - previous Pull from this cursor exhausted the optional_cursor_
-     bool pull_input_{true};
-   };
-   cpp<#)
   (:serialize (:slk))
   (:clone))
 

src/query/v2/plan/preprocess.cpp

@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
   auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
     auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
     if (!id_fun) return false;
-    if (id_fun->function_name_ != kId) return false;
+    if (id_fun->function_name_ != functions::kId) return false;
     if (id_fun->arguments_.size() != 1U) return false;
     auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
     if (!ident) return false;

src/query/v2/plan/pretty_print.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,11 +14,13 @@
 
 #include "query/v2/bindings/pretty_print.hpp"
 #include "query/v2/db_accessor.hpp"
+#include "query/v2/request_router.hpp"
 #include "utils/string.hpp"
 
 namespace memgraph::query::v2::plan {
 
-PlanPrinter::PlanPrinter(const DbAccessor *dba, std::ostream *out) : dba_(dba), out_(out) {}
+PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
+    : request_router_(request_router), out_(out) {}
 
 #define PRE_VISIT(TOp)                                   \
   bool PlanPrinter::PreVisit(TOp &) {                    \
@@ -32,7 +34,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
   WithPrintLn([&](auto &out) {
     out << "* CreateExpand (" << op.input_symbol_.name() << ")"
         << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
-        << op.edge_info_.symbol.name() << ":" << dba_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
+        << op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
         << (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
         << op.node_info_.symbol.name() << ")";
   });
@@ -52,7 +54,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabel"
-        << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << ")";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
   });
   return true;
 }
@@ -60,8 +62,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelPropertyValue"
-        << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
-        << dba_->PropertyToName(op.property_) << "})";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
@@ -69,8 +71,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelPropertyRange"
-        << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
-        << dba_->PropertyToName(op.property_) << "})";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
@@ -78,16 +80,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
 bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
   WithPrintLn([&](auto &out) {
     out << "* ScanAllByLabelProperty"
-        << " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
-        << dba_->PropertyToName(op.property_) << "})";
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
+        << request_router_->PropertyToName(op.property_) << "})";
   });
   return true;
 }
 
-bool PlanPrinter::PreVisit(ScanAllById &op) {
+bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
   WithPrintLn([&](auto &out) {
-    out << "* ScanAllById"
-        << " (" << op.output_symbol_.name() << ")";
+    out << "* ScanByPrimaryKey"
+        << " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
   });
   return true;
 }
@@ -98,7 +100,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
           << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
           << op.common_.edge_symbol.name();
     utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
-      stream << ":" << dba_->EdgeTypeToName(edge_type);
+      stream << ":" << request_router_->EdgeTypeToName(edge_type);
     });
     *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
           << op.common_.node_symbol.name() << ")";
@@ -127,7 +129,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
           << (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
           << op.common_.edge_symbol.name();
     utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
-      stream << ":" << dba_->EdgeTypeToName(edge_type);
+      stream << ":" << request_router_->EdgeTypeToName(edge_type);
     });
     *out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
           << op.common_.node_symbol.name() << ")";
@@ -261,14 +263,14 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
   --depth_;
 }
 
-void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out) {
-  PlanPrinter printer(&dba, out);
+void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
+  PlanPrinter printer(&request_router, out);
   // FIXME(mtomic): We should make visitors that take const arguments.
   const_cast<LogicalOperator *>(plan_root)->Accept(printer);
 }
 
-nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root) {
-  impl::PlanToJsonVisitor visitor(&dba);
+nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
+  impl::PlanToJsonVisitor visitor(&request_router);
   // FIXME(mtomic): We should make visitors that take const arguments.
   const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
   return visitor.output();
@@ -346,11 +348,17 @@ json ToJson(const utils::Bound<Expression *> &bound) {
 
 json ToJson(const Symbol &symbol) { return symbol.name(); }
 
-json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba) { return dba.EdgeTypeToName(edge_type); }
+json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
+  return request_router.EdgeTypeToName(edge_type);
+}
 
-json ToJson(storage::v3::LabelId label, const DbAccessor &dba) { return dba.LabelToName(label); }
+json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
+  return request_router.LabelToName(label);
+}
 
-json ToJson(storage::v3::PropertyId property, const DbAccessor &dba) { return dba.PropertyToName(property); }
+json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
+  return request_router.PropertyToName(property);
+}
 
 json ToJson(NamedExpression *nexpr) {
   json json;
@@ -359,29 +367,30 @@ json ToJson(NamedExpression *nexpr) {
   return json;
 }
 
-json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties, const DbAccessor &dba) {
+json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
+            const RequestRouterInterface &request_router) {
   json json;
   for (const auto &prop_pair : properties) {
-    json.emplace(ToJson(prop_pair.first, dba), ToJson(prop_pair.second));
+    json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
   }
   return json;
 }
 
-json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba) {
+json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
   json self;
   self["symbol"] = ToJson(node_info.symbol);
-  self["labels"] = ToJson(node_info.labels, dba);
+  self["labels"] = ToJson(node_info.labels, request_router);
   const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
-  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
+  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
   return self;
 }
 
-json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba) {
+json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
   json self;
   self["symbol"] = ToJson(edge_info.symbol);
   const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
-  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
-  self["edge_type"] = ToJson(edge_info.edge_type, dba);
+  self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
+  self["edge_type"] = ToJson(edge_info.edge_type, request_router);
   self["direction"] = ToString(edge_info.direction);
   return self;
 }
@@ -423,7 +432,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
   json self;
   self["name"] = "ScanAllByLabel";
-  self["label"] = ToJson(op.label_, *dba_);
+  self["label"] = ToJson(op.label_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
   op.input_->Accept(*this);
@@ -436,8 +445,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
   json self;
   self["name"] = "ScanAllByLabelPropertyRange";
-  self["label"] = ToJson(op.label_, *dba_);
-  self["property"] = ToJson(op.property_, *dba_);
+  self["label"] = ToJson(op.label_, *request_router_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
   self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
   self["output_symbol"] = ToJson(op.output_symbol_);
@@ -452,8 +461,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
   json self;
   self["name"] = "ScanAllByLabelPropertyValue";
-  self["label"] = ToJson(op.label_, *dba_);
-  self["property"] = ToJson(op.property_, *dba_);
+  self["label"] = ToJson(op.label_, *request_router_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["expression"] = ToJson(op.expression_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
@@ -467,8 +476,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
 bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
   json self;
   self["name"] = "ScanAllByLabelProperty";
-  self["label"] = ToJson(op.label_, *dba_);
-  self["property"] = ToJson(op.property_, *dba_);
+  self["label"] = ToJson(op.label_, *request_router_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
 
   op.input_->Accept(*this);
@@ -478,12 +487,15 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
   return false;
 }
 
-bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
+bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
   json self;
-  self["name"] = "ScanAllById";
+  self["name"] = "ScanByPrimaryKey";
+  self["label"] = ToJson(op.label_, *request_router_);
   self["output_symbol"] = ToJson(op.output_symbol_);
+
   op.input_->Accept(*this);
   self["input"] = PopOutput();
+
   output_ = std::move(self);
   return false;
 }
@@ -491,7 +503,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
 bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
   json self;
   self["name"] = "CreateNode";
-  self["node_info"] = ToJson(op.node_info_, *dba_);
+  self["node_info"] = ToJson(op.node_info_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();
@@ -504,8 +516,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
   json self;
   self["name"] = "CreateExpand";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["node_info"] = ToJson(op.node_info_, *dba_);
-  self["edge_info"] = ToJson(op.edge_info_, *dba_);
+  self["node_info"] = ToJson(op.node_info_, *request_router_);
+  self["edge_info"] = ToJson(op.edge_info_, *request_router_);
   self["existing_node"] = op.existing_node_;
 
   op.input_->Accept(*this);
@@ -521,7 +533,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
   self["input_symbol"] = ToJson(op.input_symbol_);
   self["node_symbol"] = ToJson(op.common_.node_symbol);
   self["edge_symbol"] = ToJson(op.common_.edge_symbol);
-  self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
+  self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
   self["direction"] = ToString(op.common_.direction);
   self["existing_node"] = op.common_.existing_node;
 
@@ -538,7 +550,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
   self["input_symbol"] = ToJson(op.input_symbol_);
   self["node_symbol"] = ToJson(op.common_.node_symbol);
   self["edge_symbol"] = ToJson(op.common_.edge_symbol);
-  self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
+  self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
   self["direction"] = ToString(op.common_.direction);
   self["type"] = ToString(op.type_);
   self["is_reverse"] = op.is_reverse_;
@@ -613,7 +625,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
 bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
   json self;
   self["name"] = "SetProperty";
-  self["property"] = ToJson(op.property_, *dba_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lhs"] = ToJson(op.lhs_);
   self["rhs"] = ToJson(op.rhs_);
 
@@ -650,7 +662,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
   json self;
   self["name"] = "SetLabels";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["labels"] = ToJson(op.labels_, *dba_);
+  self["labels"] = ToJson(op.labels_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();
@@ -662,7 +674,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
 bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
   json self;
   self["name"] = "RemoveProperty";
-  self["property"] = ToJson(op.property_, *dba_);
+  self["property"] = ToJson(op.property_, *request_router_);
   self["lhs"] = ToJson(op.lhs_);
 
   op.input_->Accept(*this);
@@ -676,7 +688,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
   json self;
   self["name"] = "RemoveLabels";
   self["input_symbol"] = ToJson(op.input_symbol_);
-  self["labels"] = ToJson(op.labels_, *dba_);
+  self["labels"] = ToJson(op.labels_, *request_router_);
 
   op.input_->Accept(*this);
   self["input"] = PopOutput();

src/query/v2/plan/pretty_print.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,28 +18,28 @@
 
 #include "query/v2/frontend/ast/ast.hpp"
 #include "query/v2/plan/operator.hpp"
+#include "query/v2/request_router.hpp"
 
 namespace memgraph::query::v2 {
-class DbAccessor;
 
 namespace plan {
 
 class LogicalOperator;
 
 /// Pretty print a `LogicalOperator` plan to a `std::ostream`.
-/// DbAccessor is needed for resolving label and property names.
+/// RequestRouter is needed for resolving label and property names.
 /// Note that `plan_root` isn't modified, but we can't take it as a const
 /// because we don't have support for visiting a const LogicalOperator.
-void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out);
+void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
 
 /// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
-inline void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root) {
-  PrettyPrint(dba, plan_root, &std::cout);
+inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
+  PrettyPrint(request_router, plan_root, &std::cout);
 }
 
 /// Convert a `LogicalOperator` plan to a JSON representation.
 /// DbAccessor is needed for resolving label and property names.
-nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root);
+nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
 
 class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
  public:
@@ -47,7 +47,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   using HierarchicalLogicalOperatorVisitor::PreVisit;
   using HierarchicalLogicalOperatorVisitor::Visit;
 
-  PlanPrinter(const DbAccessor *dba, std::ostream *out);
+  PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
 
   bool DefaultPreVisit() override;
 
@@ -67,7 +67,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   bool PreVisit(ScanAllByLabelPropertyValue &) override;
   bool PreVisit(ScanAllByLabelPropertyRange &) override;
   bool PreVisit(ScanAllByLabelProperty &) override;
-  bool PreVisit(ScanAllById &) override;
+  bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
 
   bool PreVisit(Expand &) override;
   bool PreVisit(ExpandVariable &) override;
@@ -114,7 +114,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
   void Branch(LogicalOperator &op, const std::string &branch_name = "");
 
   int64_t depth_{0};
-  const DbAccessor *dba_{nullptr};
+  const RequestRouterInterface *request_router_{nullptr};
   std::ostream *out_{nullptr};
 };
 
@@ -132,20 +132,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
 
 nlohmann::json ToJson(const Symbol &symbol);
 
-nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba);
+nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(storage::v3::LabelId label, const DbAccessor &dba);
+nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(storage::v3::PropertyId property, const DbAccessor &dba);
+nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
 
 nlohmann::json ToJson(NamedExpression *nexpr);
 
 nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
-                      const DbAccessor &dba);
+                      const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba);
+nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
 
-nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba);
+nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
 
 nlohmann::json ToJson(const Aggregate::Element &elem);
 
@@ -160,7 +160,7 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
 
 class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
  public:
-  explicit PlanToJsonVisitor(const DbAccessor *dba) : dba_(dba) {}
+  explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
 
   using HierarchicalLogicalOperatorVisitor::PostVisit;
   using HierarchicalLogicalOperatorVisitor::PreVisit;
@@ -194,7 +194,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
   bool PreVisit(ScanAllByLabelPropertyRange &) override;
   bool PreVisit(ScanAllByLabelPropertyValue &) override;
   bool PreVisit(ScanAllByLabelProperty &) override;
-  bool PreVisit(ScanAllById &) override;
+  bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
 
   bool PreVisit(Produce &) override;
   bool PreVisit(Accumulate &) override;
@@ -216,7 +216,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
 
  protected:
   nlohmann::json output_;
-  const DbAccessor *dba_;
+  const RequestRouterInterface *request_router_;
 
   nlohmann::json PopOutput() {
     nlohmann::json tmp;

src/query/v2/plan/profile.cpp

@@ -24,18 +24,18 @@ namespace memgraph::query::v2::plan {
 
 namespace {
 
-unsigned long long IndividualCycles(const ProfilingStats &cumulative_stats) {
+uint64_t IndividualCycles(const ProfilingStats &cumulative_stats) {
   return cumulative_stats.num_cycles - std::accumulate(cumulative_stats.children.begin(),
                                                        cumulative_stats.children.end(), 0ULL,
                                                        [](auto acc, auto &stats) { return acc + stats.num_cycles; });
 }
 
-double RelativeTime(unsigned long long num_cycles, unsigned long long total_cycles) {
-  return static_cast<double>(num_cycles) / total_cycles;
+double RelativeTime(const uint64_t num_cycles, const uint64_t total_cycles) {
+  return static_cast<double>(num_cycles) / static_cast<double>(total_cycles);
 }
 
-double AbsoluteTime(unsigned long long num_cycles, unsigned long long total_cycles,
-                    std::chrono::duration<double> total_time) {
+double AbsoluteTime(const uint64_t num_cycles, const uint64_t total_cycles,
+                    const std::chrono::duration<double> total_time) {
   return (RelativeTime(num_cycles, total_cycles) * static_cast<std::chrono::duration<double, std::milli>>(total_time))
       .count();
 }
@@ -50,26 +50,29 @@ namespace {
 
 class ProfilingStatsToTableHelper {
  public:
-  ProfilingStatsToTableHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
+  ProfilingStatsToTableHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
       : total_cycles_(total_cycles), total_time_(total_time) {}
 
   void Output(const ProfilingStats &cumulative_stats) {
     auto cycles = IndividualCycles(cumulative_stats);
+    auto custom_data_copy = cumulative_stats.custom_data;
+    ConvertCyclesToTime(custom_data_copy);
 
-    rows_.emplace_back(std::vector<TypedValue>{
-        TypedValue(FormatOperator(cumulative_stats.name)), TypedValue(cumulative_stats.actual_hits),
-        TypedValue(FormatRelativeTime(cycles)), TypedValue(FormatAbsoluteTime(cycles))});
+    rows_.emplace_back(
+        std::vector<TypedValue>{TypedValue(FormatOperator(cumulative_stats.name)),
+                                TypedValue(cumulative_stats.actual_hits), TypedValue(FormatRelativeTime(cycles)),
+                                TypedValue(FormatAbsoluteTime(cycles)), TypedValue(custom_data_copy.dump())});
 
     for (size_t i = 1; i < cumulative_stats.children.size(); ++i) {
       Branch(cumulative_stats.children[i]);
     }
 
-    if (cumulative_stats.children.size() >= 1) {
+    if (!cumulative_stats.children.empty()) {
       Output(cumulative_stats.children[0]);
     }
   }
 
-  std::vector<std::vector<TypedValue>> rows() { return rows_; }
+  std::vector<std::vector<TypedValue>> rows() const { return rows_; }
 
  private:
   void Branch(const ProfilingStats &cumulative_stats) {
@@ -80,7 +83,28 @@ class ProfilingStatsToTableHelper {
     --depth_;
   }
 
-  std::string Format(const char *str) {
+  double AbsoluteTime(const uint64_t cycles) const { return plan::AbsoluteTime(cycles, total_cycles_, total_time_); }
+
+  double RelativeTime(const uint64_t cycles) const { return plan::RelativeTime(cycles, total_cycles_); }
+
+  void ConvertCyclesToTime(nlohmann::json &custom_data) const {
+    const auto convert_cycles_in_json = [this](nlohmann::json &json) {
+      if (!json.is_object()) {
+        return;
+      }
+      if (auto it = json.find(ProfilingStats::kNumCycles); it != json.end()) {
+        auto num_cycles = it.value().get<uint64_t>();
+        json[ProfilingStats::kAbsoluteTime] = AbsoluteTime(num_cycles);
+        json[ProfilingStats::kRelativeTime] = RelativeTime(num_cycles);
+      }
+    };
+
+    for (auto &json : custom_data) {
+      convert_cycles_in_json(json);
+    }
+  }
+
+  std::string Format(const char *str) const {
     std::ostringstream ss;
     for (int64_t i = 0; i < depth_; ++i) {
       ss << "| ";
@@ -89,21 +113,21 @@ class ProfilingStatsToTableHelper {
     return ss.str();
   }
 
-  std::string Format(const std::string &str) { return Format(str.c_str()); }
+  std::string Format(const std::string &str) const { return Format(str.c_str()); }
 
-  std::string FormatOperator(const char *str) { return Format(std::string("* ") + str); }
+  std::string FormatOperator(const char *str) const { return Format(std::string("* ") + str); }
 
-  std::string FormatRelativeTime(unsigned long long num_cycles) {
-    return fmt::format("{: 10.6f} %", RelativeTime(num_cycles, total_cycles_) * 100);
+  std::string FormatRelativeTime(uint64_t num_cycles) const {
+    return fmt::format("{: 10.6f} %", RelativeTime(num_cycles) * 100);
   }
 
-  std::string FormatAbsoluteTime(unsigned long long num_cycles) {
-    return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles, total_cycles_, total_time_));
+  std::string FormatAbsoluteTime(uint64_t num_cycles) const {
+    return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles));
   }
 
   int64_t depth_{0};
   std::vector<std::vector<TypedValue>> rows_;
-  unsigned long long total_cycles_;
+  uint64_t total_cycles_;
   std::chrono::duration<double> total_time_;
 };
 
@@ -126,7 +150,7 @@ class ProfilingStatsToJsonHelper {
   using json = nlohmann::json;
 
  public:
-  ProfilingStatsToJsonHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
+  ProfilingStatsToJsonHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
       : total_cycles_(total_cycles), total_time_(total_time) {}
 
   void Output(const ProfilingStats &cumulative_stats) { return Output(cumulative_stats, &json_); }
@@ -151,7 +175,7 @@ class ProfilingStatsToJsonHelper {
   }
 
   json json_;
-  unsigned long long total_cycles_;
+  uint64_t total_cycles_;
   std::chrono::duration<double> total_time_;
 };