Add patch

This reverts commit 5780ee6c5b.

.clang-tidy

@@ -6,7 +6,6 @@ Checks: '*,
          -altera-unroll-loops,
          -android-*,
          -cert-err58-cpp,
-         -cert-str34-c,
          -cppcoreguidelines-avoid-c-arrays,
          -cppcoreguidelines-avoid-goto,
          -cppcoreguidelines-avoid-magic-numbers,
@@ -50,7 +49,6 @@ Checks: '*,
          -misc-non-private-member-variables-in-classes,
          -modernize-avoid-c-arrays,
          -modernize-concat-nested-namespaces,
-         -modernize-loop-convert,
          -modernize-pass-by-value,
          -modernize-use-equals-default,
          -modernize-use-nodiscard,
@@ -63,9 +61,7 @@ Checks: '*,
          -readability-magic-numbers,
          -readability-named-parameter,
          -misc-no-recursion,
-         -concurrency-mt-unsafe,
-         -bugprone-easily-swappable-parameters'
-
+         -concurrency-mt-unsafe'
 WarningsAsErrors: ''
 HeaderFilterRegex: 'src/.*'
 AnalyzeTemporaryDtors: false

.github/pull_request_template.md

@@ -3,9 +3,7 @@
 - [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
 - [ ] Write E2E tests
 - [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
-- [ ] Provide the full content or a guide for the final git message
 
 [master < Task] PR
 - [ ] Check, and update documentation if necessary
 - [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
-- [ ] Provide the full content or a guide for the final git message

.github/workflows/diff.yaml

@@ -1,7 +1,4 @@
 name: Diff
-concurrency:
-  group: ${{ github.head_ref || github.sha }}
-  cancel-in-progress: true
 
 on:
   push:
@@ -70,11 +67,6 @@ jobs:
           # branches and tags. (default: 1)
           fetch-depth: 0
 
-        # This is also needed if we want do to comparison against other branches
-        # See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
-      - name: Fetch all history for all tags and branches
-        run: git fetch
-
       - name: Build combined ASAN, UBSAN and coverage binaries
         run: |
           # Activate toolchain.
@@ -115,22 +107,12 @@ jobs:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
 
-      - name: Set base branch
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
-
-      - name: Set base branch # if we manually dispatch or push to master
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
-
       - name: Run clang-tidy
         run: |
           source /opt/toolchain-v4/activate
 
           # Restrict clang-tidy results only to the modified parts
-          git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
+          git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
 
           # Fail if any warning is reported
           ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@@ -171,7 +153,7 @@ jobs:
 
           # Run leftover CTest tests (all except unit and benchmark tests).
           cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
 
       - name: Run drivers tests
         run: |
@@ -262,15 +244,6 @@ jobs:
           cd build
           ctest -R memgraph__unit --output-on-failure -j$THREADS
 
-      - name: Run simulation tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__simulation --output-on-failure -j$THREADS
-
       - name: Run e2e tests
         run: |
           # TODO(gitbuda): Setup mgclient and pymgclient properly.

.github/workflows/package_all.yaml

@@ -6,11 +6,11 @@ on: workflow_dispatch
 
 jobs:
   centos-7:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -22,29 +22,29 @@ jobs:
           name: centos-7
           path: build/output/centos-7/memgraph*.rpm
 
-  centos-9:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+  centos-8:
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
         run: |
-          ./release/package/run.sh package centos-9
+          ./release/package/run.sh package centos-8
       - name: "Upload package"
         uses: actions/upload-artifact@v2
         with:
-          name: centos-9
-          path: build/output/centos-9/memgraph*.rpm
+          name: centos-8
+          path: build/output/centos-8/memgraph*.rpm
 
   debian-10:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -57,11 +57,11 @@ jobs:
           path: build/output/debian-10/memgraph*.deb
 
   debian-11:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -74,11 +74,11 @@ jobs:
           path: build/output/debian-11/memgraph*.deb
 
   docker:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -93,11 +93,11 @@ jobs:
           path: build/output/docker/memgraph*.tar.gz
 
   ubuntu-1804:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -110,11 +110,11 @@ jobs:
           path: build/output/ubuntu-18.04/memgraph*.deb
 
   ubuntu-2004:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -126,29 +126,12 @@ jobs:
           name: ubuntu-2004
           path: build/output/ubuntu-20.04/memgraph*.deb
 
-  ubuntu-2204:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package ubuntu-22.04
-      - name: "Upload package"
-        uses: actions/upload-artifact@v2
-        with:
-          name: ubuntu-2204
-          path: build/output/ubuntu-22.04/memgraph*.deb
-
   debian-11-platform:
-    runs-on: [self-hosted, DockerMgBuild, X64]
+    runs-on: [self-hosted, DockerMgBuild]
     timeout-minutes: 60
     steps:
       - name: "Set up repository"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Required because of release/get_version.py
       - name: "Build package"
@@ -159,20 +142,3 @@ jobs:
         with:
           name: debian-11-platform
           path: build/output/debian-11/memgraph*.deb
-
-  debian-11-arm:
-    runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package debian-11-arm
-      - name: "Upload package"
-        uses: actions/upload-artifact@v2
-        with:
-          name: debian-11-arm
-          path: build/output/debian-11-arm/memgraph*.deb

.github/workflows/release_docker.yaml

@@ -4,12 +4,8 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: "Memgraph binary version to publish on DockerHub."
+        description: "Memgraph binary version to publish on Dockerhub."
         required: true
-      force_release:
-        type: boolean
-        required: false
-        default: false
 
 jobs:
   docker_publish:
@@ -40,22 +36,6 @@ jobs:
           curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
           curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
 
-      - name: Check if specified version is already pushed
-        run: |
-          EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
-          echo $EXISTS
-          if [[ ${EXISTS} -eq 0 ]]; then
-              echo 'The specified version has been already released to DockerHub.'
-              if [[ ${{ github.event.inputs.force_release }} = true ]]; then
-                echo 'Forcing the release!'
-              else
-                echo 'Stopping the release!'
-                exit 1
-              fi
-          else
-              echo 'All good the specified version has not been release to DockerHub.'
-          fi
-
       - name: Build & push docker images
         run: |
           cd release/docker

.gitignore

@@ -23,7 +23,6 @@ cmake-build-*
 cmake/DownloadProject/
 dist/
 src/query/frontend/opencypher/generated/
-src/query/v2/frontend/opencypher/generated/
 tags
 ve/
 ve3/
@@ -51,21 +50,15 @@ src/distributed/pull_produce_rpc_messages.hpp
 src/distributed/storage_gc_rpc_messages.hpp
 src/distributed/token_sharing_rpc_messages.hpp
 src/distributed/updates_rpc_messages.hpp
-src/query/v2/frontend/ast/ast.hpp
 src/query/frontend/ast/ast.hpp
 src/query/distributed/frontend/ast/ast_serialization.hpp
-src/query/v2/distributed/frontend/ast/ast_serialization.hpp
 src/durability/distributed/state_delta.hpp
 src/durability/single_node/state_delta.hpp
 src/durability/single_node_ha/state_delta.hpp
 src/query/frontend/semantic/symbol.hpp
-src/query/v2/frontend/semantic/symbol.hpp
 src/query/distributed/frontend/semantic/symbol_serialization.hpp
-src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
 src/query/distributed/plan/ops.hpp
-src/query/v2/distributed/plan/ops.hpp
 src/query/plan/operator.hpp
-src/query/v2/plan/operator.hpp
 src/raft/log_entry.hpp
 src/raft/raft_rpc_messages.hpp
 src/raft/snapshot_metadata.hpp

CODEOWNERS

@@ -0,0 +1 @@
+* @antaljanosbenjamin @kostasrim

cmake/FindSodium.cmake

@@ -0,0 +1,297 @@
+# Written in 2016 by Henrik Steffen Gaßmann <henrik@gassmann.onl>
+#
+# To the extent possible under law, the author(s) have dedicated all
+# copyright and related and neighboring rights to this software to the
+# public domain worldwide. This software is distributed without any warranty.
+#
+# You should have received a copy of the CC0 Public Domain Dedication
+# along with this software. If not, see
+#
+#     http://creativecommons.org/publicdomain/zero/1.0/
+#
+########################################################################
+# Tries to find the local libsodium installation.
+#
+# On Windows the sodium_DIR environment variable is used as a default
+# hint which can be overridden by setting the corresponding cmake variable.
+#
+# Once done the following variables will be defined:
+#
+#   sodium_FOUND
+#   sodium_INCLUDE_DIR
+#   sodium_LIBRARY_DEBUG
+#   sodium_LIBRARY_RELEASE
+#
+#
+# Furthermore an imported "sodium" target is created.
+#
+
+if (CMAKE_C_COMPILER_ID STREQUAL "GNU"
+    OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
+    set(_GCC_COMPATIBLE 1)
+endif()
+
+# static library option
+if (NOT DEFINED sodium_USE_STATIC_LIBS)
+    option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
+endif()
+if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST))
+    unset(sodium_LIBRARY CACHE)
+    unset(sodium_LIBRARY_DEBUG CACHE)
+    unset(sodium_LIBRARY_RELEASE CACHE)
+    unset(sodium_DLL_DEBUG CACHE)
+    unset(sodium_DLL_RELEASE CACHE)
+    set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+endif()
+
+
+########################################################################
+# UNIX
+if (UNIX)
+    # import pkg-config
+    find_package(PkgConfig QUIET)
+    if (PKG_CONFIG_FOUND)
+        pkg_check_modules(sodium_PKG QUIET libsodium)
+    endif()
+
+    if(sodium_USE_STATIC_LIBS)
+        foreach(_libname ${sodium_PKG_STATIC_LIBRARIES})
+            if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a
+                list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a")
+            endif()
+        endforeach()
+        list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES)
+
+        # if pkgconfig for libsodium doesn't provide
+        # static lib info, then override PKG_STATIC here..
+        if (NOT sodium_PKG_STATIC_FOUND)
+            set(sodium_PKG_STATIC_LIBRARIES libsodium.a)
+        endif()
+
+        set(XPREFIX sodium_PKG_STATIC)
+    else()
+        if (NOT sodium_PKG_FOUND)
+            set(sodium_PKG_LIBRARIES sodium)
+        endif()
+
+        set(XPREFIX sodium_PKG)
+    endif()
+
+    find_path(sodium_INCLUDE_DIR sodium.h
+        HINTS ${${XPREFIX}_INCLUDE_DIRS}
+    )
+    find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES}
+        HINTS ${${XPREFIX}_LIBRARY_DIRS}
+    )
+    find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES}
+        HINTS ${${XPREFIX}_LIBRARY_DIRS}
+    )
+
+
+########################################################################
+# Windows
+elseif (WIN32)
+    set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory")
+    mark_as_advanced(sodium_DIR)
+
+    find_path(sodium_INCLUDE_DIR sodium.h
+        HINTS ${sodium_DIR}
+        PATH_SUFFIXES include
+    )
+
+    if (MSVC)
+        # detect target architecture
+        file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[
+            #if defined _M_IX86
+            #error ARCH_VALUE x86_32
+            #elif defined _M_X64
+            #error ARCH_VALUE x86_64
+            #endif
+            #error ARCH_VALUE unknown
+        ]=])
+        try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp"
+            OUTPUT_VARIABLE _COMPILATION_LOG
+        )
+        string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}")
+
+        # construct library path
+        if (_TARGET_ARCH STREQUAL "x86_32")
+            string(APPEND _PLATFORM_PATH "Win32")
+        elseif(_TARGET_ARCH STREQUAL "x86_64")
+            string(APPEND _PLATFORM_PATH "x64")
+        else()
+            message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.")
+        endif()
+        string(APPEND _PLATFORM_PATH "/$$CONFIG$$")
+
+        if (MSVC_VERSION LESS 1900)
+            math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60")
+        else()
+            math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50")
+        endif()
+        string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}")
+
+        if (sodium_USE_STATIC_LIBS)
+            string(APPEND _PLATFORM_PATH "/static")
+        else()
+            string(APPEND _PLATFORM_PATH "/dynamic")
+        endif()
+
+        string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}")
+        string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}")
+
+        find_library(sodium_LIBRARY_DEBUG libsodium.lib
+            HINTS ${sodium_DIR}
+            PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
+        )
+        find_library(sodium_LIBRARY_RELEASE libsodium.lib
+            HINTS ${sodium_DIR}
+            PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
+        )
+        if (NOT sodium_USE_STATIC_LIBS)
+            set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES})
+            set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll")
+            find_library(sodium_DLL_DEBUG libsodium
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
+            )
+            find_library(sodium_DLL_RELEASE libsodium
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
+            )
+            set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
+        endif()
+
+    elseif(_GCC_COMPATIBLE)
+        if (sodium_USE_STATIC_LIBS)
+            find_library(sodium_LIBRARY_DEBUG libsodium.a
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES lib
+            )
+            find_library(sodium_LIBRARY_RELEASE libsodium.a
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES lib
+            )
+        else()
+            find_library(sodium_LIBRARY_DEBUG libsodium.dll.a
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES lib
+            )
+            find_library(sodium_LIBRARY_RELEASE libsodium.dll.a
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES lib
+            )
+
+            file(GLOB _DLL
+                LIST_DIRECTORIES false
+                RELATIVE "${sodium_DIR}/bin"
+                "${sodium_DIR}/bin/libsodium*.dll"
+            )
+            find_library(sodium_DLL_DEBUG ${_DLL} libsodium
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES bin
+            )
+            find_library(sodium_DLL_RELEASE ${_DLL} libsodium
+                HINTS ${sodium_DIR}
+                PATH_SUFFIXES bin
+            )
+        endif()
+    else()
+        message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
+    endif()
+
+
+########################################################################
+# unsupported
+else()
+    message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
+endif()
+
+
+########################################################################
+# common stuff
+
+# extract sodium version
+if (sodium_INCLUDE_DIR)
+    set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h")
+    if (EXISTS _VERSION_HEADER)
+        file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT)
+        string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1"
+            sodium_VERSION "${_VERSION_HEADER_CONTENT}")
+        set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE)
+    endif()
+endif()
+
+# communicate results
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    Sodium # The name must be either uppercase or match the filename case.
+    REQUIRED_VARS
+        sodium_LIBRARY_RELEASE
+        sodium_LIBRARY_DEBUG
+        sodium_INCLUDE_DIR
+    VERSION_VAR
+        sodium_VERSION
+)
+
+if(Sodium_FOUND)
+    set(sodium_LIBRARIES
+        optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG})
+endif()
+
+# mark file paths as advanced
+mark_as_advanced(sodium_INCLUDE_DIR)
+mark_as_advanced(sodium_LIBRARY_DEBUG)
+mark_as_advanced(sodium_LIBRARY_RELEASE)
+if (WIN32)
+    mark_as_advanced(sodium_DLL_DEBUG)
+    mark_as_advanced(sodium_DLL_RELEASE)
+endif()
+
+# create imported target
+if(sodium_USE_STATIC_LIBS)
+    set(_LIB_TYPE STATIC)
+else()
+    set(_LIB_TYPE SHARED)
+endif()
+
+if(NOT TARGET sodium)
+    add_library(sodium ${_LIB_TYPE} IMPORTED)
+endif()
+
+set_target_properties(sodium PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+)
+
+if (sodium_USE_STATIC_LIBS)
+    set_target_properties(sodium PROPERTIES
+        INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC"
+        IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
+        IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
+    )
+else()
+    if (UNIX)
+        set_target_properties(sodium PROPERTIES
+            IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
+            IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
+        )
+    elseif (WIN32)
+        set_target_properties(sodium PROPERTIES
+            IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}"
+            IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}"
+        )
+        if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND"))
+            set_target_properties(sodium PROPERTIES
+                IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}"
+            )
+        endif()
+        if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND"))
+            set_target_properties(sodium PROPERTIES
+                IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}"
+                IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}"
+                IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}"
+            )
+        endif()
+    endif()
+endif()

cmake/MgThrift.cmake

@@ -0,0 +1,169 @@
+find_package(Gflags)
+find_package(Folly)
+find_package(wangle)
+find_package(fizz)
+find_package(proxygen)
+find_package(FBThrift)
+set(THRIFTCPP2 "FBThrift::thriftcpp2")
+set(THRIFT1 ${FBTHRIFT_COMPILER})
+
+include(${FBTHRIFT_INCLUDE_DIR}/thrift/ThriftLibrary.cmake)
+
+set(MG_INTERFACE_TARGET_NAME_PREFIX "mg-interface")
+
+function(_mg_thrift_generate
+  file_name
+  services
+  language
+  options
+  file_path
+  output_path
+  include_prefix
+)
+  cmake_parse_arguments(THRIFT_GENERATE   # Prefix
+    "" # Options
+    "" # One Value args
+    "THRIFT_INCLUDE_DIRECTORIES" # Multi-value args
+    "${ARGN}")
+
+  set(thrift_include_directories)
+  foreach(dir ${THRIFT_GENERATE_THRIFT_INCLUDE_DIRECTORIES})
+    list(APPEND thrift_include_directories "-I" "${dir}")
+  endforeach()
+
+  set("${file_name}-${language}-HEADERS"
+    ${output_path}/gen-${language}/${file_name}_constants.h
+    ${output_path}/gen-${language}/${file_name}_data.h
+    ${output_path}/gen-${language}/${file_name}_metadata.h
+    ${output_path}/gen-${language}/${file_name}_types.h
+    ${output_path}/gen-${language}/${file_name}_types.tcc
+  )
+  set("${file_name}-${language}-SOURCES"
+    ${output_path}/gen-${language}/${file_name}_constants.cpp
+    ${output_path}/gen-${language}/${file_name}_data.cpp
+    ${output_path}/gen-${language}/${file_name}_types.cpp
+  )
+  if(NOT "${options}" MATCHES "no_metadata")
+    set("${file_name}-${language}-SOURCES"
+      ${${file_name}-${language}-SOURCES}
+      ${output_path}/gen-${language}/${file_name}_metadata.cpp
+    )
+  endif()
+  foreach(service ${services})
+    set("${file_name}-${language}-HEADERS"
+      ${${file_name}-${language}-HEADERS}
+      ${output_path}/gen-${language}/${service}.h
+      ${output_path}/gen-${language}/${service}.tcc
+      ${output_path}/gen-${language}/${service}AsyncClient.h
+      ${output_path}/gen-${language}/${service}_custom_protocol.h
+    )
+    set("${file_name}-${language}-SOURCES"
+      ${${file_name}-${language}-SOURCES}
+      ${output_path}/gen-${language}/${service}.cpp
+      ${output_path}/gen-${language}/${service}AsyncClient.cpp
+    )
+  endforeach()
+  if("${include_prefix}" STREQUAL "")
+    set(include_prefix_text "")
+  else()
+    set(include_prefix_text "include_prefix=${include_prefix}")
+    if(NOT "${options}" STREQUAL "")
+      set(include_prefix_text ",${include_prefix_text}")
+    endif()
+  endif()
+  set(gen_language ${language})
+  if("${language}" STREQUAL "cpp2")
+    set(gen_language "mstch_cpp2")
+  elseif("${language}" STREQUAL "py3")
+    set(gen_language "mstch_py3")
+    file(WRITE "${output_path}/gen-${language}/${file_name}/__init__.py")
+  endif()
+  add_custom_command(
+    OUTPUT ${${file_name}-${language}-HEADERS}
+      ${${file_name}-${language}-SOURCES}
+    COMMAND ${THRIFT1}
+      --gen "${gen_language}:${options}${include_prefix_text}"
+      -o ${output_path}
+      ${thrift_include_directories}
+      "${file_path}/${file_name}.thrift"
+    DEPENDS
+      ${THRIFT1}
+      "${file_path}/${file_name}.thrift"
+    COMMENT "Generating ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language} files. Output: ${output_path}"
+  )
+  add_custom_target(
+    ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target ALL
+    DEPENDS ${${language}-${language}-HEADERS}
+      ${${file_name}-${language}-SOURCES}
+  )
+
+  set("${file_name}-${language}-SOURCES" ${${file_name}-${language}-SOURCES} PARENT_SCOPE)
+  install(
+    DIRECTORY gen-${language}
+    DESTINATION include/${include_prefix}
+    FILES_MATCHING PATTERN "*.h")
+  install(
+    DIRECTORY gen-${language}
+    DESTINATION include/${include_prefix}
+    FILES_MATCHING PATTERN "*.tcc")
+endfunction()
+
+
+function(_mg_thrift_object
+  file_name
+  services
+  language
+  options
+  file_path
+  output_path
+  include_prefix
+)
+  _mg_thrift_generate(
+    "${file_name}"
+    "${services}"
+    "${language}"
+    "${options}"
+    "${file_path}"
+    "${output_path}"
+    "${include_prefix}"
+    "${ARGN}"
+  )
+  bypass_source_check(${${file_name}-${language}-SOURCES})
+  add_library(
+    "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj"
+    OBJECT
+    ${${file_name}-${language}-SOURCES}
+  )
+  add_dependencies(
+    "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj"
+    "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target"
+  )
+  target_include_directories(${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj PUBLIC ${output_path})
+  message(STATUS "MgThrift will create the Object file : ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj")
+endfunction()
+
+function(mg_thrift_library
+  file_name
+  services
+  file_path
+  output_path
+  include_prefix
+)
+  _mg_thrift_object(
+    "${file_name}"
+    "${services}"
+    "cpp2"
+    "stack_arguments" # options
+    "${file_path}"
+    "${output_path}"
+    "${include_prefix}"
+    THRIFT_INCLUDE_DIRECTORIES "${FBTHRIFT_INCLUDE_DIR}"
+  )
+  set(LIBRARY_NAME "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-cpp2")
+  add_library(
+    "${LIBRARY_NAME}"
+    $<TARGET_OBJECTS:${LIBRARY_NAME}-obj>
+  )
+  target_link_libraries("${LIBRARY_NAME}" ${THRIFTCPP2})
+  message("MgThrift will create the library file : ${LIBRARY_NAME}")
+endfunction()

environment/os/centos-7.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "centos-7"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc gcc-c++ make # generic build tools
     wget # used for archive download

environment/os/centos-8.sh

@@ -0,0 +1,161 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils-common gcc gcc-c++ make # generic build tools
+    wget # used for archive download
+    gnupg2 # used for archive signature verification
+    tar gzip bzip2 xz unzip # used for archive unpacking
+    zlib-devel # zlib library used for all builds
+    expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
+    libcurl-devel # for cmake
+    curl # snappy
+    readline-devel # for cmake and llvm
+    libffi-devel libxml2-devel # for llvm
+    libedit-devel pcre-devel automake bison # for swig
+    file
+    openssl-devel
+    gmp-devel
+    gperf
+    patch
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz # used for archive unpacking
+    zlib # zlib library used for all builds
+    expat libipt libbabeltrace xz-libs python36 # for gdb
+    readline # for cmake and llvm
+    libffi libxml2 # for llvm
+    openssl-devel
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkgconf-pkg-config # build system
+    curl wget # for downloading libs
+    libuuid-devel java-11-openjdk # required by antlr
+    readline-devel # for memgraph console
+    python36-devel # for query modules
+    openssl-devel
+    libseccomp-devel
+    python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
+    #
+    # IMPORTANT: python3-yaml does NOT exist on CentOS
+    # Install it manually using `pip3 install PyYAML`
+    #
+    PyYAML # Package name here does not correspond to the yum package!
+    libcurl-devel # mg-requests
+    rpm-build rpmlint # for RPM package building
+    doxygen graphviz # source documentation generators
+    which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
+    sbcl # for custom Lisp C++ preprocessing
+    autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    local missing=""
+    for pkg in $1; do
+        if [ "$pkg" == "PyYAML" ]; then
+            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
+                missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+install() {
+    cd "$DIR"
+    if [ "$EUID" -ne 0 ]; then
+        echo "Please run as root."
+        exit 1
+    fi
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    dnf install -y epel-release
+    dnf install -y 'dnf-command(config-manager)'
+    dnf config-manager --set-enabled powertools # Required to install texinfo.
+    dnf update -y
+    dnf install -y wget git python36 python3-pip
+    for pkg in $1; do
+        if [ "$pkg" == libipt ]; then
+            if ! dnf list installed libipt >/dev/null 2>/dev/null; then
+                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == libipt-devel ]; then
+            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
+                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        # Install GDB dependencies not present in the standard repos.
+        # https://bugs.centos.org/view.php?id=17068
+        # https://centos.pkgs.org
+	    # Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
+        # within GDB yet (an assumption).
+        # http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
+        if [ "$pkg" == libbabeltrace-devel ]; then
+            if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
+                dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == sbcl ]; then
+            if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
+                dnf install -y 	https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
+            fi
+            if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
+                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
+            fi
+            if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
+                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
+                wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
+                rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm
+                dnf update -y
+                dnf install -y dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        if [ "$pkg" == PyYAML ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
+            continue
+        fi
+        dnf install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/centos-9.sh

@@ -5,16 +5,15 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "centos-9"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
-    coreutils-common gcc gcc-c++ make # generic build tools
+    coreutils gcc gcc-c++ make # generic build tools
     wget # used for archive download
     gnupg2 # used for archive signature verification
     tar gzip bzip2 xz unzip # used for archive unpacking
     zlib-devel # zlib library used for all builds
-    expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
+    expat-devel xz-devel python3-devel texinfo # for gdb
+    libcurl-devel # for cmake
+    curl # snappy
     readline-devel # for cmake and llvm
     libffi-devel libxml2-devel # for llvm
     libedit-devel pcre-devel automake bison # for swig
@@ -22,9 +21,6 @@ TOOLCHAIN_BUILD_DEPS=(
     openssl-devel
     gmp-devel
     gperf
-    diffutils
-    libipt libipt-devel # intel
-    patch
 )
 
 TOOLCHAIN_RUN_DEPS=(
@@ -35,19 +31,18 @@ TOOLCHAIN_RUN_DEPS=(
     readline # for cmake and llvm
     libffi libxml2 # for llvm
     openssl-devel
-    perl # for openssl
 )
 
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkgconf-pkg-config # build system
-    wget # for downloading libs
+    curl wget # for downloading libs
     libuuid-devel java-11-openjdk # required by antlr
     readline-devel # for memgraph console
     python3-devel # for query modules
     openssl-devel
     libseccomp-devel
-    python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
+    python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
     #
     # IMPORTANT: python3-yaml does NOT exist on CentOS
     # Install it manually using `pip3 install PyYAML`
@@ -78,6 +73,12 @@ check() {
         if [ "$pkg" == "python3-virtualenv" ]; then
             continue
         fi
+        if [ "$pkg" == sbcl ]; then
+            if ! sbcl --version &> /dev/null; then
+	        missing="$pkg $missing"
+            fi
+            continue
+        fi
         if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
             missing="$pkg $missing"
         fi
@@ -104,37 +105,13 @@ install() {
     yum update -y
     yum install -y wget git python3 python3-pip
     for pkg in $1; do
-        # Since there is no support for libipt-devel for CentOS 9 we install
-        # Fedoras version of same libs, they are the same version but released
-        # for different OS
-        # TODO Update when libipt-devel releases for CentOS 9
-        if [ "$pkg" == libipt ]; then
-            if ! dnf list installed libipt >/dev/null 2>/dev/null; then
-                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libipt-devel ]; then
-            if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
-                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libbabeltrace-devel ]; then
-            if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
-                dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
-            fi
-            continue
-        fi
         if [ "$pkg" == sbcl ]; then
-            if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
-                dnf install -y 	https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
-            fi
-            if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
-                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
-            fi
-            if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
-                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
+            if ! sbcl --version &> /dev/null; then
+	        curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
+		tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
+		pushd /tmp/sbcl-1.4.2-arm64-linux
+		INSTALL_ROOT=/usr/local sh install.sh
+		popd
             fi
             continue
         fi
@@ -148,11 +125,9 @@ install() {
         fi
         if [ "$pkg" == python3-virtualenv ]; then
             if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
-                pip3 install virtualenv
-                pip3 install virtualenvwrapper
+                pip3 install --user virtualenv
             else # Running using sudo.
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
             fi
             continue
         fi

environment/os/debian-10.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "debian-10"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc g++ build-essential make # generic build tools
     wget # used for archive download

environment/os/debian-11-arm.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "debian-11"
-check_architecture "arm64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc g++ build-essential make # generic build tools
     wget # used for archive download
@@ -90,6 +87,15 @@ EOF
     fi
     apt install -y wget
     for pkg in $1; do
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
+                wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+                dpkg -i packages-microsoft-prod.deb
+                apt-get update
+                apt-get install -y apt-transport-https dotnet-sdk-3.1
+            fi
+            continue
+        fi
         apt install -y "$pkg"
     done
 }

environment/os/debian-11.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "debian-11"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc g++ build-essential make # generic build tools
     wget # used for archive download
@@ -90,7 +87,7 @@ EOF
     fi
     apt install -y wget
     for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-3.1  ]; then
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
             if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
                 wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
                 dpkg -i packages-microsoft-prod.deb

environment/os/template.sh

@@ -5,8 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "todo-os-name"
-
 TOOLCHAIN_BUILD_DEPS=(
     pkg
 )

environment/os/ubuntu-18.04.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "ubuntu-18.04"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc g++ build-essential make # generic build tools
     wget # archive download

environment/os/ubuntu-20.04.sh

@@ -5,9 +5,6 @@ set -Eeuo pipefail
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 source "$DIR/../util.sh"
 
-check_operating_system "ubuntu-20.04"
-check_architecture "x86_64"
-
 TOOLCHAIN_BUILD_DEPS=(
     coreutils gcc g++ build-essential make # generic build tools
     wget # used for archive download

environment/os/ubuntu-22.04.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "ubuntu-22.04"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    curl # snappy
-    file
-    git # for thrift
-    libgmp-dev # for gdb
-    gperf # for proxygen
-    libssl-dev
-    libedit-dev libpcre3-dev automake bison # for swig
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    libreadline8 # for cmake and llvm
-    libffi7 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    dotnet-sdk-6.0 golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cd "$DIR"
-    apt update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-6.0 ]; then
-            if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
-                wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-                dpkg -i packages-microsoft-prod.deb
-                apt-get update
-                apt-get install -y apt-transport-https dotnet-sdk-6.0
-            fi
-            continue
-        fi
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/toolchain/fbthrift.patch

@@ -0,0 +1,12 @@
+diff -ur a/thrift/lib/cpp2/server/IOWorkerContext.h b/thrift/lib/cpp2/server/IOWorkerContext.h
+--- a/thrift/lib/cpp2/server/IOWorkerContext.h  2022-06-08 11:50:43.043948657 +0200
++++ b/thrift/lib/cpp2/server/IOWorkerContext.h  2022-06-08 11:47:33.232695125 +0200
+@@ -59,7 +59,7 @@
+           auto aliveLocked = alive->rlock();
+           if (*aliveLocked) {
+             // IOWorkerContext is still alive and so is replyQueue_
+-            queue->startConsumingInternal(&evb);
++            queue->startConsuming(&evb);
+           }
+         });
+   }

environment/toolchain/v4.sh

@@ -675,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
 SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
 SNAPPY_VERSION=1.1.9
 XZ_VERSION=5.2.5 # for LZMA
-ZLIB_VERSION=1.2.12
+ZLIB_VERSION=1.2.11
 ZSTD_VERSION=1.5.0
 WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
 
@@ -1178,21 +1178,12 @@ popd
 
 # create toolchain archive
 if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
-    DISTRO_FULL_NAME=${DISTRO}
-    if [[ "${DISTRO}" == centos* ]]; then
-        if [[ "$for_arm" = "true" ]]; then
-            DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
-        else
-            DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
-        fi
+    DISTRO_FULL_NAME=$DISTRO
+    if [ "$for_arm" = true ]; then
+        DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
     else
-        if [[ "$for_arm" = "true" ]]; then
-            DISTRO_FULL_NAME="$DISTRO_FULL_NAME-arm64"
-        else
-            DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
-        fi
+        DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
     fi
-
     tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
 fi
 

environment/util.sh

@@ -5,28 +5,10 @@ operating_system() {
     sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
 }
 
-check_operating_system() {
-    if [ "$(operating_system)" != "$1" ]; then
-        echo "Not the right operating system!"
-        exit 1
-    else
-        echo "The right operating system."
-    fi
-}
-
 architecture() {
     uname -m
 }
 
-check_architecture() {
-    if [ "$(architecture)" != "$1" ]; then
-        echo "Not the right architecture!"
-        exit 1
-    else
-        echo "The right architecture."
-    fi
-}
-
 check_all_yum() {
     local missing=""
     for pkg in $1; do

init

@@ -24,7 +24,7 @@ function setup_virtualenv () {
     fi
 
     # create new virtualenv
-    python3 -m virtualenv -p python3 ve3 || exit 1
+    virtualenv -p python3 ve3 || exit 1
     source ve3/bin/activate
     pip --timeout 1000 install -r requirements.txt || exit 1
     deactivate
@@ -66,7 +66,7 @@ fi
 
 DISTRO=$(operating_system)
 ARCHITECTURE=$(architecture)
-if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
+if [ "${ARCHITECTURE}" = "arm64" ]; then
   OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
 else
   OS_SCRIPT=$DIR/environment/os/$DISTRO.sh
@@ -139,6 +139,3 @@ done;
 # Install precommit hook
 python3 -m pip install pre-commit
 python3 -m pre_commit install
-
-# Link `include/mgp.py` with `release/mgp/mgp.py`
-ln -v -f include/mgp.py release/mgp/mgp.py

libs/.gitignore

@@ -4,5 +4,5 @@
 !cleanup.sh
 !CMakeLists.txt
 !__main.cpp
+!jemalloc.cmake
 !pulsar.patch
-!antlr4.10.1.patch

libs/CMakeLists.txt

@@ -106,7 +106,6 @@ import_external_library(antlr4 STATIC
              -DWITH_LIBCXX=OFF # because of debian bug
              -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
              -DCMAKE_CXX_STANDARD=20
-             -DANTLR_BUILD_CPP_TESTS=OFF
   BUILD_COMMAND $(MAKE) antlr4_static
   INSTALL_COMMAND $(MAKE) install)
 

libs/antlr4.10.1.patch

@@ -1,13 +0,0 @@
-diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
-index baf46cac9..2e7756de8 100644
---- a/runtime/Cpp/runtime/CMakeLists.txt
-+++ b/runtime/Cpp/runtime/CMakeLists.txt
-@@ -134,7 +134,7 @@ set_target_properties(antlr4_static
-                                  ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
-                                  COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-
--install(TARGETS antlr4_shared
-+install(TARGETS antlr4_shared OPTIONAL
-         EXPORT antlr4-targets
-         ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-         LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

libs/antlr4.patch

@@ -0,0 +1,43 @@
+diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
+index a8503bb..11362cf 100644
+--- a/runtime/Cpp/runtime/CMakeLists.txt
++++ b/runtime/Cpp/runtime/CMakeLists.txt
+@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
+ set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
+ ExternalProject_Add(
+   utfcpp
+-  GIT_REPOSITORY        "git://github.com/nemtrif/utfcpp"
+-  GIT_TAG               "v3.1.1"
++  GIT_REPOSITORY        "https://github.com/nemtrif/utfcpp"
++  GIT_TAG               "v3.2.1"
+   SOURCE_DIR            ${UTFCPP_DIR}
+   UPDATE_DISCONNECTED   1
+   CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
+@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
+                                  ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
+                                  COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
+ 
+-install(TARGETS antlr4_shared
++install(TARGETS antlr4_shared OPTIONAL
+         DESTINATION lib
+         EXPORT antlr4-targets)
+ install(TARGETS antlr4_static
+diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
+index 468db98..65a473b 100644
+--- a/runtime/Cpp/runtime/src/support/Any.h
++++ b/runtime/Cpp/runtime/src/support/Any.h
+@@ -122,12 +122,12 @@ private:
+     }
+ 
+   private:
+-    template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
++    template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
+     Base* clone() const {
+       return new Derived<T>(value);
+     }
+ 
+-    template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
++    template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
+     Base* clone() const {
+       return nullptr;
+     }

libs/jemalloc.cmake

@@ -0,0 +1,55 @@
+set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
+
+set(JEMALLOC_SRCS
+    ${JEMALLOC_DIR}/src/arena.c
+    ${JEMALLOC_DIR}/src/background_thread.c
+    ${JEMALLOC_DIR}/src/base.c
+    ${JEMALLOC_DIR}/src/bin.c
+    ${JEMALLOC_DIR}/src/bitmap.c
+    ${JEMALLOC_DIR}/src/ckh.c
+    ${JEMALLOC_DIR}/src/ctl.c
+    ${JEMALLOC_DIR}/src/div.c
+    ${JEMALLOC_DIR}/src/extent.c
+    ${JEMALLOC_DIR}/src/extent_dss.c
+    ${JEMALLOC_DIR}/src/extent_mmap.c
+    ${JEMALLOC_DIR}/src/hash.c
+    ${JEMALLOC_DIR}/src/hook.c
+    ${JEMALLOC_DIR}/src/jemalloc.c
+    ${JEMALLOC_DIR}/src/large.c
+    ${JEMALLOC_DIR}/src/log.c
+    ${JEMALLOC_DIR}/src/malloc_io.c
+    ${JEMALLOC_DIR}/src/mutex.c
+    ${JEMALLOC_DIR}/src/mutex_pool.c
+    ${JEMALLOC_DIR}/src/nstime.c
+    ${JEMALLOC_DIR}/src/pages.c
+    ${JEMALLOC_DIR}/src/prng.c
+    ${JEMALLOC_DIR}/src/prof.c
+    ${JEMALLOC_DIR}/src/rtree.c
+    ${JEMALLOC_DIR}/src/sc.c
+    ${JEMALLOC_DIR}/src/stats.c
+    ${JEMALLOC_DIR}/src/sz.c
+    ${JEMALLOC_DIR}/src/tcache.c
+    ${JEMALLOC_DIR}/src/test_hooks.c
+    ${JEMALLOC_DIR}/src/ticker.c
+    ${JEMALLOC_DIR}/src/tsd.c
+    ${JEMALLOC_DIR}/src/witness.c
+    ${JEMALLOC_DIR}/src/safety_check.c
+)
+
+add_library(jemalloc ${JEMALLOC_SRCS})
+target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
+
+find_package(Threads REQUIRED)
+target_link_libraries(jemalloc PUBLIC Threads::Threads)
+
+target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
+
+if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
+endif()
+
+target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
+# for RTLD_NEXT
+target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
+
+set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

libs/setup.sh

@@ -105,7 +105,7 @@ repo_clone_try_double () {
 # Download from primary_urls might fail because the cache is not installed.
 declare -A primary_urls=(
   ["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
-  ["antlr4-generator"]="http://$local_cache_host/file/antlr-4.10.1-complete.jar"
+  ["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
   ["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
   ["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
   ["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
@@ -130,7 +130,7 @@ declare -A primary_urls=(
 # should fail.
 declare -A secondary_urls=(
   ["antlr4-code"]="https://github.com/antlr/antlr4.git"
-  ["antlr4-generator"]="https://www.antlr.org/download/antlr-4.10.1-complete.jar"
+  ["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
   ["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
   ["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
   ["gbenchmark"]="https://github.com/google/benchmark.git"
@@ -152,10 +152,10 @@ declare -A secondary_urls=(
 # antlr
 file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
 
-antlr4_tag="4.10.1" # v4.10.1
+antlr4_tag="4.9.2" # v4.9.2
 repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
 pushd antlr4
-git apply ../antlr4.10.1.patch
+git apply ../antlr4.patch
 popd
 
 # cppitertools v2.0 2019-12-23
@@ -199,7 +199,7 @@ git apply ../rocksdb.patch
 popd
 
 # mgclient
-mgclient_tag="v1.4.0" # (2022-06-14)
+mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
 repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
 

release/CMakeLists.txt

@@ -67,7 +67,7 @@ It aims to deliver developers the speed, simplicity and scale required to build
 the next generation of applications driver by real-time connected data.")
 # Add `openssl` package to dependencies list. Used to generate SSL certificates.
 # We also depend on `python3` because we embed it in Memgraph.
-set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate")
+set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6")
 
 # All variables must be set before including.
 include(CPack)

release/docker/memgraph_rpm.dockerfile

@@ -0,0 +1,31 @@
+FROM dokken/centos-stream-9
+# NOTE: If you change the base distro update release/package as well.
+
+ARG release
+
+RUN yum update && yum install -y \
+    openssl libcurl libseccomp python3 python3-pip \
+    --nobest --allowerasing \
+  && rm -rf /tmp/* \
+  && yum clean all
+
+RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
+
+COPY ${release} /
+
+# Install memgraph package
+RUN rpm -i ${release}
+
+# Memgraph listens for Bolt Protocol on this port by default.
+EXPOSE 7687
+# Snapshots and logging volumes
+VOLUME /var/log/memgraph
+VOLUME /var/lib/memgraph
+# Configuration volume
+VOLUME /etc/memgraph
+
+USER memgraph
+WORKDIR /usr/lib/memgraph
+
+ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
+CMD [""]

release/mgp/.gitignore

@@ -1,3 +0,0 @@
-.venv
-dist
-mgp.py

release/mgp/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

release/mgp/README.md

@@ -1,4 +0,0 @@
-# mgp
-
-PyPi package used for type hinting when creating MAGE modules. The get started
-using MAGE repository checkout the repository here: https://github.com/memgraph/mage.

release/mgp/_mgp.py

@@ -1,255 +0,0 @@
-from typing import Any
-
-
-class MgpIterable:
-    def get() -> Any:
-        pass
-
-    def next() -> Any:
-        pass
-
-
-class Vertex:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-    def underlying_graph_is_mutable() -> bool:  # type: ignore
-        pass
-
-    def iter_properties() -> MgpIterable:  # type: ignore
-        pass
-
-    def get_property(self, property_name: str) -> "Property":  # type: ignore
-        pass
-
-    def set_property(self, property_name: str, value: Any) -> "Property":  # type: ignore
-        pass
-
-    def get_id() -> "VertexId":  # type: ignore
-        pass
-
-    def label_at(self, index: int) -> "Label":  # type: ignore
-        pass
-
-    def labels_count() -> int:  # type: ignore
-        pass
-
-    def add_label(self, label: Any):
-        pass
-
-    def remove_label(self, label: Any):
-        pass
-
-    def iter_in_edges() -> MgpIterable:  # type: ignore
-        pass
-
-    def iter_out_edges() -> MgpIterable:  # type: ignore
-        pass
-
-
-class Edge:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-    def underlying_graph_is_mutable() -> bool:  # type: ignore
-        pass
-
-    def iter_properties() -> MgpIterable:  # type: ignore
-        pass
-
-    def get_property(self, property_name: str) -> "Property":  # type: ignore
-        pass
-
-    def set_property(self, property_name: str, valuse: Any) -> "Property":  # type: ignore
-        pass
-
-    def get_type_name() -> str:  # type: ignore
-        pass
-
-    def get_id() -> "EdgeId":  # type: ignore
-        pass
-
-    def from_vertex() -> Vertex:  # type: ignore
-        pass
-
-    def to_vertex() -> Vertex:  # type: ignore
-        pass
-
-
-class Path:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-    @staticmethod
-    def make_with_start(vertex: Vertex) -> "Path":  # type: ignore
-        pass
-
-
-class Graph:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-
-class CypherType:
-    pass
-
-
-class Message:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-    def source_type() -> str:  # type: ignore
-        pass
-
-    def topic_name() -> str:  # type: ignore
-        pass
-
-    def key() -> bytes:  # type: ignore
-        pass
-
-    def timestamp() -> int:  # type: ignore
-        pass
-
-    def offset() -> int:  # type: ignore
-        pass
-
-    def payload() -> bytes:  # type: ignore
-        pass
-
-
-class Messages:
-    def is_valid() -> bool:  # type: ignore
-        pass
-
-    def message_at(self, id: int) -> Message:  # type: ignore
-        pass
-
-    def total_messages() -> int:  # type: ignore
-        pass
-
-
-class UnknownError(Exception):
-    pass
-
-
-class UnableToAllocateError(Exception):
-    pass
-
-
-class InsufficientBufferError(Exception):
-    pass
-
-
-class OutOfRangeError(Exception):
-    pass
-
-
-class LogicErrorError(Exception):
-
-    pass
-
-
-class DeletedObjectError(Exception):
-    pass
-
-
-class InvalidArgumentError(Exception):
-    pass
-
-
-class KeyAlreadyExistsError(Exception):
-    pass
-
-
-class ImmutableObjectError(Exception):
-    pass
-
-
-class ValueConversionError(Exception):
-    pass
-
-
-class SerializationError(Exception):
-    pass
-
-
-def type_nullable(elem: Any):
-    pass
-
-
-def type_list(elem: Any):
-    pass
-
-
-def type_bool():
-    pass
-
-
-def type_string():
-    pass
-
-
-def type_int():
-    pass
-
-
-def type_float():
-    pass
-
-
-def type_number():
-    pass
-
-
-def type_map():
-    pass
-
-
-def type_node():
-    pass
-
-
-def type_relationship():
-    pass
-
-
-def type_path():
-    pass
-
-
-def type_date():
-    pass
-
-
-def type_local_time():
-    pass
-
-
-def type_local_date_time():
-    pass
-
-
-def type_duration():
-    pass
-
-
-def type_any():
-    pass
-
-
-class _MODULE:
-    @staticmethod
-    def add_read_procedure(wrapper):
-        pass
-
-    @staticmethod
-    def add_write_procedure(wrapper):
-        pass
-
-    @staticmethod
-    def add_transformation(wrapper):
-        pass
-
-    @staticmethod
-    def add_function(wrapper):
-        pass

release/mgp/instructions.md

@@ -1,22 +0,0 @@
-# How to publish new versions
-## Prerequisites
-1. Installed poetry
-```
-pip install poetry
-```
-2. Set up [API tokens](https://pypi.org/help/#apitoken)
-3. Be a collaborator on [pypi](https://pypi.org/project/mgp/)
-
-## Making changes
-1. Make changes to the package
-2. Bump version in `pyproject.tml`
-3. `poetry build`
-4. `poetry publish`
-
-## Why is this not automatized?
-
-Because someone always has to manually bump up the version in `pyproject.toml`
-
-## Why does `_mgp.py` exists?
-Because we are mocking here all the types that are created by Memgraph
-in order to fix typing errors in `mgp.py`.

release/mgp/pyproject.toml

@@ -1,23 +0,0 @@
-[tool.poetry]
-name = "mgp"
-version = "1.0.0"
-description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
-authors = [
-    "MasterMedo <mislav.vuletic@gmail.com>",
-    "jbajic <jure.bajic@memgraph.io>",
-    "katarinasupe <katarina.supe@memgraph.io>",
-    "antejavor <ante.javor@memgraph.io>",
-    "antaljanosbenjamin <benjamin.antal@memgraph.io>",
-]
-license = "Apache-2.0"
-readme = "README.md"
-include = ["mgp.py", "_mgp.py"]
-
-[tool.poetry.dependencies]
-python = "^3.7"
-
-[tool.poetry.dev-dependencies]
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"

release/package/centos-7/Dockerfile

@@ -7,8 +7,8 @@ RUN yum -y update \
 # Do NOT be smart here and clean the cache because the container is used in the
 # stateful context.
 
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz -C /opt
 
 ENTRYPOINT ["sleep", "infinity"]

release/package/centos-8/Dockerfile

@@ -0,0 +1,14 @@
+FROM centos:8
+
+ARG TOOLCHAIN_VERSION
+
+RUN dnf -y update \
+    && dnf install -y wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/centos-9/Dockerfile

@@ -1,14 +0,0 @@
-FROM quay.io/centos/centos:stream9
-
-ARG TOOLCHAIN_VERSION
-
-RUN yum -y update \
-    && yum install -y wget git
-# Do NOT be smart here and clean the cache because the container is used in the
-# stateful context.
-
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt
-
-ENTRYPOINT ["sleep", "infinity"]

release/package/debian-10/Dockerfile

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
 # Do NOT be smart here and clean the cache because the container is used in the
 # stateful context.
 
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz -C /opt
 
 ENTRYPOINT ["sleep", "infinity"]

release/package/debian-11-arm/Dockerfile

@@ -1,17 +0,0 @@
-FROM debian:11
-
-ARG TOOLCHAIN_VERSION
-
-# Stops tzdata interactive configuration.
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt update && apt install -y \
-    ca-certificates wget git
-# Do NOT be smart here and clean the cache because the container is used in the
-# stateful context.
-
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt
-
-ENTRYPOINT ["sleep", "infinity"]

release/package/debian-11/Dockerfile

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
 # Do NOT be smart here and clean the cache because the container is used in the
 # stateful context.
 
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz -C /opt
 
 ENTRYPOINT ["sleep", "infinity"]

release/package/docker-compose.yml

@@ -4,10 +4,10 @@ services:
     build:
       context: centos-7
     container_name: "mgbuild_centos-7"
-  mgbuild_centos-9:
+  mgbuild_centos-8:
     build:
-      context: centos-9
-    container_name: "mgbuild_centos-9"
+      context: centos-8
+    container_name: "mgbuild_centos-8"
   mgbuild_debian-10:
     build:
       context: debian-10
@@ -24,7 +24,3 @@ services:
     build:
       context: ubuntu-20.04
     container_name: "mgbuild_ubuntu-20.04"
-  mgbuild_ubuntu-22.04:
-    build:
-      context: ubuntu-22.04
-    container_name: "mgbuild_ubuntu-22.04"

release/package/run.sh

@@ -3,7 +3,7 @@
 set -Eeuo pipefail
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm)
+SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
 PROJECT_ROOT="$SCRIPT_DIR/../.."
 TOOLCHAIN_VERSION="toolchain-v4"
 ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
@@ -67,18 +67,14 @@ make_package () {
     # environment/os/{os}.sh does not come within the toolchain package. When
     # migrating to the next version of toolchain do that, and remove the
     # TOOLCHAIN_RUN_DEPS installation from here.
-    echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
+    echo "Installing dependencies..."
     docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
     docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
 
     echo "Building targeted package..."
     docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
     docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
-    if [[ "$os" == "debian-11-arm" ]]; then
-        docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
-    else
-        docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
-    fi
+    docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
     # ' is used instead of " because we need to run make within the allowed
     # container resources.
     # shellcheck disable=SC2016

release/package/ubuntu-18.04/Dockerfile

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
 # Do NOT be smart here and clean the cache because the container is used in the
 # stateful context.
 
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz -C /opt
 
 ENTRYPOINT ["sleep", "infinity"]

release/package/ubuntu-20.04/Dockerfile

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
 # Do NOT be smart here and clean the cache because the container is used in the
 # stateful context.
 
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
+    -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
+    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz -C /opt
 
 ENTRYPOINT ["sleep", "infinity"]

release/package/ubuntu-22.04/Dockerfile

@@ -1,17 +0,0 @@
-FROM ubuntu:22.04
-
-ARG TOOLCHAIN_VERSION
-
-# Stops tzdata interactive configuration.
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt update && apt install -y \
-    ca-certificates wget git
-# Do NOT be smart here and clean the cache because the container is used in the
-# stateful context.
-
-RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
-    -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
-    && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt
-
-ENTRYPOINT ["sleep", "infinity"]

src/CMakeLists.txt

@@ -1,24 +1,21 @@
 # CMake configuration for the main memgraph library and executable
 
 # add memgraph sub libraries, ordered by dependency
+add_subdirectory(interface)
 add_subdirectory(lisp)
 add_subdirectory(utils)
 add_subdirectory(requests)
 add_subdirectory(io)
-add_subdirectory(io/simulator)
 add_subdirectory(kvstore)
 add_subdirectory(telemetry)
 add_subdirectory(communication)
 add_subdirectory(memory)
 add_subdirectory(storage/v2)
-add_subdirectory(storage/v3)
 add_subdirectory(integrations)
 add_subdirectory(query)
-add_subdirectory(query/v2)
 add_subdirectory(slk)
 add_subdirectory(rpc)
 add_subdirectory(auth)
-add_subdirectory(coordinator)
 
 if (MG_ENTERPRISE)
   add_subdirectory(audit)
@@ -41,7 +38,7 @@ set(mg_single_node_v2_sources
 )
 
 set(mg_single_node_v2_libs stdc++fs Threads::Threads
-    telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
+    telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-interface-storage-cpp2)
 if (MG_ENTERPRISE)
   # These are enterprise subsystems
   set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)

src/auth/models.cpp

@@ -37,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
     Permission::CONSTRAINT, Permission::DUMP,      Permission::AUTH,        Permission::REPLICATION,
     Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
     Permission::CONFIG,     Permission::STREAM,    Permission::MODULE_READ, Permission::MODULE_WRITE,
-    Permission::WEBSOCKET,  Permission::SCHEMA};
+    Permission::WEBSOCKET};
 }  // namespace
 
 std::string PermissionToString(Permission permission) {
@@ -84,8 +84,6 @@ std::string PermissionToString(Permission permission) {
       return "MODULE_WRITE";
     case Permission::WEBSOCKET:
       return "WEBSOCKET";
-    case Permission::SCHEMA:
-      return "SCHEMA";
   }
 }
 

src/auth/models.hpp

@@ -38,8 +38,7 @@ enum class Permission : uint64_t {
   STREAM       = 1U << 17U,
   MODULE_READ  = 1U << 18U,
   MODULE_WRITE = 1U << 19U,
-  WEBSOCKET    = 1U << 20U,
-  SCHEMA       = 1U << 21U
+  WEBSOCKET    = 1U << 20U
 };
 // clang-format on
 

src/common/types.hpp

@@ -1,19 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <cstdint>
-
-namespace memgraph::common {
-enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
-
-}  // namespace memgraph::common

src/communication/bolt/v1/session.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/communication/bolt/v1/states/error.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/communication/bolt/v1/states/executing.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/communication/bolt/v1/states/handshake.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/communication/bolt/v1/states/init.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/communication/v2/session.hpp

@@ -105,16 +105,9 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
     boost::asio::socket_base::keep_alive option(true);
 
     // Set a decorator to change the Server of the handshake
-    ws_.set_option(boost::beast::websocket::stream_base::decorator([&req](boost::beast::websocket::response_type &res) {
+    ws_.set_option(boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::response_type &res) {
       res.set(boost::beast::http::field::server, std::string("Memgraph Bolt WS"));
-
-      // We need to do this to support WASM clients, which explicitly send this flag
-      // in their upgrade request
-      // Neo4j client breaks when this flag is sent
-      if (const auto secondary_protocol = req.base().find(boost::beast::http::field::sec_websocket_protocol);
-          secondary_protocol != res.base().end() && secondary_protocol->value() == "binary") {
-        res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
-      }
+      res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
     }));
     ws_.binary(true);
 
@@ -169,7 +162,7 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
         boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnRead, shared_from_this())));
   }
 
-  void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
+  void OnRead(const boost::system::error_code &ec, [[maybe_unused]] const size_t bytes_transferred) {
     // This indicates that the WebsocketSession was closed
     if (ec == boost::beast::websocket::error::closed) {
       return;
@@ -329,8 +322,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
       socket.lowest_layer().non_blocking(false);
     });
     timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
-    spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
-                 remote_endpoint_.port());
+    spdlog::info("Accepted a connection from {}:", service_name_, remote_endpoint_.address(), remote_endpoint_.port());
   }
 
   void DoRead() {

src/coordinator/CMakeLists.txt

@@ -1,10 +0,0 @@
-set(coordinator_src_files
-    coordinator.hpp
-    shard_map.hpp
-    hybrid_logical_clock.hpp)
-
-find_package(fmt REQUIRED)
-find_package(Threads REQUIRED)
-
-add_library(mg-coordinator STATIC ${coordinator_src_files})
-target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils)

src/coordinator/coordinator.hpp

@@ -1,243 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <unordered_set>
-
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "coordinator/shard_map.hpp"
-#include "io/simulator/simulator.hpp"
-#include "io/time.hpp"
-#include "io/transport.hpp"
-
-namespace memgraph::coordinator {
-
-using Address = memgraph::io::Address;
-using SimT = memgraph::io::simulator::SimulatorTransport;
-
-struct HlcRequest {
-  Hlc last_shard_map_version;
-};
-
-struct HlcResponse {
-  Hlc new_hlc;
-  std::optional<ShardMap> fresher_shard_map;
-};
-
-struct GetShardMapRequest {
-  // No state
-};
-
-struct GetShardMapResponse {
-  ShardMap shard_map;
-};
-
-struct AllocateHlcBatchRequest {
-  Hlc low;
-  Hlc high;
-};
-
-struct AllocateHlcBatchResponse {
-  bool success;
-  Hlc low;
-  Hlc high;
-};
-
-struct AllocateEdgeIdBatchRequest {
-  size_t batch_size;
-};
-
-struct AllocateEdgeIdBatchResponse {
-  uint64_t low;
-  uint64_t high;
-};
-
-struct SplitShardRequest {
-  Hlc previous_shard_map_version;
-  Label label;
-  CompoundKey split_key;
-};
-
-struct SplitShardResponse {
-  bool success;
-};
-
-struct RegisterStorageEngineRequest {
-  Address address;
-};
-
-struct RegisterStorageEngineResponse {
-  bool success;
-};
-
-struct DeregisterStorageEngineRequest {
-  Address address;
-};
-
-struct DeregisterStorageEngineResponse {
-  bool success;
-};
-
-struct InitializeLabelRequest {
-  std::string label_name;
-  Hlc last_shard_map_version;
-};
-
-struct InitializeLabelResponse {
-  bool success;
-  std::optional<ShardMap> fresher_shard_map;
-};
-
-using WriteRequests =
-    std::variant<AllocateHlcBatchRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
-                 DeregisterStorageEngineRequest, InitializeLabelRequest>;
-using WriteResponses =
-    std::variant<AllocateHlcBatchResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
-                 RegisterStorageEngineResponse, DeregisterStorageEngineResponse, InitializeLabelResponse>;
-
-using ReadRequests = std::variant<HlcRequest, GetShardMapRequest>;
-using ReadResponses = std::variant<HlcResponse, GetShardMapResponse>;
-
-class Coordinator {
-  ShardMap shard_map_;
-  /// The highest reserved timestamp / highest allocated timestamp
-  /// is a way for minimizing communication involved in query engines
-  /// reserving Hlc's for their transaction processing.
-  /// Periodically, the coordinator will allocate a batch of timestamps
-  /// and this will need to go over consensus. From that point forward,
-  /// each timestamp in that batch can be given out to "readers" who issue
-  /// HlcRequest without blocking on consensus first. But if
-  /// highest_allocated_timestamp_ approaches highest_reserved_timestamp_,
-  /// it is time to allocate another batch, so that we can keep guaranteeing
-  /// forward progress.
-  /// Any time a coordinator becomes a new leader, it will need to issue
-  /// a new AllocateHlcBatchRequest to create a pool of IDs to allocate.
-  uint64_t highest_allocated_timestamp_;
-  uint64_t highest_reserved_timestamp_;
-
-  /// Query engines need to periodically request batches of unique edge IDs.
-  uint64_t highest_allocated_edge_id_;
-
-  /// Increment our
-  ReadResponses HandleRead(HlcRequest &&hlc_request) {
-    HlcResponse res{};
-
-    auto hlc_shard_map = shard_map_.GetHlc();
-
-    MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
-
-    res.new_hlc = shard_map_.IncrementShardMapVersion();
-
-    // res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id
-    //                             ? std::make_optional(shard_map_)
-    //                             : std::nullopt;
-
-    // Allways return fresher shard_map for now.
-    res.fresher_shard_map = std::make_optional(shard_map_);
-
-    return res;
-  }
-
-  ReadResponses HandleRead(GetShardMapRequest &&get_shard_map_request) {
-    GetShardMapResponse res;
-    res.shard_map = shard_map_;
-    return res;
-  }
-
-  WriteResponses ApplyWrite(AllocateHlcBatchRequest &&ahr) {
-    AllocateHlcBatchResponse res{};
-
-    return res;
-  }
-
-  WriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
-    AllocateEdgeIdBatchResponse res{};
-
-    uint64_t low = highest_allocated_edge_id_;
-
-    highest_allocated_edge_id_ += ahr.batch_size;
-
-    uint64_t high = highest_allocated_edge_id_;
-
-    res.low = low;
-    res.high = high;
-
-    return res;
-  }
-
-  /// This splits the shard immediately beneath the provided
-  /// split key, keeping the assigned peers identical for now,
-  /// but letting them be gradually migrated over time.
-  WriteResponses ApplyWrite(SplitShardRequest &&split_shard_request) {
-    SplitShardResponse res{};
-
-    if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
-      res.success = false;
-    } else {
-      res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label,
-                                          split_shard_request.split_key);
-    }
-
-    return res;
-  }
-
-  /// This adds the provided storage engine to the standby storage engine pool,
-  /// which can be used to rebalance storage over time.
-  WriteResponses ApplyWrite(RegisterStorageEngineRequest &&register_storage_engine_request) {
-    RegisterStorageEngineResponse res{};
-    // TODO
-
-    return res;
-  }
-
-  /// This begins the process of draining the provided storage engine from all raft
-  /// clusters that it might be participating in.
-  WriteResponses ApplyWrite(DeregisterStorageEngineRequest &&register_storage_engine_request) {
-    DeregisterStorageEngineResponse res{};
-    // TODO
-    // const Address &address = register_storage_engine_request.address;
-    // storage_engine_pool_.erase(address);
-    // res.success = true;
-
-    return res;
-  }
-
-  WriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
-    InitializeLabelResponse res{};
-
-    bool success = shard_map_.InitializeNewLabel(initialize_label_request.label_name,
-                                                 initialize_label_request.last_shard_map_version);
-
-    if (success) {
-      res.fresher_shard_map = shard_map_;
-      res.success = false;
-    } else {
-      res.fresher_shard_map = std::nullopt;
-      res.success = true;
-    }
-
-    return res;
-  }
-
- public:
-  explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {}
-
-  ReadResponses Read(ReadRequests requests) {
-    return std::visit([&](auto &&requests) { return HandleRead(std::move(requests)); }, std::move(requests));
-  }
-
-  WriteResponses Apply(WriteRequests requests) {
-    return std::visit([&](auto &&requests) { return ApplyWrite(std::move(requests)); }, std::move(requests));
-  }
-};
-
-}  // namespace memgraph::coordinator

src/coordinator/hybrid_logical_clock.hpp

@@ -1,30 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include "io/time.hpp"
-
-namespace memgraph::coordinator {
-
-using Time = memgraph::io::Time;
-
-/// Hybrid-logical clock
-struct Hlc {
-  uint64_t logical_id;
-  Time coordinator_wall_clock;
-
-  bool operator==(const Hlc &other) const {
-    return (logical_id == other.logical_id) && (coordinator_wall_clock == other.coordinator_wall_clock);
-  }
-};
-
-}  // namespace memgraph::coordinator

src/coordinator/shard_map.hpp

@@ -1,135 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <map>
-#include <vector>
-
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "io/address.hpp"
-#include "storage/v3/property_value.hpp"
-
-namespace memgraph::coordinator {
-
-enum class Status : uint8_t {
-  CONSENSUS_PARTICIPANT,
-  INITIALIZING,
-  // TODO(tyler) this will possibly have more states,
-  // depending on the reconfiguration protocol that we
-  // implement.
-};
-
-struct AddressAndStatus {
-  memgraph::io::Address address;
-  Status status;
-};
-
-using memgraph::io::Address;
-
-using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
-using Shard = std::vector<AddressAndStatus>;
-using Shards = std::map<CompoundKey, Shard>;
-
-// use string for intermachine communication and NameIdMapper within the machine
-using Label = std::string;
-
-struct ShardMap {
-  Hlc shard_map_version;
-  std::map<Label, Shards> shards;
-
-  // TODO(gabor) later we will want to update the wallclock time with
-  // the given Io<impl>'s time as well
-  Hlc IncrementShardMapVersion() noexcept {
-    ++shard_map_version.logical_id;
-    return shard_map_version;
-  }
-
-  Hlc GetHlc() const noexcept { return shard_map_version; }
-
-  bool SplitShard(Hlc previous_shard_map_version, Label label, CompoundKey key) {
-    if (CompareShardMapVersions(previous_shard_map_version, shard_map_version)) {
-      MG_ASSERT(shards.contains(label));
-      auto &shards_in_map = shards[label];
-      MG_ASSERT(!shards_in_map.contains(key));
-
-      // Finding the Shard that the new CompoundKey should map to.
-      Shard shard_to_map_to;
-      CompoundKey prev_key = ((*shards_in_map.begin()).first);
-
-      for (auto iter = std::next(shards_in_map.begin()); iter != shards_in_map.end(); ++iter) {
-        const auto &current_key = (*iter).first;
-        if (key > prev_key && key < current_key) {
-          shard_to_map_to = shards_in_map[prev_key];
-        }
-
-        prev_key = (*iter).first;
-      }
-
-      // Apply the split
-      shards_in_map[key] = shard_to_map_to;
-
-      return true;
-    }
-
-    return false;
-  }
-
-  bool InitializeNewLabel(std::string label_name, Hlc last_shard_map_version) {
-    if (shard_map_version != last_shard_map_version) {
-      return false;
-    }
-
-    if (shards.contains(label_name)) {
-      return false;
-    }
-
-    shards.emplace(label_name, Shards{});
-
-    IncrementShardMapVersion();
-
-    return true;
-  }
-
-  void AddServer(Address server_address) {
-    // Find a random place for the server to plug in
-  }
-
-  std::map<Label, Shards> &GetShards() noexcept { return shards; }
-
-  Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end);
-
-  Shard GetShardForKey(Label label, CompoundKey key) {
-    auto shard_for_label = shards.at(label);
-
-    auto max = (--shard_for_label.end())->first;
-
-    if (key > max) {
-      return shard_for_label[max];
-    }
-
-    for (auto it = shard_for_label.lower_bound(key);; --it) {
-      MG_ASSERT(it->first <= key);
-      return it->second;
-    }
-
-    MG_ASSERT(false, "failed to find shard with a key that is less than or equal to the provided key");
-  }
-
- private:
-  // TODO(gabor) later we will want to update the wallclock time with
-  // the given Io<impl>'s time as well. This function should just be
-  // replaced with operator== since it is already overloaded for Hlc
-  // objects.
-  bool CompareShardMapVersions(Hlc one, Hlc two) { return one.logical_id == two.logical_id; }
-};
-
-}  // namespace memgraph::coordinator

src/glue/v2/auth.cpp

@@ -1,64 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "glue/v2/auth.hpp"
-
-namespace memgraph::glue::v2 {
-
-auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege) {
-  switch (privilege) {
-    case query::v2::AuthQuery::Privilege::MATCH:
-      return auth::Permission::MATCH;
-    case query::v2::AuthQuery::Privilege::CREATE:
-      return auth::Permission::CREATE;
-    case query::v2::AuthQuery::Privilege::MERGE:
-      return auth::Permission::MERGE;
-    case query::v2::AuthQuery::Privilege::DELETE:
-      return auth::Permission::DELETE;
-    case query::v2::AuthQuery::Privilege::SET:
-      return auth::Permission::SET;
-    case query::v2::AuthQuery::Privilege::REMOVE:
-      return auth::Permission::REMOVE;
-    case query::v2::AuthQuery::Privilege::INDEX:
-      return auth::Permission::INDEX;
-    case query::v2::AuthQuery::Privilege::STATS:
-      return auth::Permission::STATS;
-    case query::v2::AuthQuery::Privilege::CONSTRAINT:
-      return auth::Permission::CONSTRAINT;
-    case query::v2::AuthQuery::Privilege::DUMP:
-      return auth::Permission::DUMP;
-    case query::v2::AuthQuery::Privilege::REPLICATION:
-      return auth::Permission::REPLICATION;
-    case query::v2::AuthQuery::Privilege::DURABILITY:
-      return auth::Permission::DURABILITY;
-    case query::v2::AuthQuery::Privilege::READ_FILE:
-      return auth::Permission::READ_FILE;
-    case query::v2::AuthQuery::Privilege::FREE_MEMORY:
-      return auth::Permission::FREE_MEMORY;
-    case query::v2::AuthQuery::Privilege::TRIGGER:
-      return auth::Permission::TRIGGER;
-    case query::v2::AuthQuery::Privilege::CONFIG:
-      return auth::Permission::CONFIG;
-    case query::v2::AuthQuery::Privilege::AUTH:
-      return auth::Permission::AUTH;
-    case query::v2::AuthQuery::Privilege::STREAM:
-      return auth::Permission::STREAM;
-    case query::v2::AuthQuery::Privilege::MODULE_READ:
-      return auth::Permission::MODULE_READ;
-    case query::v2::AuthQuery::Privilege::MODULE_WRITE:
-      return auth::Permission::MODULE_WRITE;
-    case query::v2::AuthQuery::Privilege::WEBSOCKET:
-      return auth::Permission::WEBSOCKET;
-    case query::v2::AuthQuery::Privilege::SCHEMA:
-      return auth::Permission::SCHEMA;
-  }
-}
-}  // namespace memgraph::glue::v2

src/glue/v2/auth.hpp

@@ -1,23 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "auth/models.hpp"
-#include "query/v2/frontend/ast/ast.hpp"
-
-namespace memgraph::glue::v2 {
-
-/**
- * This function converts query::AuthQuery::Privilege to its corresponding
- * auth::Permission.
- */
-auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege);
-
-}  // namespace memgraph::glue::v2

src/glue/v2/communication.cpp

@@ -1,275 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "glue/v2/communication.hpp"
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "storage/v3/edge_accessor.hpp"
-#include "storage/v3/storage.hpp"
-#include "storage/v3/vertex_accessor.hpp"
-#include "utils/temporal.hpp"
-
-using memgraph::communication::bolt::Value;
-
-namespace memgraph::glue::v2 {
-
-query::v2::TypedValue ToTypedValue(const Value &value) {
-  switch (value.type()) {
-    case Value::Type::Null:
-      return {};
-    case Value::Type::Bool:
-      return query::v2::TypedValue(value.ValueBool());
-    case Value::Type::Int:
-      return query::v2::TypedValue(value.ValueInt());
-    case Value::Type::Double:
-      return query::v2::TypedValue(value.ValueDouble());
-    case Value::Type::String:
-      return query::v2::TypedValue(value.ValueString());
-    case Value::Type::List: {
-      std::vector<query::v2::TypedValue> list;
-      list.reserve(value.ValueList().size());
-      for (const auto &v : value.ValueList()) list.push_back(ToTypedValue(v));
-      return query::v2::TypedValue(std::move(list));
-    }
-    case Value::Type::Map: {
-      std::map<std::string, query::v2::TypedValue> map;
-      for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
-      return query::v2::TypedValue(std::move(map));
-    }
-    case Value::Type::Vertex:
-    case Value::Type::Edge:
-    case Value::Type::UnboundedEdge:
-    case Value::Type::Path:
-      throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
-    case Value::Type::Date:
-      return query::v2::TypedValue(value.ValueDate());
-    case Value::Type::LocalTime:
-      return query::v2::TypedValue(value.ValueLocalTime());
-    case Value::Type::LocalDateTime:
-      return query::v2::TypedValue(value.ValueLocalDateTime());
-    case Value::Type::Duration:
-      return query::v2::TypedValue(value.ValueDuration());
-  }
-}
-
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
-                                                              const storage::v3::Storage &db, storage::v3::View view) {
-  return ToBoltVertex(vertex.impl_, db, view);
-}
-
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
-                                                          const storage::v3::Storage &db, storage::v3::View view) {
-  return ToBoltEdge(edge.impl_, db, view);
-}
-
-storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Storage &db,
-                                       storage::v3::View view) {
-  switch (value.type()) {
-    case query::v2::TypedValue::Type::Null:
-      return Value();
-    case query::v2::TypedValue::Type::Bool:
-      return Value(value.ValueBool());
-    case query::v2::TypedValue::Type::Int:
-      return Value(value.ValueInt());
-    case query::v2::TypedValue::Type::Double:
-      return Value(value.ValueDouble());
-    case query::v2::TypedValue::Type::String:
-      return Value(std::string(value.ValueString()));
-    case query::v2::TypedValue::Type::List: {
-      std::vector<Value> values;
-      values.reserve(value.ValueList().size());
-      for (const auto &v : value.ValueList()) {
-        auto maybe_value = ToBoltValue(v, db, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
-        values.emplace_back(std::move(*maybe_value));
-      }
-      return Value(std::move(values));
-    }
-    case query::v2::TypedValue::Type::Map: {
-      std::map<std::string, Value> map;
-      for (const auto &kv : value.ValueMap()) {
-        auto maybe_value = ToBoltValue(kv.second, db, view);
-        if (maybe_value.HasError()) return maybe_value.GetError();
-        map.emplace(kv.first, std::move(*maybe_value));
-      }
-      return Value(std::move(map));
-    }
-    case query::v2::TypedValue::Type::Vertex: {
-      auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
-      if (maybe_vertex.HasError()) return maybe_vertex.GetError();
-      return Value(std::move(*maybe_vertex));
-    }
-    case query::v2::TypedValue::Type::Edge: {
-      auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
-      if (maybe_edge.HasError()) return maybe_edge.GetError();
-      return Value(std::move(*maybe_edge));
-    }
-    case query::v2::TypedValue::Type::Path: {
-      auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
-      if (maybe_path.HasError()) return maybe_path.GetError();
-      return Value(std::move(*maybe_path));
-    }
-    case query::v2::TypedValue::Type::Date:
-      return Value(value.ValueDate());
-    case query::v2::TypedValue::Type::LocalTime:
-      return Value(value.ValueLocalTime());
-    case query::v2::TypedValue::Type::LocalDateTime:
-      return Value(value.ValueLocalDateTime());
-    case query::v2::TypedValue::Type::Duration:
-      return Value(value.ValueDuration());
-  }
-}
-
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
-                                                              const storage::v3::Storage &db, storage::v3::View view) {
-  auto id = communication::bolt::Id::FromUint(vertex.Gid().AsUint());
-  auto maybe_labels = vertex.Labels(view);
-  if (maybe_labels.HasError()) return maybe_labels.GetError();
-  std::vector<std::string> labels;
-  labels.reserve(maybe_labels->size());
-  for (const auto &label : *maybe_labels) {
-    labels.push_back(db.LabelToName(label));
-  }
-  auto maybe_properties = vertex.Properties(view);
-  if (maybe_properties.HasError()) return maybe_properties.GetError();
-  std::map<std::string, Value> properties;
-  for (const auto &prop : *maybe_properties) {
-    properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
-  }
-  return communication::bolt::Vertex{id, labels, properties};
-}
-
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
-                                                          const storage::v3::Storage &db, storage::v3::View view) {
-  auto id = communication::bolt::Id::FromUint(edge.Gid().AsUint());
-  auto from = communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
-  auto to = communication::bolt::Id::FromUint(edge.ToVertex().Gid().AsUint());
-  const auto &type = db.EdgeTypeToName(edge.EdgeType());
-  auto maybe_properties = edge.Properties(view);
-  if (maybe_properties.HasError()) return maybe_properties.GetError();
-  std::map<std::string, Value> properties;
-  for (const auto &prop : *maybe_properties) {
-    properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
-  }
-  return communication::bolt::Edge{id, from, to, type, properties};
-}
-
-storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
-                                                          storage::v3::View view) {
-  std::vector<communication::bolt::Vertex> vertices;
-  vertices.reserve(path.vertices().size());
-  for (const auto &v : path.vertices()) {
-    auto maybe_vertex = ToBoltVertex(v, db, view);
-    if (maybe_vertex.HasError()) return maybe_vertex.GetError();
-    vertices.emplace_back(std::move(*maybe_vertex));
-  }
-  std::vector<communication::bolt::Edge> edges;
-  edges.reserve(path.edges().size());
-  for (const auto &e : path.edges()) {
-    auto maybe_edge = ToBoltEdge(e, db, view);
-    if (maybe_edge.HasError()) return maybe_edge.GetError();
-    edges.emplace_back(std::move(*maybe_edge));
-  }
-  return communication::bolt::Path(vertices, edges);
-}
-
-storage::v3::PropertyValue ToPropertyValue(const Value &value) {
-  switch (value.type()) {
-    case Value::Type::Null:
-      return {};
-    case Value::Type::Bool:
-      return storage::v3::PropertyValue(value.ValueBool());
-    case Value::Type::Int:
-      return storage::v3::PropertyValue(value.ValueInt());
-    case Value::Type::Double:
-      return storage::v3::PropertyValue(value.ValueDouble());
-    case Value::Type::String:
-      return storage::v3::PropertyValue(value.ValueString());
-    case Value::Type::List: {
-      std::vector<storage::v3::PropertyValue> vec;
-      vec.reserve(value.ValueList().size());
-      for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
-      return storage::v3::PropertyValue(std::move(vec));
-    }
-    case Value::Type::Map: {
-      std::map<std::string, storage::v3::PropertyValue> map;
-      for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
-      return storage::v3::PropertyValue(std::move(map));
-    }
-    case Value::Type::Vertex:
-    case Value::Type::Edge:
-    case Value::Type::UnboundedEdge:
-    case Value::Type::Path:
-      throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
-    case Value::Type::Date:
-      return storage::v3::PropertyValue(
-          storage::v3::TemporalData(storage::v3::TemporalType::Date, value.ValueDate().MicrosecondsSinceEpoch()));
-    case Value::Type::LocalTime:
-      return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalTime,
-                                                                  value.ValueLocalTime().MicrosecondsSinceEpoch()));
-    case Value::Type::LocalDateTime:
-      return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalDateTime,
-                                                                  value.ValueLocalDateTime().MicrosecondsSinceEpoch()));
-    case Value::Type::Duration:
-      return storage::v3::PropertyValue(
-          storage::v3::TemporalData(storage::v3::TemporalType::Duration, value.ValueDuration().microseconds));
-  }
-}
-
-Value ToBoltValue(const storage::v3::PropertyValue &value) {
-  switch (value.type()) {
-    case storage::v3::PropertyValue::Type::Null:
-      return {};
-    case storage::v3::PropertyValue::Type::Bool:
-      return {value.ValueBool()};
-    case storage::v3::PropertyValue::Type::Int:
-      return {value.ValueInt()};
-      break;
-    case storage::v3::PropertyValue::Type::Double:
-      return {value.ValueDouble()};
-    case storage::v3::PropertyValue::Type::String:
-      return {value.ValueString()};
-    case storage::v3::PropertyValue::Type::List: {
-      const auto &values = value.ValueList();
-      std::vector<Value> vec;
-      vec.reserve(values.size());
-      for (const auto &v : values) {
-        vec.push_back(ToBoltValue(v));
-      }
-      return {std::move(vec)};
-    }
-    case storage::v3::PropertyValue::Type::Map: {
-      const auto &map = value.ValueMap();
-      std::map<std::string, Value> dv_map;
-      for (const auto &kv : map) {
-        dv_map.emplace(kv.first, ToBoltValue(kv.second));
-      }
-      return {std::move(dv_map)};
-    }
-    case storage::v3::PropertyValue::Type::TemporalData:
-      const auto &type = value.ValueTemporalData();
-      switch (type.type) {
-        case storage::v3::TemporalType::Date:
-          return {utils::Date(type.microseconds)};
-        case storage::v3::TemporalType::LocalTime:
-          return {utils::LocalTime(type.microseconds)};
-        case storage::v3::TemporalType::LocalDateTime:
-          return {utils::LocalDateTime(type.microseconds)};
-        case storage::v3::TemporalType::Duration:
-          return {utils::Duration(type.microseconds)};
-      }
-  }
-}
-
-}  // namespace memgraph::glue::v2

src/glue/v2/communication.hpp

@@ -1,68 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-/// @file Conversion functions between Value and other memgraph types.
-#pragma once
-
-#include "communication/bolt/v1/value.hpp"
-#include "query/v2/typed_value.hpp"
-#include "storage/v3/property_value.hpp"
-#include "storage/v3/result.hpp"
-#include "storage/v3/view.hpp"
-
-namespace memgraph::storage::v3 {
-class EdgeAccessor;
-class Storage;
-class VertexAccessor;
-}  // namespace memgraph::storage::v3
-
-namespace memgraph::glue::v2 {
-
-/// @param storage::v3::VertexAccessor for converting to
-///        communication::bolt::Vertex.
-/// @param storage::v3::Storage for getting label and property names.
-/// @param storage::v3::View for deciding which vertex attributes are visible.
-///
-/// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
-                                                              const storage::v3::Storage &db, storage::v3::View view);
-
-/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
-/// @param storage::v3::Storage for getting edge type and property names.
-/// @param storage::v3::View for deciding which edge attributes are visible.
-///
-/// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
-                                                          const storage::v3::Storage &db, storage::v3::View view);
-
-/// @param query::v2::Path for converting to communication::bolt::Path.
-/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
-/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
-///
-/// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
-                                                          storage::v3::View view);
-
-/// @param query::v2::TypedValue for converting to communication::bolt::Value.
-/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
-/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
-///
-/// @throw std::bad_alloc
-storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
-                                                            const storage::v3::Storage &db, storage::v3::View view);
-
-query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
-
-communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
-
-storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
-
-}  // namespace memgraph::glue::v2

src/integrations/constants.hpp

@@ -17,7 +17,6 @@
 namespace memgraph::integrations {
 
 inline constexpr int64_t kDefaultCheckBatchLimit{1};
-inline constexpr int64_t kMinimumStartBatchLimit{1};
 inline constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
 inline constexpr std::chrono::milliseconds kMinimumInterval{1};
 inline constexpr int64_t kMinimumSize{1};

src/integrations/kafka/consumer.cpp

@@ -74,36 +74,6 @@ utils::BasicResult<std::string, std::vector<Message>> GetBatch(RdKafka::KafkaCon
 
   return std::move(batch);
 }
-
-void CheckAndDestroyLastAssignmentIfNeeded(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
-                                           std::vector<RdKafka::TopicPartition *> &last_assignment) {
-  if (!last_assignment.empty()) {
-    if (const auto err = consumer.assign(last_assignment); err != RdKafka::ERR_NO_ERROR) {
-      throw ConsumerStartFailedException(info.consumer_name,
-                                         fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
-    }
-    RdKafka::TopicPartition::destroy(last_assignment);
-  }
-}
-
-void TryToConsumeBatch(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
-                       const ConsumerFunction &consumer_function, const std::vector<Message> &batch) {
-  consumer_function(batch);
-  std::vector<RdKafka::TopicPartition *> partitions;
-  utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
-
-  if (const auto err = consumer.assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
-    throw ConsumerCommitFailedException(
-        info.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
-  }
-  if (const auto err = consumer.position(partitions); err != RdKafka::ERR_NO_ERROR) {
-    throw ConsumerCommitFailedException(info.consumer_name,
-                                        fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
-  }
-  if (const auto err = consumer.commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
-    throw ConsumerCommitFailedException(info.consumer_name, RdKafka::err2str(err));
-  }
-}
 }  // namespace
 
 Message::Message(std::unique_ptr<RdKafka::Message> &&message) : message_{std::move(message)} {
@@ -251,21 +221,10 @@ void Consumer::Start() {
   StartConsuming();
 }
 
-void Consumer::StartWithLimit(const uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
-  if (is_running_) {
-    throw ConsumerRunningException(info_.consumer_name);
+void Consumer::StartIfStopped() {
+  if (!is_running_) {
+    StartConsuming();
   }
-  if (limit_batches < kMinimumStartBatchLimit) {
-    throw ConsumerStartFailedException(
-        info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
-  }
-  if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
-    throw ConsumerStartFailedException(
-        info_.consumer_name,
-        fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
-  }
-
-  StartConsumingWithLimit(limit_batches, timeout);
 }
 
 void Consumer::Stop() {
@@ -285,7 +244,7 @@ void Consumer::StopIfRunning() {
   }
 }
 
-void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
+void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
                      const ConsumerFunction &check_consumer_function) const {
   // NOLINTNEXTLINE (modernize-use-nullptr)
   if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
@@ -385,7 +344,13 @@ void Consumer::StartConsuming() {
 
   is_running_.store(true);
 
-  CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
+  if (!last_assignment_.empty()) {
+    if (const auto err = consumer_->assign(last_assignment_); err != RdKafka::ERR_NO_ERROR) {
+      throw ConsumerStartFailedException(info_.consumer_name,
+                                         fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
+    }
+    RdKafka::TopicPartition::destroy(last_assignment_);
+  }
 
   thread_ = std::thread([this] {
     static constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
@@ -396,18 +361,33 @@ void Consumer::StartConsuming() {
     while (is_running_) {
       auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
       if (maybe_batch.HasError()) {
-        throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
+        spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
+                     maybe_batch.GetError());
+        break;
       }
       const auto &batch = maybe_batch.GetValue();
 
-      if (batch.empty()) {
-        continue;
-      }
+      if (batch.empty()) continue;
 
       spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
 
       try {
-        TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
+        consumer_function_(batch);
+        std::vector<RdKafka::TopicPartition *> partitions;
+        utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
+
+        if (const auto err = consumer_->assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
+          throw ConsumerCheckFailedException(
+              info_.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
+        }
+        if (const auto err = consumer_->position(partitions); err != RdKafka::ERR_NO_ERROR) {
+          throw ConsumerCheckFailedException(
+              info_.consumer_name, fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
+        }
+        if (const auto err = consumer_->commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
+          spdlog::warn("Committing offset of consumer {} failed: {}", info_.consumer_name, RdKafka::err2str(err));
+          break;
+        }
       } catch (const std::exception &e) {
         spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
         break;
@@ -418,44 +398,6 @@ void Consumer::StartConsuming() {
   });
 }
 
-void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
-  MG_ASSERT(!is_running_, "Cannot start already running consumer!");
-
-  if (is_running_.exchange(true)) {
-    throw ConsumerRunningException(info_.consumer_name);
-  }
-  utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
-
-  CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
-
-  const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
-  const auto start = std::chrono::steady_clock::now();
-
-  for (uint64_t batch_count = 0; batch_count < limit_batches;) {
-    const auto now = std::chrono::steady_clock::now();
-    if (now - start >= timeout_to_use) {
-      throw ConsumerStartFailedException(info_.consumer_name, "Timeout reached");
-    }
-
-    const auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
-    if (maybe_batch.HasError()) {
-      throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
-    }
-    const auto &batch = maybe_batch.GetValue();
-
-    if (batch.empty()) {
-      continue;
-    }
-    ++batch_count;
-
-    spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
-
-    TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
-
-    spdlog::info("Kafka consumer {} finished processing", info_.consumer_name);
-  }
-}
-
 void Consumer::StopConsuming() {
   is_running_.store(false);
   if (thread_.joinable()) thread_.join();

src/integrations/kafka/consumer.hpp

@@ -113,19 +113,11 @@ class Consumer final : public RdKafka::EventCb {
   /// This method will start a new thread which will poll all the topics for messages.
   ///
   /// @throws ConsumerRunningException if the consumer is already running
-  /// @throws ConsumerStartFailedException if the commited offsets cannot be restored
   void Start();
 
-  /// Starts consuming messages.
+  /// Starts consuming messages if it is not started already.
   ///
-  /// This method will start a new thread which will poll all the topics for messages.
-  ///
-  /// @param limit_batches the consumer will only consume the given number of batches.
-  /// @param timeout the maximum duration during which the command should run.
-  ///
-  /// @throws ConsumerRunningException if the consumer is already running
-  /// @throws ConsumerStartFailedException if the commited offsets cannot be restored
-  void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
+  void StartIfStopped();
 
   /// Stops consuming messages.
   ///
@@ -144,9 +136,9 @@ class Consumer final : public RdKafka::EventCb {
   ///                      used.
   /// @param check_consumer_function a function to feed the received messages in, only used during this dry-run.
   ///
-  /// @throws ConsumerRunningException if the consumer is already running.
+  /// @throws ConsumerRunningException if the consumer is alredy running.
   /// @throws ConsumerCheckFailedException if check isn't successful.
-  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
              const ConsumerFunction &check_consumer_function) const;
 
   /// Returns true if the consumer is actively consuming messages.
@@ -165,7 +157,6 @@ class Consumer final : public RdKafka::EventCb {
   void event_cb(RdKafka::Event &event) override;
 
   void StartConsuming();
-  void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
 
   void StopConsuming();
 
@@ -187,6 +178,7 @@ class Consumer final : public RdKafka::EventCb {
   ConsumerFunction consumer_function_;
   mutable std::atomic<bool> is_running_{false};
   mutable std::vector<RdKafka::TopicPartition *> last_assignment_;  // Protected by is_running_
+  std::optional<int64_t> limit_batches_{std::nullopt};
   std::unique_ptr<RdKafka::KafkaConsumer, std::function<void(RdKafka::KafkaConsumer *)>> consumer_;
   std::thread thread_;
   ConsumerRebalanceCb cb_;

src/integrations/kafka/exceptions.hpp

@@ -64,16 +64,4 @@ class TopicNotFoundException : public KafkaStreamException {
   TopicNotFoundException(const std::string_view consumer_name, const std::string_view topic_name)
       : KafkaStreamException("Kafka consumer {} cannot find topic {}", consumer_name, topic_name) {}
 };
-
-class ConsumerCommitFailedException : public KafkaStreamException {
- public:
-  ConsumerCommitFailedException(const std::string_view consumer_name, const std::string_view error)
-      : KafkaStreamException("Committing offset of consumer {} failed: {}", consumer_name, error) {}
-};
-
-class ConsumerReadMessagesFailedException : public KafkaStreamException {
- public:
-  ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
-      : KafkaStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
-};
 }  // namespace memgraph::integrations::kafka

src/integrations/pulsar/consumer.cpp

@@ -11,14 +11,13 @@
 
 #include "integrations/pulsar/consumer.hpp"
 
-#include <algorithm>
-#include <chrono>
-#include <thread>
-
 #include <fmt/format.h>
 #include <pulsar/Client.h>
 #include <pulsar/InitialPosition.h>
 
+#include <chrono>
+#include <thread>
+
 #include "integrations/constants.hpp"
 #include "integrations/pulsar/exceptions.hpp"
 #include "utils/concepts.hpp"
@@ -34,10 +33,6 @@ namespace {
 template <typename T>
 concept PulsarConsumer = utils::SameAsAnyOf<T, pulsar_client::Consumer, pulsar_client::Reader>;
 
-template <typename TFunc>
-concept PulsarMessageGetter =
-    std::same_as<const pulsar_client::Message &, std::invoke_result_t<TFunc, const Message &>>;
-
 pulsar_client::Result ConsumeMessage(pulsar_client::Consumer &consumer, pulsar_client::Message &message,
                                      int remaining_timeout_in_ms) {
   return consumer.receive(message, remaining_timeout_in_ms);
@@ -102,26 +97,6 @@ pulsar_client::Client CreateClient(const std::string &service_url) {
   conf.setLogger(new SpdlogLoggerFactory);
   return {service_url, conf};
 }
-
-template <PulsarConsumer TConsumer, PulsarMessageGetter TPulsarMessageGetter>
-void TryToConsumeBatch(TConsumer &consumer, const ConsumerInfo &info, const ConsumerFunction &consumer_function,
-                       pulsar_client::MessageId &last_message_id, const std::vector<Message> &batch,
-                       const TPulsarMessageGetter &message_getter) {
-  consumer_function(batch);
-
-  auto has_message_failed = [&consumer, &info, &last_message_id, &message_getter](const auto &message) {
-    if (const auto result = consumer.acknowledge(message_getter(message)); result != pulsar_client::ResultOk) {
-      spdlog::warn("Acknowledging a message of consumer {} failed: {}", info.consumer_name, result);
-      return true;
-    }
-    last_message_id = message_getter(message).getMessageId();
-    return false;
-  };
-
-  if (std::ranges::any_of(batch, has_message_failed)) {
-    throw ConsumerAcknowledgeMessagesFailedException(info.consumer_name);
-  }
-}
 }  // namespace
 
 Message::Message(pulsar_client::Message &&message) : message_{std::move(message)} {}
@@ -162,24 +137,6 @@ void Consumer::Start() {
   StartConsuming();
 }
 
-void Consumer::StartWithLimit(const uint64_t limit_batches,
-                              const std::optional<std::chrono::milliseconds> timeout) const {
-  if (is_running_) {
-    throw ConsumerRunningException(info_.consumer_name);
-  }
-  if (limit_batches < kMinimumStartBatchLimit) {
-    throw ConsumerStartFailedException(
-        info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
-  }
-  if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
-    throw ConsumerStartFailedException(
-        info_.consumer_name,
-        fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
-  }
-
-  StartConsumingWithLimit(limit_batches, timeout);
-}
-
 void Consumer::Stop() {
   if (!is_running_) {
     throw ConsumerStoppedException(info_.consumer_name);
@@ -197,7 +154,7 @@ void Consumer::StopIfRunning() {
   }
 }
 
-void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
+void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
                      const ConsumerFunction &check_consumer_function) const {
   // NOLINTNEXTLINE (modernize-use-nullptr)
   if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
@@ -283,7 +240,9 @@ void Consumer::StartConsuming() {
       auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
 
       if (maybe_batch.HasError()) {
-        throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
+        spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
+                     maybe_batch.GetError());
+        break;
       }
 
       const auto &batch = maybe_batch.GetValue();
@@ -295,8 +254,18 @@ void Consumer::StartConsuming() {
       spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
 
       try {
-        TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
-                          [&](const Message &message) -> const pulsar_client::Message & { return message.message_; });
+        consumer_function_(batch);
+
+        if (std::any_of(batch.begin(), batch.end(), [&](const auto &message) {
+              if (const auto result = consumer_.acknowledge(message.message_); result != pulsar_client::ResultOk) {
+                spdlog::warn("Acknowledging a message of consumer {} failed: {}", info_.consumer_name, result);
+                return true;
+              }
+              last_message_id_ = message.message_.getMessageId();
+              return false;
+            })) {
+          break;
+        }
       } catch (const std::exception &e) {
         spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
         break;
@@ -308,43 +277,6 @@ void Consumer::StartConsuming() {
   });
 }
 
-void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
-  if (is_running_.exchange(true)) {
-    throw ConsumerRunningException(info_.consumer_name);
-  }
-  utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
-
-  const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
-  const auto start = std::chrono::steady_clock::now();
-
-  for (uint64_t batch_count = 0; batch_count < limit_batches;) {
-    const auto now = std::chrono::steady_clock::now();
-    if (now - start >= timeout_to_use) {
-      throw ConsumerCheckFailedException(info_.consumer_name, "Timeout reached");
-    }
-
-    const auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
-
-    if (maybe_batch.HasError()) {
-      throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
-    }
-
-    const auto &batch = maybe_batch.GetValue();
-
-    if (batch.empty()) {
-      continue;
-    }
-    ++batch_count;
-
-    spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
-
-    TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
-                      [](const Message &message) -> const pulsar_client::Message & { return message.message_; });
-
-    spdlog::info("Pulsar consumer {} finished processing", info_.consumer_name);
-  }
-}
-
 void Consumer::StopConsuming() {
   is_running_.store(false);
   if (thread_.joinable()) {

src/integrations/pulsar/consumer.hpp

@@ -58,27 +58,25 @@ class Consumer final {
 
   bool IsRunning() const;
   void Start();
-  void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
   void Stop();
   void StopIfRunning();
 
-  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
              const ConsumerFunction &check_consumer_function) const;
 
   const ConsumerInfo &Info() const;
 
  private:
   void StartConsuming();
-  void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
   void StopConsuming();
 
   ConsumerInfo info_;
   mutable pulsar_client::Client client_;
-  mutable pulsar_client::Consumer consumer_;
+  pulsar_client::Consumer consumer_;
   ConsumerFunction consumer_function_;
 
   mutable std::atomic<bool> is_running_{false};
-  mutable pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};  // Protected by is_running_
+  pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};
   std::thread thread_;
 };
 }  // namespace memgraph::integrations::pulsar

src/integrations/pulsar/exceptions.hpp

@@ -55,16 +55,4 @@ class TopicNotFoundException : public PulsarStreamException {
   TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
       : PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
 };
-
-class ConsumerReadMessagesFailedException : public PulsarStreamException {
- public:
-  ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
-      : PulsarStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
-};
-
-class ConsumerAcknowledgeMessagesFailedException : public PulsarStreamException {
- public:
-  explicit ConsumerAcknowledgeMessagesFailedException(const std::string_view consumer_name)
-      : PulsarStreamException("Acknowledging a message of consumer {} has failed!", consumer_name) {}
-};
 }  // namespace memgraph::integrations::pulsar

src/interface/.gitignore

@@ -0,0 +1 @@
+gen-cpp2

src/interface/CMakeLists.txt

@@ -0,0 +1,27 @@
+include(MgThrift)
+
+set(MG_INTERFACE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+
+mg_thrift_library(
+  "storage" #file_name
+  "Storage" #services
+  "${MG_INTERFACE_PATH}" #file_path
+  "${MG_INTERFACE_PATH}" #output_path
+  "interface" #include_prefix
+)
+
+mg_thrift_library(
+  "meta" #file_name
+  "Meta" #services
+  "${MG_INTERFACE_PATH}" #file_path
+  "${MG_INTERFACE_PATH}" #output_path
+  "interface" #include_prefix
+)
+
+mg_thrift_library(
+  "trial" #file_name
+  "PingPong" #services
+  "${MG_INTERFACE_PATH}" #file_path
+  "${MG_INTERFACE_PATH}" #output_path
+  "interface" #include_prefix
+)

src/interface/meta.thrift

@@ -0,0 +1,74 @@
+namespace cpp2 interface.meta
+
+typedef i64 LabelId
+typedef i64 IndexId
+
+struct Result {
+    1: bool success;
+}
+
+struct CreatePrimaryLabelRequest {
+    1: binary name;
+    2: list<binary> primary_keys;
+}
+
+struct CreateLabelResponse {
+    1: Result result;
+    2: LabelId label_id;
+}
+
+struct GetLabelInfosRequest {
+    //  The empty list means return all of the label infos
+    1: list<binary> label_names;
+}
+
+struct LabelInfo {
+    1: binary name;
+    2: LabelId label_id;
+    3: list<binary> primary_keys;
+}
+
+struct GetLabelInfoResponse {
+    1: Result result;
+    2: LabelInfo label_info;
+}
+
+struct GetLabelInfosResponse {
+    1: Result result;
+    2: list<LabelInfo> label_infos;
+}
+
+struct CreateIndexRequest {
+    1: LabelId label_id;
+    2: list<binary> property_names;
+}
+
+struct CreateIndexResponse {
+    1: Result result;
+    2: IndexId index_id;
+}
+
+struct IndexInfo {
+    1: LabelId label_id;
+    2: list<binary> property_names;
+}
+
+struct GetIndexInfosResponse {
+    1: Result result;
+    2: list<IndexInfo> index_infos;
+}
+
+
+service Meta {
+    CreateLabelResponse createPrimaryLabel(1: CreatePrimaryLabelRequest req);
+    CreateLabelResponse createSecondaryLabel(1: binary label_name);
+    Result dropLabel(1: binary label_name);
+    GetLabelInfoResponse getLabelInfo(1:binary label_name);
+    GetLabelInfosResponse getLabelInfos(1: list<binary> label_names);
+
+    CreateIndexResponse createIndex(1: CreateIndexRequest req);
+    // Don't have index names, and the label doesn't identify the index uniquely, therefore
+    Result dropIndex(1: IndexId index_id);
+    GetIndexInfosResponse getIndexInfos();
+    GetIndexInfosResponse getIndexInfosForLabel(1: LabelId label_id);
+}

src/interface/storage.thrift

@@ -0,0 +1,273 @@
+namespace cpp2 interface.storage
+// https://stackoverflow.com/a/34234874/6639989
+
+cpp_include "storage/v2/view.hpp"
+
+typedef i64 VertexId
+typedef i64 Gid
+
+// TODO(antaljanosbenjamin): Use this after introducing 128 bit vertex ids
+// struct VertexId {
+//     1: i64 upper_half;
+//     2: i64 lower_half;
+// }
+
+struct Label {
+    1: i64 id;
+}
+
+struct EdgeType {
+    1: binary name;
+}
+
+struct EdgeId {
+    1: VertexId src;
+    // QUESTION(antaljanosbenjamin): is it okay to have vertex based (edge id = vertex id + edge id inside vertex)?
+    2: Gid gid;
+}
+
+struct Date {
+    1: i16 year;
+    2: byte month;
+    3: byte day;
+}
+
+struct LocalTime {
+    1: byte hour;
+    2: byte minute;
+    3: byte second;
+    4: i16 millisecond;
+    5: i16 microsecond;
+}
+
+struct LocalDateTime {
+    1: Date date;
+    2: LocalTime local_time;
+}
+
+struct Duration {
+    1: i64 milliseconds;
+}
+
+union Value {
+    1: Null null_v;
+    2: bool bool_v;
+    3: i64 int_v;
+    4: double double_v;
+    5: binary string_v;
+    6: list<Value> list_v;
+    7: map<binary, Value> (cpp.template = "std::unordered_map") map_v (cpp2.ref_type = "unique");
+    8: Vertex vertex_v (cpp2.ref_type = "unique");
+    9: Edge edge_v (cpp2.ref_type = "unique");
+    10: Path path_v (cpp2.ref_type = "unique");
+    11: Date date_v;
+    12: LocalTime local_time_v;
+    13: LocalDateTime local_date_time_v;
+    14: Duration duration_v;
+}
+
+struct Null {
+}
+
+struct Vertex {
+    1: VertexId id;
+    // TODO(antaljanosbenjamin): Change to sperate primary and secondary labels when schema is implemented
+    2: list<Label> labels;
+}
+
+struct Edge {
+    1: VertexId src;
+    2: VertexId dst;
+    3: EdgeType type;
+}
+
+struct PathPart {
+    1: Vertex dst;
+    2: Edge edge;
+}
+
+struct Path {
+    1: Vertex src;
+    2: list<PathPart> parts;
+}
+
+struct ValuesMap {
+    1: map<i64, Value> (cpp.template = "std::unordered_map") values_map;
+}
+
+struct MappedValues {
+    1: list<ValuesMap> properties;
+}
+
+struct ListedValues {
+    1: list<list<Value>> properties;
+}
+
+union Values {
+    // This struct is necessary because depending on the request the response
+    // has two different formats:
+    // 1. When the request specifies the returned properties, then they are
+    //    returned in that order, therefore no extra mapping is necessary.
+    // 2. When the request doesn't specify the returned properties, then all
+    //    of the properties are returned. In this case the `mapped` field is
+    //    used. To extract the <key,value> pairs from this struct the
+    //    mapping of i64 -> property name has to be used.
+    1: ListedValues listed;
+    2: MappedValues mapped;
+}
+
+struct Expression {
+    1: binary alias;
+    2: binary expression;
+}
+
+struct Filter {
+    1: binary filter_expression;
+}
+
+enum OrderingDirection {
+    ASCENDING = 1;
+    DESCENDING = 2;
+}
+
+struct OrderBy {
+    1: Expression expression;
+    2: OrderingDirection direction;
+}
+
+struct Result {
+    // Just placeholder data for now
+    1: bool success;
+}
+
+enum View {
+    OLD = 0,
+    NEW = 1
+}  (cpp.enum_strict, cpp.type = "memgraph::storage::View")
+
+struct ScanVerticesRequest {
+    1: i64 transaction_id;
+    2: optional i64 start_id;
+    // Special values are accepted:
+    // * __mg__id (Vertex, but without labels)
+    // * __mg__labels (Vertex, but without the id)
+    // If both of them is specified, then it will result in a single, fully populated vertex
+    // QUESTION(antaljanosbenjamin): Does the `__mg__labels` is necessary? What about passing the `labels` function
+    //                               as an expression? Maybe it is an optimization. For communicating the vertex id
+    //                               the Vertex struct is really handy.
+    3: optional list<binary> props_to_return;
+    4: list<Expression> expressions;
+    5: optional i64 limit;
+    6: View view;
+    7: optional Filter filter;
+}
+
+struct ScanVerticesResponse {
+    1: Result result;
+    2: Values values;
+    3: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
+    // contains the next start_id if there is any
+    4: optional VertexId next_start_id;
+}
+
+union VertexOrEdgeIds {
+    1: list<VertexId> vertex_ids;
+    2: list<EdgeId> edge_ids;
+}
+
+struct GetPropertiesRequest {
+    1:  i64 transaction_id;
+    2:  VertexOrEdgeIds vertex_or_edge_ids;
+    3:  list<binary> property_names;
+    4:  list<Expression> expressions;
+    5:  bool only_unique = false;
+    6:  optional list<OrderBy> order_by;
+    7:  optional i64 limit;
+    8:  optional Filter filter;
+}
+
+struct GetPropertiesResponse {
+    1: Values values;
+    2: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
+}
+
+enum EdgeDirection {
+    OUT = 1;
+    IN = 2;
+    BOTH = 3;
+}
+
+struct ExpandOneRequest {
+    1:  i64 transaction_id;
+    2:  list<VertexId> src_vertices;
+    3:  list<EdgeType> edge_types;
+    4:  EdgeDirection direction;
+    5:  bool only_unique_neighbor_rows = false;
+    //  The empty optional means return all of the properties, while an empty
+    //  list means do not return any properties
+    //  TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object
+    //                            after schema is implemented
+    //  Special values are accepted:
+    //  * __mg__labels
+    6:  optional list<binary> src_vertex_properties;
+    //  TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object
+    //                            after schema is implemented
+    //  Special values are accepted:
+    //  * __mg__dst_id (Vertex, but without labels)
+    //  * __mg__type (binary)
+    7:  optional list<binary> edge_properties;
+    //  QUESTION(antaljanosbenjamin): Maybe also add possibility to expressions evaluated on the source vertex?
+    //  List of expressions evaluated on edges
+    8:  list<Expression> expressions;
+    9:  optional list<OrderBy> order_by;
+    10:  optional i64 limit;
+    11: optional Filter filter;
+}
+
+struct ExpandOneResultRow {
+    // NOTE: This struct could be a single Values with columns something like this:
+    // src_vertex(Vertex), vertex_prop1(Value), vertex_prop2(Value), edges(list<Value>)
+    // where edges might be a list of:
+    // 1. list<Value> if only a defined list of edge properties are returned
+    // 2. map<binary, Value> if all of the edge properties are returned
+    // The drawback of this is currently the key of the map is always interpreted as a string in Value, not as an
+    // integer, which should be in case of mapped properties.
+    1: Vertex src_vertex;
+    2: optional Values src_vertex_properties;
+    3: Values edges;
+}
+
+struct ExpandOneResponse {
+    // This approach might not suit the expand with per shard parrallelization,
+    // because the property_name_map has to be accessed from multiple threads
+    // in order to avoid duplicated keys (two threads might map the same
+    // property with different numbers) and multiple passes (to unify the
+    // mapping amond result returned from different shards).
+    1: list<ExpandOneResultRow> result;
+    2: optional map<i64, binary>  (cpp.template = "std::unordered_map") property_name_map;
+}
+
+struct NewVertex {
+    1: list<i64> label_ids;
+    2: map<i64, Value> properties;
+}
+
+struct CreateVerticesRequest {
+    1: required i64 transaction_id;
+    2: map<i64, binary> (cpp.template = "std::unordered_map") labels_name_map;
+    3: map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
+    4: list<NewVertex> new_vertices;
+}
+
+
+service Storage {
+    i64 startTransaction()
+    Result commitTransaction(1: i64 transaction_id)
+    void abortTransaction(1: i64 transaction_id)
+
+    Result createVertices(1: CreateVerticesRequest req)
+    ScanVerticesResponse scanVertices(1: ScanVerticesRequest req)
+    GetPropertiesResponse getProperties(1: GetPropertiesRequest req)
+    ExpandOneResponse expandOne(1: ExpandOneRequest req)
+
+}

src/interface/trial.thrift

@@ -0,0 +1,11 @@
+struct Ping {
+    1: binary message;
+}
+
+struct Pong{
+    1: binary message;
+}
+
+service PingPong {
+    Pong ping(1: Ping req)
+}

src/io/address.hpp

@@ -1,60 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <compare>
-
-#include <fmt/format.h>
-#include <boost/asio/ip/tcp.hpp>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_io.hpp>
-
-namespace memgraph::io {
-struct Address {
-  // It's important for all participants to have a
-  // unique identifier - IP and port alone are not
-  // enough, and may change over the lifecycle of
-  // the nodes. Particularly storage nodes may change
-  // their IP addresses over time, and the system
-  // should gracefully update its information
-  // about them.
-  boost::uuids::uuid unique_id;
-  boost::asio::ip::address last_known_ip;
-  uint16_t last_known_port;
-
-  static Address TestAddress(uint16_t port) {
-    Address ret;
-    ret.last_known_port = port;
-    return ret;
-  }
-
-  friend bool operator==(const Address &lhs, const Address &rhs) = default;
-
-  /// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
-  friend bool operator<(const Address &lhs, const Address &rhs) {
-    if (lhs.unique_id != rhs.unique_id) {
-      return lhs.unique_id < rhs.unique_id;
-    }
-
-    if (lhs.last_known_ip != rhs.last_known_ip) {
-      return lhs.last_known_ip < rhs.last_known_ip;
-    }
-
-    return lhs.last_known_port < rhs.last_known_port;
-  }
-
-  std::string ToString() const {
-    return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
-                       boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
-  }
-};
-};  // namespace memgraph::io

src/io/errors.hpp

@@ -1,26 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-namespace memgraph::io {
-// Signifies that a retriable operation was unable to
-// complete after a configured number of retries.
-struct RetriesExhausted {};
-
-// Signifies that a request was unable to receive a response
-// within some configured timeout duration. It is important
-// to remember that in distributed systems, a timeout does
-// not signify that a request was not received or processed.
-// It may be the case that the request was fully processed
-// but that the response was not received.
-struct TimedOut {};
-};  // namespace memgraph::io

src/io/future.hpp

@@ -1,262 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <condition_variable>
-#include <memory>
-#include <mutex>
-#include <optional>
-#include <thread>
-#include <utility>
-
-#include "io/errors.hpp"
-#include "utils/logging.hpp"
-
-namespace memgraph::io {
-
-// Shared is in an anonymous namespace, and the only way to
-// construct a Promise or Future is to pass a Shared in. This
-// ensures that Promises and Futures can only be constructed
-// in this translation unit.
-namespace details {
-template <typename T>
-class Shared {
-  mutable std::condition_variable cv_;
-  mutable std::mutex mu_;
-  std::optional<T> item_;
-  bool consumed_ = false;
-  bool waiting_ = false;
-  std::function<bool()> simulator_notifier_ = nullptr;
-
- public:
-  explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
-  Shared() = default;
-  Shared(Shared &&) = delete;
-  Shared &operator=(Shared &&) = delete;
-  Shared(const Shared &) = delete;
-  Shared &operator=(const Shared &) = delete;
-  ~Shared() = default;
-
-  /// Takes the item out of our optional item_ and returns it.
-  T Take() {
-    MG_ASSERT(item_, "Take called without item_ being present");
-    MG_ASSERT(!consumed_, "Take called on already-consumed Future");
-
-    T ret = std::move(item_).value();
-    item_.reset();
-
-    consumed_ = true;
-
-    return ret;
-  }
-
-  T Wait() {
-    std::unique_lock<std::mutex> lock(mu_);
-    waiting_ = true;
-
-    while (!item_) {
-      bool simulator_progressed = false;
-      if (simulator_notifier_) [[unlikely]] {
-        // We can't hold our own lock while notifying
-        // the simulator because notifying the simulator
-        // involves acquiring the simulator's mutex
-        // to guarantee that our notification linearizes
-        // with the simulator's condition variable.
-        // However, the simulator may acquire our
-        // mutex to check if we are being awaited,
-        // while determining system quiescence,
-        // so we have to get out of its way to avoid
-        // a cyclical deadlock.
-        lock.unlock();
-        simulator_progressed = std::invoke(simulator_notifier_);
-        lock.lock();
-        if (item_) {
-          // item may have been filled while we
-          // had dropped our mutex while notifying
-          // the simulator of our waiting_ status.
-          break;
-        }
-      }
-      if (!simulator_progressed) [[likely]] {
-        cv_.wait(lock);
-      }
-      MG_ASSERT(!consumed_, "Future consumed twice!");
-    }
-
-    waiting_ = false;
-
-    return Take();
-  }
-
-  bool IsReady() const {
-    std::unique_lock<std::mutex> lock(mu_);
-    return item_;
-  }
-
-  std::optional<T> TryGet() {
-    std::unique_lock<std::mutex> lock(mu_);
-
-    if (item_) {
-      return Take();
-    }
-
-    return std::nullopt;
-  }
-
-  void Fill(T item) {
-    {
-      std::unique_lock<std::mutex> lock(mu_);
-
-      MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
-      MG_ASSERT(!item_, "Promise filled twice!");
-
-      item_ = item;
-    }  // lock released before condition variable notification
-
-    cv_.notify_all();
-  }
-
-  bool IsAwaited() const {
-    std::unique_lock<std::mutex> lock(mu_);
-    return waiting_;
-  }
-};
-}  // namespace details
-
-template <typename T>
-class Future {
-  bool consumed_or_moved_ = false;
-  std::shared_ptr<details::Shared<T>> shared_;
-
- public:
-  explicit Future(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
-
-  Future() = delete;
-  Future(Future &&old) noexcept {
-    MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
-    shared_ = std::move(old.shared_);
-    consumed_or_moved_ = old.consumed_or_moved_;
-    old.consumed_or_moved_ = true;
-  }
-
-  Future &operator=(Future &&old) noexcept {
-    MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
-    shared_ = std::move(old.shared_);
-    old.consumed_or_moved_ = true;
-  }
-
-  Future(const Future &) = delete;
-  Future &operator=(const Future &) = delete;
-  ~Future() = default;
-
-  /// Returns true if the Future is ready to
-  /// be consumed using TryGet or Wait (prefer Wait
-  /// if you know it's ready, because it doesn't
-  /// return an optional.
-  bool IsReady() {
-    MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!");
-    return shared_->IsReady();
-  }
-
-  /// Non-blocking method that returns the inner
-  /// item if it's already ready, or std::nullopt
-  /// if it is not ready yet.
-  std::optional<T> TryGet() {
-    MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!");
-    std::optional<T> ret = shared_->TryGet();
-    if (ret) {
-      consumed_or_moved_ = true;
-    }
-    return ret;
-  }
-
-  /// Block on the corresponding promise to be filled,
-  /// returning the inner item when ready.
-  T Wait() && {
-    MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!");
-    T ret = shared_->Wait();
-    consumed_or_moved_ = true;
-    return ret;
-  }
-
-  /// Marks this Future as canceled.
-  void Cancel() {
-    MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!");
-    consumed_or_moved_ = true;
-  }
-};
-
-template <typename T>
-class Promise {
-  std::shared_ptr<details::Shared<T>> shared_;
-  bool filled_or_moved_ = false;
-
- public:
-  explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
-
-  Promise() = delete;
-  Promise(Promise &&old) noexcept {
-    MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
-    shared_ = std::move(old.shared_);
-    old.filled_or_moved_ = true;
-  }
-
-  Promise &operator=(Promise &&old) noexcept {
-    MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
-    shared_ = std::move(old.shared_);
-    old.filled_or_moved_ = true;
-  }
-  Promise(const Promise &) = delete;
-  Promise &operator=(const Promise &) = delete;
-
-  ~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
-
-  // Fill the expected item into the Future.
-  void Fill(T item) {
-    MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
-    shared_->Fill(item);
-    filled_or_moved_ = true;
-  }
-
-  bool IsAwaited() { return shared_->IsAwaited(); }
-
-  /// Moves this Promise into a unique_ptr.
-  std::unique_ptr<Promise<T>> ToUnique() && {
-    std::unique_ptr<Promise<T>> up = std::make_unique<Promise<T>>(std::move(shared_));
-
-    filled_or_moved_ = true;
-
-    return up;
-  }
-};
-
-template <typename T>
-std::pair<Future<T>, Promise<T>> FuturePromisePair() {
-  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>();
-
-  Future<T> future = Future<T>(shared);
-  Promise<T> promise = Promise<T>(shared);
-
-  return std::make_pair(std::move(future), std::move(promise));
-}
-
-template <typename T>
-std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
-  std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
-
-  Future<T> future = Future<T>(shared);
-  Promise<T> promise = Promise<T>(shared);
-
-  return std::make_pair(std::move(future), std::move(promise));
-}
-
-};  // namespace memgraph::io

src/io/network/epoll.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/io/network/socket.cpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2021 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/io/rsm/coordinator_rsm.hpp

@@ -1,25 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include "coordinator/coordinator.hpp"
-#include "io/rsm/raft.hpp"
-
-namespace memgraph::io::rsm {
-
-// TODO(tyler) don't
-using namespace memgraph::coordinator;
-
-template <typename IoImpl>
-using CoordinatorRsm = io::rsm::Raft<IoImpl, Coordinator, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
-
-}  // namespace memgraph::io::rsm

src/io/rsm/raft.hpp

@@ -1,845 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-// TODO(tyler) buffer out-of-order Append buffers on the Followers to reassemble more quickly
-// TODO(tyler) handle granular batch sizes based on simple flow control
-
-#pragma once
-
-#include <deque>
-#include <iostream>
-#include <map>
-#include <set>
-#include <thread>
-#include <unordered_map>
-#include <vector>
-
-#include "io/simulator/simulator.hpp"
-#include "io/transport.hpp"
-
-namespace memgraph::io::rsm {
-
-using memgraph::io::Address;
-using memgraph::io::Duration;
-using memgraph::io::Io;
-using memgraph::io::ResponseFuture;
-using memgraph::io::ResponseResult;
-using memgraph::io::Time;
-
-using Term = uint64_t;
-using LogIndex = uint64_t;
-using RequestId = uint64_t;
-
-template <typename WriteOperation>
-struct WriteRequest {
-  WriteOperation operation;
-};
-
-/// WriteResponse is returned to a client after
-/// their WriteRequest was entered in to the raft
-/// log and it reached consensus.
-///
-/// WriteReturn is the result of applying the WriteRequest to
-/// ReplicatedState, and if the ReplicatedState::write
-/// method is deterministic, all replicas will
-/// have the same ReplicatedState after applying
-/// the submitted WriteRequest.
-template <typename WriteReturn>
-struct WriteResponse {
-  bool success;
-  WriteReturn write_return;
-  std::optional<Address> retry_leader;
-};
-
-template <typename ReadOperation>
-struct ReadRequest {
-  ReadOperation operation;
-};
-
-template <typename ReadReturn>
-struct ReadResponse {
-  bool success;
-  ReadReturn read_return;
-  std::optional<Address> retry_leader;
-};
-
-/// AppendRequest is a raft-level message that the Leader
-/// periodically broadcasts to all Follower peers. This
-/// serves three main roles:
-/// 1. acts as a heartbeat from the Leader to the Follower
-/// 2. replicates new data that the Leader has received to the Follower
-/// 3. informs Follower peers when the commit index has increased,
-///    signalling that it is now safe to apply log items to the
-///    replicated state machine
-template <typename WriteRequest>
-struct AppendRequest {
-  Term term = 0;
-  LogIndex last_log_index;
-  Term last_log_term;
-  std::vector<std::pair<Term, WriteRequest>> entries;
-  LogIndex leader_commit;
-};
-
-struct AppendResponse {
-  bool success;
-  Term term;
-  Term last_log_term;
-  // a small optimization over the raft paper, tells
-  // the leader the offset that we are interested in
-  // to send log offsets from for us. This will only
-  // be useful at the beginning of a leader's term.
-  LogIndex last_log_index;
-};
-
-struct VoteRequest {
-  Term term = 0;
-  LogIndex last_log_index;
-  Term last_log_term;
-};
-
-struct VoteResponse {
-  Term term = 0;
-  LogIndex committed_log_size;
-  bool vote_granted = false;
-};
-
-template <typename WriteRequest>
-struct CommonState {
-  Term term = 0;
-  std::vector<std::pair<Term, WriteRequest>> log;
-  LogIndex committed_log_size = 0;
-  LogIndex applied_size = 0;
-};
-
-struct FollowerTracker {
-  LogIndex next_index = 0;
-  LogIndex confirmed_contiguous_index = 0;
-};
-
-struct PendingClientRequest {
-  RequestId request_id;
-  Address address;
-  Time received_at;
-};
-
-struct Leader {
-  std::map<Address, FollowerTracker> followers;
-  std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
-  Time last_broadcast = Time::min();
-
-  std::string ToString() { return "\tLeader   \t"; }
-};
-
-struct Candidate {
-  std::map<Address, LogIndex> successful_votes;
-  Time election_began = Time::min();
-  std::set<Address> outstanding_votes;
-
-  std::string ToString() { return "\tCandidate\t"; }
-};
-
-struct Follower {
-  Time last_received_append_entries_timestamp;
-  Address leader_address;
-
-  std::string ToString() { return "\tFollower \t"; }
-};
-
-using Role = std::variant<Candidate, Leader, Follower>;
-
-/*
-
-all ReplicatedState classes should have an Apply method
-that returns our WriteResponseValue after consensus, and
-a Read method that returns our ReadResponseValue without
-requiring consensus.
-
-ReadResponse Read(ReadOperation);
-WriteResponseValue ReplicatedState::Apply(WriteRequest);
-
-For example:
-If the state is uint64_t, and WriteRequest is `struct PlusOne {};`,
-and WriteResponseValue is also uint64_t (the new value), then
-each call to state.Apply(PlusOne{}) will return the new value
-after incrementing it. 0, 1, 2, 3... and this will be sent back
-to the client that requested the mutation.
-
-In practice, these mutations will usually be predicated on some
-previous value, so that they are idempotent, functioning similarly
-to a CAS operation.
-*/
-template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
-          typename ReadResponseValue>
-concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
-  { state.Read(r) } -> std::same_as<ReadResponseValue>;
-  { state.Apply(w) } -> std::same_as<WriteResponseValue>;
-};
-
-/// Parameter           Purpose
-/// --------------------------
-/// IoImpl              the concrete Io provider - SimulatorTransport, ThriftTransport, etc...
-/// ReplicatedState     the high-level data structure that is managed by the raft-backed replicated state machine
-/// WriteOperation      the individual operation type that is applied to the ReplicatedState in identical order
-///                     across each replica
-/// WriteResponseValue  the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in
-///                     identical order across all replicas after an WriteOperation reaches consensus.
-/// ReadOperation       the type of operations that do not require consensus before executing directly
-///                     on a const ReplicatedState &
-/// ReadResponseValue   the return value of calling ReplicatedState::Read(ReadOperation), which is executed directly
-///                     without going through consensus first
-template <typename IoImpl, typename ReplicatedState, typename WriteOperation, typename WriteResponseValue,
-          typename ReadOperation, typename ReadResponseValue>
-requires Rsm<WriteOperation, ReadOperation, ReplicatedState, WriteResponseValue, ReadResponseValue>
-class Raft {
-  CommonState<WriteOperation> state_;
-  Role role_ = Candidate{};
-  Io<IoImpl> io_;
-  std::vector<Address> peers_;
-  ReplicatedState replicated_state_;
-
- public:
-  Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
-      : io_(std::move(io)), peers_(peers), replicated_state_(std::move(replicated_state)) {}
-
-  void Run() {
-    Time last_cron = io_.Now();
-
-    while (!io_.ShouldShutDown()) {
-      const auto now = io_.Now();
-      const Duration random_cron_interval = RandomTimeout(1000, 2000);
-      if (now - last_cron > random_cron_interval) {
-        Cron();
-        last_cron = now;
-      }
-
-      Duration receive_timeout = RandomTimeout(10000, 50000);
-
-      auto request_result =
-          io_.template ReceiveWithTimeout<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
-                                          WriteRequest<WriteOperation>, VoteRequest, VoteResponse>(receive_timeout);
-      if (request_result.HasError()) {
-        continue;
-      }
-
-      auto request = std::move(request_result.GetValue());
-
-      Handle(std::move(request.message), request.request_id, request.from_address);
-    }
-  }
-
- private:
-  // Raft paper - 5.3
-  // When the entry has been safely replicated, the leader applies the
-  // entry to its state machine and returns the result of that
-  // execution to the client.
-  //
-  // "Safely replicated" is defined as being known to be present
-  // on at least a majority of all peers (inclusive of the Leader).
-  void BumpCommitIndexAndReplyToClients(Leader &leader) {
-    auto indices = std::vector<LogIndex>{};
-
-    // We include our own log size in the calculation of the log
-    // index that is present on at least a majority of all peers.
-    indices.push_back(state_.log.size());
-
-    for (const auto &[addr, f] : leader.followers) {
-      indices.push_back(f.confirmed_contiguous_index);
-      Log("at port ", addr.last_known_port, " has confirmed contiguous index of: ", f.confirmed_contiguous_index);
-    }
-
-    // reverse sort from highest to lowest (using std::ranges::greater)
-    std::ranges::sort(indices, std::ranges::greater());
-
-    // This is a particularly correctness-critical calculation because it
-    // determines which index we will consider to be the committed index.
-    //
-    // If the following indexes are recorded for clusters of different sizes,
-    // these are the expected indexes that are considered to have reached
-    // consensus:
-    // state           | expected value | (indices.size() / 2)
-    // [1]              1                 (1 / 2) => 0
-    // [2, 1]           1                 (2 / 2) => 1
-    // [3, 2, 1]        2                 (3 / 2) => 1
-    // [4, 3, 2, 1]     2                 (4 / 2) => 2
-    // [5, 4, 3, 2, 1]  3                 (5 / 2) => 2
-    size_t index_present_on_majority = indices.size() / 2;
-    LogIndex new_committed_log_size = indices[index_present_on_majority];
-
-    // We never go backwards in history.
-    MG_ASSERT(state_.committed_log_size <= new_committed_log_size);
-
-    state_.committed_log_size = new_committed_log_size;
-
-    // For each index between the old index and the new one (inclusive),
-    // Apply that log's WriteOperation to our replicated_state_,
-    // and use the specific return value of the ReplicatedState::Apply
-    // method (WriteResponseValue) to respond to the requester.
-    for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
-      const LogIndex apply_index = state_.applied_size;
-      const auto &write_request = state_.log[apply_index].second;
-      WriteResponseValue write_return = replicated_state_.Apply(write_request);
-
-      if (leader.pending_client_requests.contains(apply_index)) {
-        PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
-
-        WriteResponse<WriteResponseValue> resp;
-        resp.success = true;
-        resp.write_return = std::move(write_return);
-
-        io_.Send(client_request.address, client_request.request_id, std::move(resp));
-        leader.pending_client_requests.erase(apply_index);
-      }
-    }
-
-    Log("committed_log_size is now ", state_.committed_log_size);
-  }
-
-  // Raft paper - 5.1
-  // AppendEntries RPCs are initiated by leaders to replicate log entries and to provide a form of heartbeat
-  void BroadcastAppendEntries(std::map<Address, FollowerTracker> &followers) {
-    for (auto &[address, follower] : followers) {
-      const LogIndex index = follower.confirmed_contiguous_index;
-
-      std::vector<std::pair<Term, WriteOperation>> entries;
-
-      if (state_.log.size() > index) {
-        entries.insert(entries.begin(), state_.log.begin() + index, state_.log.end());
-      }
-
-      const Term previous_term_from_index = PreviousTermFromIndex(index);
-
-      Log("sending ", entries.size(), " entries to Follower ", address.last_known_port,
-          " which are above its known index of ", index);
-
-      AppendRequest<WriteOperation> ar{
-          .term = state_.term,
-          .last_log_index = index,
-          .last_log_term = previous_term_from_index,
-          .entries = entries,
-          .leader_commit = state_.committed_log_size,
-      };
-
-      // request_id not necessary to set because it's not a Future-backed Request.
-      static constexpr RequestId request_id = 0;
-
-      io_.Send(address, request_id, ar);
-    }
-  }
-
-  // Raft paper - 5.2
-  // Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly
-  Duration RandomTimeout(Duration min, Duration max) {
-    std::uniform_int_distribution time_distrib(min.count(), max.count());
-
-    auto rand_micros = io_.Rand(time_distrib);
-
-    return std::chrono::microseconds{rand_micros};
-  }
-
-  Duration RandomTimeout(int min_micros, int max_micros) {
-    std::uniform_int_distribution time_distrib(min_micros, max_micros);
-
-    int rand_micros = io_.Rand(time_distrib);
-
-    return std::chrono::microseconds{rand_micros};
-  }
-
-  Term PreviousTermFromIndex(LogIndex index) const {
-    if (index == 0 || state_.log.size() + 1 <= index) {
-      return 0;
-    }
-
-    const auto &[term, data] = state_.log.at(index - 1);
-    return term;
-  }
-
-  LogIndex CommittedLogIndex() { return state_.committed_log_size; }
-
-  Term CommittedLogTerm() {
-    MG_ASSERT(state_.log.size() >= state_.committed_log_size);
-    if (state_.log.empty() || state_.committed_log_size == 0) {
-      return 0;
-    }
-
-    auto &[term, data] = state_.log.at(state_.committed_log_size - 1);
-    return term;
-  }
-
-  LogIndex LastLogIndex() { return state_.log.size(); }
-
-  Term LastLogTerm() const {
-    if (state_.log.empty()) {
-      return 0;
-    }
-
-    const auto &[term, data] = state_.log.back();
-    return term;
-  }
-
-  template <typename... Ts>
-  void Log(Ts &&...args) {
-    const Time now = io_.Now();
-    auto micros = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
-
-    const Term term = state_.term;
-
-    std::ostringstream out;
-
-    out << '\t' << (int)micros << "\t" << term << "\t" << io_.GetAddress().last_known_port;
-
-    std::string role_string = std::visit([&](auto &&role) { return role.ToString(); }, role_);
-
-    out << role_string;
-
-    (out << ... << args);
-
-    spdlog::debug(out.str());
-  }
-
-  /////////////////////////////////////////////////////////////
-  /// Raft-related Cron methods
-  ///
-  /// Cron + std::visit is how events are dispatched
-  /// to certain code based on Raft role.
-  ///
-  /// Cron(role) takes as the first argument a reference to its
-  /// role, and as the second argument, the message that has
-  /// been received.
-  /////////////////////////////////////////////////////////////
-
-  /// Periodic protocol maintenance.
-  void Cron() {
-    // dispatch periodic logic based on our role to a specific Cron method.
-    std::optional<Role> new_role = std::visit([&](auto &&role) { return Cron(role); }, role_);
-
-    if (new_role) {
-      role_ = std::move(new_role).value();
-    }
-  }
-
-  // Raft paper - 5.2
-  // Candidates keep sending Vote to peers until:
-  // 1. receiving Append with a higher term (become Follower)
-  // 2. receiving Vote with a higher term (become a Follower)
-  // 3. receiving a quorum of responses to our last batch of Vote (become a Leader)
-  std::optional<Role> Cron(Candidate &candidate) {
-    const auto now = io_.Now();
-    const Duration election_timeout = RandomTimeout(100000, 200000);
-    const auto election_timeout_us = std::chrono::duration_cast<std::chrono::milliseconds>(election_timeout).count();
-
-    if (now - candidate.election_began > election_timeout) {
-      state_.term++;
-      Log("becoming Candidate for term ", state_.term, " after leader timeout of ", election_timeout_us,
-          " elapsed since last election attempt");
-
-      const VoteRequest request{
-          .term = state_.term,
-          .last_log_index = LastLogIndex(),
-          .last_log_term = LastLogTerm(),
-      };
-
-      auto outstanding_votes = std::set<Address>();
-
-      for (const auto &peer : peers_) {
-        // request_id not necessary to set because it's not a Future-backed Request.
-        static constexpr auto request_id = 0;
-        io_.template Send<VoteRequest>(peer, request_id, request);
-        outstanding_votes.insert(peer);
-      }
-
-      return Candidate{
-          .successful_votes = std::map<Address, LogIndex>(),
-          .election_began = now,
-          .outstanding_votes = outstanding_votes,
-      };
-    }
-    return std::nullopt;
-  }
-
-  // Raft paper - 5.2
-  // Followers become candidates if we haven't heard from the leader
-  // after a randomized timeout.
-  std::optional<Role> Cron(Follower &follower) {
-    const auto now = io_.Now();
-    const auto time_since_last_append_entries = now - follower.last_received_append_entries_timestamp;
-    Duration election_timeout = RandomTimeout(100000, 200000);
-
-    // randomized follower timeout with a range of 100-150ms.
-    if (time_since_last_append_entries > election_timeout) {
-      // become a Candidate if we haven't heard from the Leader after this timeout
-      return Candidate{};
-    }
-
-    return std::nullopt;
-  }
-
-  // Leaders (re)send AppendRequest to followers.
-  std::optional<Role> Cron(Leader &leader) {
-    const Time now = io_.Now();
-    const Duration broadcast_timeout = RandomTimeout(40000, 60000);
-
-    if (now - leader.last_broadcast > broadcast_timeout) {
-      BroadcastAppendEntries(leader.followers);
-      leader.last_broadcast = now;
-    }
-
-    return std::nullopt;
-  }
-
-  /////////////////////////////////////////////////////////////
-  /// Raft-related Handle methods
-  ///
-  /// Handle + std::visit is how events are dispatched
-  /// to certain code based on Raft role.
-  ///
-  /// Handle(role, message, ...)
-  /// takes as the first argument a reference
-  /// to its role, and as the second argument, the
-  /// message that has been received.
-  /////////////////////////////////////////////////////////////
-
-  void Handle(std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
-                           WriteRequest<WriteOperation>, VoteRequest, VoteResponse> &&message_variant,
-              RequestId request_id, Address from_address) {
-    // dispatch the message to a handler based on our role,
-    // which can be specified in the Handle first argument,
-    // or it can be `auto` if it's a handler for several roles
-    // or messages.
-    std::optional<Role> new_role =
-        std::visit([&](auto &&msg, auto &&role) { return Handle(role, std::move(msg), request_id, from_address); },
-                   std::move(message_variant), role_);
-
-    // TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
-    if (new_role) {
-      role_ = std::move(new_role).value();
-    }
-  }
-
-  // all roles can receive Vote and possibly become a follower
-  template <typename AllRoles>
-  std::optional<Role> Handle(AllRoles &, VoteRequest &&req, RequestId request_id, Address from_address) {
-    Log("received Vote from ", from_address.last_known_port, " with term ", req.term);
-    const bool last_log_term_dominates = req.last_log_term >= LastLogTerm();
-    const bool term_dominates = req.term > state_.term;
-    const bool last_log_index_dominates = req.last_log_index >= LastLogIndex();
-    const bool new_leader = last_log_term_dominates && term_dominates && last_log_index_dominates;
-
-    if (new_leader) {
-      MG_ASSERT(req.term > state_.term);
-      MG_ASSERT(std::max(req.term, state_.term) == req.term);
-    }
-
-    const VoteResponse res{
-        .term = std::max(req.term, state_.term),
-        .committed_log_size = state_.committed_log_size,
-        .vote_granted = new_leader,
-    };
-
-    io_.Send(from_address, request_id, res);
-
-    if (new_leader) {
-      // become a follower
-      state_.term = req.term;
-      return Follower{
-          .last_received_append_entries_timestamp = io_.Now(),
-          .leader_address = from_address,
-      };
-    } else if (term_dominates) {
-      Log("received a vote from an inferior candidate. Becoming Candidate");
-      state_.term = std::max(state_.term, req.term) + 1;
-      return Candidate{};
-    }
-
-    return std::nullopt;
-  }
-
-  std::optional<Role> Handle(Candidate &candidate, VoteResponse &&res, RequestId, Address from_address) {
-    Log("received VoteResponse");
-
-    if (!res.vote_granted || res.term != state_.term) {
-      Log("received unsuccessful VoteResponse from term ", res.term, " when our candidacy term is ", state_.term);
-      // we received a delayed VoteResponse from the past, which has to do with an election that is
-      // no longer valid. We can simply drop this.
-      return std::nullopt;
-    }
-
-    MG_ASSERT(candidate.outstanding_votes.contains(from_address),
-              "Received unexpected VoteResponse from server not present in Candidate's outstanding_votes!");
-    candidate.outstanding_votes.erase(from_address);
-
-    MG_ASSERT(!candidate.successful_votes.contains(from_address),
-              "Received unexpected VoteResponse from server already in Candidate's successful_votes!");
-    candidate.successful_votes.insert({from_address, res.committed_log_size});
-
-    if (candidate.successful_votes.size() >= candidate.outstanding_votes.size()) {
-      std::map<Address, FollowerTracker> followers{};
-
-      for (const auto &[address, committed_log_size] : candidate.successful_votes) {
-        FollowerTracker follower{
-            .next_index = committed_log_size,
-            .confirmed_contiguous_index = committed_log_size,
-        };
-        followers.insert({address, follower});
-      }
-      for (const auto &address : candidate.outstanding_votes) {
-        FollowerTracker follower{
-            .next_index = state_.log.size(),
-            .confirmed_contiguous_index = 0,
-        };
-        followers.insert({address, follower});
-      }
-
-      Log("becoming Leader at term ", state_.term);
-
-      BroadcastAppendEntries(followers);
-
-      return Leader{
-          .followers = std::move(followers),
-          .pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
-      };
-    }
-
-    return std::nullopt;
-  }
-
-  template <typename AllRoles>
-  std::optional<Role> Handle(AllRoles &, VoteResponse &&, RequestId, Address) {
-    Log("non-Candidate received VoteResponse");
-    return std::nullopt;
-  }
-
-  template <typename AllRoles>
-  std::optional<Role> Handle(AllRoles &role, AppendRequest<WriteOperation> &&req, RequestId request_id,
-                             Address from_address) {
-    AppendResponse res{
-        .success = false,
-        .term = state_.term,
-        .last_log_term = CommittedLogTerm(),
-        .last_log_index = CommittedLogIndex(),
-    };
-
-    if constexpr (std::is_same<AllRoles, Leader>()) {
-      MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
-    }
-
-    const bool is_candidate = std::is_same<AllRoles, Candidate>();
-    const bool is_failed_competitor = is_candidate && req.term == state_.term;
-    const Time now = io_.Now();
-
-    // Raft paper - 5.2
-    // While waiting for votes, a candidate may receive an
-    // AppendEntries RPC from another server claiming to be leader. If
-    // the leader’s term (included in its RPC) is at least as large as
-    // the candidate’s current term, then the candidate recognizes the
-    // leader as legitimate and returns to follower state.
-    if (req.term > state_.term || is_failed_competitor) {
-      // become follower of this leader, reply with our log status
-      state_.term = req.term;
-
-      io_.Send(from_address, request_id, res);
-
-      Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
-      return Follower{
-          .last_received_append_entries_timestamp = now,
-          .leader_address = from_address,
-      };
-    } else if (req.term < state_.term) {
-      // nack this request from an old leader
-      io_.Send(from_address, request_id, res);
-
-      return std::nullopt;
-    }
-
-    // at this point, we're dealing with our own leader
-    if constexpr (std::is_same<AllRoles, Follower>()) {
-      // small specialization for when we're already a Follower
-      MG_ASSERT(role.leader_address == from_address, "Multiple Leaders are acting under the same term number!");
-      role.last_received_append_entries_timestamp = now;
-    } else {
-      Log("Somehow entered Follower-specific logic as a non-Follower");
-      MG_ASSERT(false, "Somehow entered Follower-specific logic as a non-Follower");
-    }
-
-    res.last_log_term = LastLogTerm();
-    res.last_log_index = LastLogIndex();
-
-    Log("returning last_log_index of ", res.last_log_index);
-
-    // Handle steady-state conditions.
-    if (req.last_log_index != LastLogIndex()) {
-      Log("req.last_log_index is above our last applied log index");
-    } else if (req.last_log_term != LastLogTerm()) {
-      Log("req.last_log_term differs from our leader term at that slot, expected: ", LastLogTerm(), " but got ",
-          req.last_log_term);
-    } else {
-      // happy path - Apply log
-      Log("applying batch of entries to log of size ", req.entries.size());
-
-      MG_ASSERT(req.last_log_index >= state_.committed_log_size,
-                "Applied history from Leader which goes back in time from our commit_index");
-
-      // possibly chop-off stuff that was replaced by
-      // things with different terms (we got data that
-      // hasn't reached consensus yet, which is normal)
-      state_.log.resize(req.last_log_index);
-
-      state_.log.insert(state_.log.end(), req.entries.begin(), req.entries.end());
-
-      MG_ASSERT(req.leader_commit >= state_.committed_log_size);
-      state_.committed_log_size = std::min(req.leader_commit, LastLogIndex());
-
-      for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
-        const auto &write_request = state_.log[state_.applied_size].second;
-        replicated_state_.Apply(write_request);
-      }
-
-      res.success = true;
-    }
-
-    io_.Send(from_address, request_id, res);
-
-    return std::nullopt;
-  }
-
-  std::optional<Role> Handle(Leader &leader, AppendResponse &&res, RequestId, Address from_address) {
-    if (res.term != state_.term) {
-    } else if (!leader.followers.contains(from_address)) {
-      Log("received AppendResponse from unknown Follower");
-      MG_ASSERT(false, "received AppendResponse from unknown Follower");
-    } else {
-      if (res.success) {
-        Log("got successful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
-            res.last_log_index);
-      } else {
-        Log("got unsuccessful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
-            res.last_log_index);
-      }
-      FollowerTracker &follower = leader.followers.at(from_address);
-      follower.next_index = std::max(follower.next_index, res.last_log_index);
-      follower.confirmed_contiguous_index = std::max(follower.confirmed_contiguous_index, res.last_log_index);
-
-      BumpCommitIndexAndReplyToClients(leader);
-    }
-    return std::nullopt;
-  }
-
-  template <typename AllRoles>
-  std::optional<Role> Handle(AllRoles &, AppendResponse &&, RequestId, Address) {
-    // we used to be the leader, and are getting old delayed responses
-    return std::nullopt;
-  }
-
-  /////////////////////////////////////////////////////////////
-  /// RSM-related handle methods
-  /////////////////////////////////////////////////////////////
-
-  // Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
-  std::optional<Role> Handle(Leader &, ReadRequest<ReadOperation> &&req, RequestId request_id, Address from_address) {
-    Log("handling ReadOperation");
-    ReadOperation read_operation = req.operation;
-
-    ReadResponseValue read_return = replicated_state_.Read(read_operation);
-
-    ReadResponse<ReadResponseValue> resp{
-        .success = true,
-        .read_return = std::move(read_return),
-        .retry_leader = std::nullopt,
-    };
-
-    io_.Send(from_address, request_id, resp);
-
-    return std::nullopt;
-  }
-
-  // Candidates should respond with a failure, similar to the Candidate + WriteRequest failure below
-  std::optional<Role> Handle(Candidate &, ReadRequest<ReadOperation> &&, RequestId request_id, Address from_address) {
-    Log("received ReadOperation - not redirecting because no Leader is known");
-    auto res = ReadResponse<ReadResponseValue>{};
-
-    res.success = false;
-
-    Cron();
-
-    io_.Send(from_address, request_id, res);
-
-    return std::nullopt;
-  }
-
-  // Followers should respond with a redirection, similar to the Follower + WriteRequest response below
-  std::optional<Role> Handle(Follower &follower, ReadRequest<ReadOperation> &&, RequestId request_id,
-                             Address from_address) {
-    auto res = ReadResponse<ReadResponseValue>{};
-
-    res.success = false;
-    Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
-    res.retry_leader = follower.leader_address;
-
-    io_.Send(from_address, request_id, res);
-
-    return std::nullopt;
-  }
-
-  // Raft paper - 8
-  // When a client first starts up, it connects to a randomly chosen
-  // server. If the client’s first choice is not the leader, that
-  // server will reject the client’s request and supply information
-  // about the most recent leader it has heard from.
-  std::optional<Role> Handle(Follower &follower, WriteRequest<WriteOperation> &&, RequestId request_id,
-                             Address from_address) {
-    auto res = WriteResponse<WriteResponseValue>{};
-
-    res.success = false;
-    Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
-    res.retry_leader = follower.leader_address;
-
-    io_.Send(from_address, request_id, res);
-
-    return std::nullopt;
-  }
-
-  std::optional<Role> Handle(Candidate &, WriteRequest<WriteOperation> &&, RequestId request_id, Address from_address) {
-    Log("received WriteRequest - not redirecting because no Leader is known");
-    auto res = WriteResponse<WriteResponseValue>{};
-
-    res.success = false;
-
-    Cron();
-
-    io_.Send(from_address, request_id, res);
-
-    return std::nullopt;
-  }
-
-  // only leaders actually handle replication requests from clients
-  std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
-                             Address from_address) {
-    Log("handling WriteRequest");
-
-    // we are the leader. add item to log and send Append to peers
-    state_.log.emplace_back(std::pair(state_.term, std::move(req.operation)));
-
-    LogIndex log_index = state_.log.size() - 1;
-
-    PendingClientRequest pcr{
-        .request_id = request_id,
-        .address = from_address,
-        .received_at = io_.Now(),
-    };
-
-    leader.pending_client_requests.emplace(log_index, pcr);
-
-    BroadcastAppendEntries(leader.followers);
-
-    return std::nullopt;
-  }
-};
-
-};  // namespace memgraph::io::rsm

src/io/rsm/rsm_client.hpp

@@ -1,137 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include <iostream>
-#include <optional>
-#include <vector>
-
-#include "io/address.hpp"
-#include "io/rsm/raft.hpp"
-#include "utils/result.hpp"
-
-namespace memgraph::io::rsm {
-
-using memgraph::io::Address;
-using memgraph::io::Duration;
-using memgraph::io::ResponseEnvelope;
-using memgraph::io::ResponseFuture;
-using memgraph::io::ResponseResult;
-using memgraph::io::Time;
-using memgraph::io::TimedOut;
-using memgraph::io::rsm::ReadRequest;
-using memgraph::io::rsm::ReadResponse;
-using memgraph::io::rsm::WriteRequest;
-using memgraph::io::rsm::WriteResponse;
-using memgraph::utils::BasicResult;
-
-template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
-          typename ReadResponseT>
-class RsmClient {
-  using ServerPool = std::vector<Address>;
-
-  IoImpl io_;
-  Address leader_;
-
-  std::mt19937 cli_rng_{0};
-  ServerPool server_addrs_;
-
-  template <typename ResponseT>
-  void PossiblyRedirectLeader(const ResponseT &response) {
-    if (response.retry_leader) {
-      MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
-      leader_ = response.retry_leader.value();
-      spdlog::debug("client redirected to leader server {}", leader_.ToString());
-    } else if (!response.success) {
-      std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
-      size_t addr_index = addr_distrib(cli_rng_);
-      leader_ = server_addrs_[addr_index];
-
-      spdlog::debug(
-          "client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
-          "a RSM Candidate) trying a random one at index {} with address {}",
-          addr_index, leader_.ToString());
-    }
-  }
-
- public:
-  RsmClient(IoImpl io, Address leader, ServerPool server_addrs)
-      : io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
-
-  RsmClient() = delete;
-
-  BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
-    WriteRequest<WriteRequestT> client_req;
-    client_req.operation = req;
-
-    const Duration overall_timeout = io_.GetDefaultTimeout();
-    const Time before = io_.Now();
-
-    do {
-      spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
-      ResponseFuture<WriteResponse<WriteResponseT>> response_future =
-          io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
-      ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
-
-      if (response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        // continue;
-        return response_result.GetError();
-      }
-
-      ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
-      WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
-
-      if (write_response.success) {
-        return std::move(write_response.write_return);
-      }
-
-      PossiblyRedirectLeader(write_response);
-    } while (io_.Now() < before + overall_timeout);
-
-    return TimedOut{};
-  }
-
-  BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
-    ReadRequest<ReadRequestT> read_req;
-    read_req.operation = req;
-
-    const Duration overall_timeout = io_.GetDefaultTimeout();
-    const Time before = io_.Now();
-
-    do {
-      spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
-
-      ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
-          io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
-
-      // receive response
-      ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
-
-      if (get_response_result.HasError()) {
-        spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
-        return get_response_result.GetError();
-      }
-
-      ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
-      ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
-
-      if (read_get_response.success) {
-        return std::move(read_get_response.read_return);
-      }
-
-      PossiblyRedirectLeader(read_get_response);
-    } while (io_.Now() < before + overall_timeout);
-
-    return TimedOut{};
-  }
-};
-
-}  // namespace memgraph::io::rsm

src/io/rsm/shard_rsm.hpp

@@ -1,165 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-/// The StorageRsm is a simple in-memory raft-backed kv store that can be used for simple testing
-/// and implementation of some query engine logic before storage engines are fully implemented.
-///
-/// To implement multiple read and write commands, change the StorageRead* and StorageWrite* requests
-/// and responses to a std::variant of the different options, and route them to specific handlers in
-/// the StorageRsm's Read and Apply methods. Remember that Read is called immediately when the Raft
-/// leader receives the request, and does not replicate anything over Raft. Apply is called only
-/// AFTER the StorageWriteRequest is replicated to a majority of Raft peers, and the result of calling
-/// StorageRsm::Apply(StorageWriteRequest) is returned to the client that submitted the request.
-
-#include <deque>
-#include <iostream>
-#include <map>
-#include <optional>
-#include <set>
-#include <thread>
-#include <vector>
-
-#include "coordinator/hybrid_logical_clock.hpp"
-#include "io/address.hpp"
-#include "io/rsm/raft.hpp"
-#include "io/simulator/simulator.hpp"
-#include "io/simulator/simulator_transport.hpp"
-#include "storage/v2/property_value.hpp"
-#include "utils/logging.hpp"
-
-using memgraph::coordinator::Hlc;
-using memgraph::io::Address;
-using memgraph::io::Io;
-using memgraph::io::ResponseEnvelope;
-using memgraph::io::ResponseFuture;
-using memgraph::io::ResponseResult;
-using memgraph::io::rsm::Raft;
-using memgraph::io::rsm::ReadRequest;
-using memgraph::io::rsm::ReadResponse;
-using memgraph::io::rsm::WriteRequest;
-using memgraph::io::rsm::WriteResponse;
-using memgraph::io::simulator::Simulator;
-using memgraph::io::simulator::SimulatorConfig;
-using memgraph::io::simulator::SimulatorStats;
-using memgraph::io::simulator::SimulatorTransport;
-using memgraph::storage::PropertyValue;
-
-namespace memgraph::io::rsm {
-
-using ShardRsmKey = std::vector<PropertyValue>;
-
-struct StorageWriteRequest {
-  ShardRsmKey key;
-  std::optional<int> value;
-};
-
-struct StorageWriteResponse {
-  bool shard_rsm_success;
-  std::optional<int> last_value;
-  // Only has a value if the given shard does not contain the requested key
-  std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
-};
-
-struct StorageReadRequest {
-  ShardRsmKey key;
-};
-
-struct StorageReadResponse {
-  bool shard_rsm_success;
-  std::optional<int> value;
-  // Only has a value if the given shard does not contain the requested key
-  std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
-};
-
-class StorageRsm {
-  std::map<ShardRsmKey, int> state_;
-  ShardRsmKey minimum_key_;
-  std::optional<ShardRsmKey> maximum_key_{std::nullopt};
-  Hlc shard_map_version_;
-
-  // The key is not located in this shard
-  bool IsKeyInRange(const ShardRsmKey &key) {
-    if (maximum_key_) [[likely]] {
-      return (key >= minimum_key_ && key <= maximum_key_);
-    }
-    return key >= minimum_key_;
-  }
-
- public:
-  StorageReadResponse Read(StorageReadRequest request) {
-    StorageReadResponse ret;
-
-    if (!IsKeyInRange(request.key)) {
-      ret.latest_known_shard_map_version = shard_map_version_;
-      ret.shard_rsm_success = false;
-    } else if (state_.contains(request.key)) {
-      ret.value = state_[request.key];
-      ret.shard_rsm_success = true;
-    } else {
-      ret.shard_rsm_success = false;
-      ret.value = std::nullopt;
-    }
-    return ret;
-  }
-
-  StorageWriteResponse Apply(StorageWriteRequest request) {
-    StorageWriteResponse ret;
-
-    // Key is outside the prohibited range
-    if (!IsKeyInRange(request.key)) {
-      ret.latest_known_shard_map_version = shard_map_version_;
-      ret.shard_rsm_success = false;
-    }
-    // Key exist
-    else if (state_.contains(request.key)) {
-      auto &val = state_[request.key];
-
-      /*
-       *   Delete
-       */
-      if (!request.value) {
-        ret.shard_rsm_success = true;
-        ret.last_value = val;
-        state_.erase(state_.find(request.key));
-      }
-
-      /*
-       *   Update
-       */
-      // Does old_value match?
-      if (request.value == val) {
-        ret.last_value = val;
-        ret.shard_rsm_success = true;
-
-        val = request.value.value();
-
-      } else {
-        ret.last_value = val;
-        ret.shard_rsm_success = false;
-      }
-    }
-    /*
-     *   Create
-     */
-    else {
-      ret.last_value = std::nullopt;
-      ret.shard_rsm_success = true;
-
-      state_.emplace(request.key, std::move(request.value).value());
-    }
-
-    return ret;
-  }
-};
-
-}  // namespace memgraph::io::rsm

src/io/simulator/CMakeLists.txt

@@ -1,8 +0,0 @@
-set(io_simulator_sources
-    simulator_handle.cpp)
-
-find_package(fmt REQUIRED)
-find_package(Threads REQUIRED)
-
-add_library(mg-io-simulator STATIC ${io_simulator_sources})
-target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils)

src/io/simulator/message_conversion.hpp

@@ -1,177 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include "io/transport.hpp"
-
-namespace memgraph::io::simulator {
-
-using memgraph::io::Duration;
-using memgraph::io::Message;
-using memgraph::io::Time;
-
-struct OpaqueMessage {
-  Address from_address;
-  uint64_t request_id;
-  std::any message;
-
-  /// Recursively tries to match a specific type from the outer
-  /// variant's parameter pack against the type of the std::any,
-  /// and if it matches, make it concrete and return it. Otherwise,
-  /// move on and compare the any with the next type from the
-  /// parameter pack.
-  ///
-  /// Return is the full std::variant<Ts...> type that holds the
-  /// full parameter pack without interfering with recursive
-  /// narrowing expansion.
-  template <typename Return, Message Head, Message... Rest>
-  std::optional<Return> Unpack(std::any &&a) {
-    if (typeid(Head) == a.type()) {
-      Head concrete = std::any_cast<Head>(std::move(a));
-      return concrete;
-    }
-
-    if constexpr (sizeof...(Rest) > 0) {
-      return Unpack<Return, Rest...>(std::move(a));
-    } else {
-      return std::nullopt;
-    }
-  }
-
-  /// High level "user-facing" conversion function that lets
-  /// people interested in conversion only supply a single
-  /// parameter pack for the types that they want to compare
-  /// with the any and potentially include in the returned
-  /// variant.
-  template <Message... Ms>
-  requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
-    return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
-  }
-
-  template <Message... Ms>
-  requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
-    std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
-
-    if (m_opt) {
-      return RequestEnvelope<Ms...>{
-          .message = std::move(*m_opt),
-          .request_id = request_id,
-          .from_address = from_address,
-      };
-    }
-
-    return std::nullopt;
-  }
-};
-
-class OpaquePromiseTraitBase {
- public:
-  virtual const std::type_info *TypeInfo() const = 0;
-  virtual bool IsAwaited(void *ptr) const = 0;
-  virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
-  virtual void TimeOut(void *ptr) const = 0;
-
-  virtual ~OpaquePromiseTraitBase() = default;
-  OpaquePromiseTraitBase() = default;
-  OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete;
-  OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete;
-  OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete;
-  OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete;
-};
-
-template <typename T>
-class OpaquePromiseTrait : public OpaquePromiseTraitBase {
- public:
-  const std::type_info *TypeInfo() const override { return &typeid(T); };
-
-  bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
-
-  void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
-    T message = std::any_cast<T>(std::move(opaque_message.message));
-    auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
-                                                 .request_id = opaque_message.request_id,
-                                                 .from_address = opaque_message.from_address};
-    auto promise = static_cast<ResponsePromise<T> *>(ptr);
-    auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
-    unique_promise->Fill(std::move(response_envelope));
-  };
-
-  void TimeOut(void *ptr) const override {
-    auto promise = static_cast<ResponsePromise<T> *>(ptr);
-    auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
-    ResponseResult<T> result = TimedOut{};
-    unique_promise->Fill(std::move(result));
-  }
-};
-
-class OpaquePromise {
-  void *ptr_;
-  std::unique_ptr<OpaquePromiseTraitBase> trait_;
-
- public:
-  OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) {
-    MG_ASSERT(old.ptr_ != nullptr);
-    old.ptr_ = nullptr;
-  }
-
-  OpaquePromise &operator=(OpaquePromise &&old) noexcept {
-    MG_ASSERT(ptr_ == nullptr);
-    MG_ASSERT(old.ptr_ != nullptr);
-    MG_ASSERT(this != &old);
-    ptr_ = old.ptr_;
-    trait_ = std::move(old.trait_);
-    old.ptr_ = nullptr;
-    return *this;
-  }
-
-  OpaquePromise(const OpaquePromise &) = delete;
-  OpaquePromise &operator=(const OpaquePromise &) = delete;
-
-  template <typename T>
-  std::unique_ptr<ResponsePromise<T>> Take() && {
-    MG_ASSERT(typeid(T) == *trait_->TypeInfo());
-    MG_ASSERT(ptr_ != nullptr);
-
-    auto ptr = static_cast<ResponsePromise<T> *>(ptr_);
-
-    ptr_ = nullptr;
-
-    return std::unique_ptr<T>(ptr);
-  }
-
-  template <typename T>
-  explicit OpaquePromise(std::unique_ptr<ResponsePromise<T>> promise)
-      : ptr_(static_cast<void *>(promise.release())), trait_(std::make_unique<OpaquePromiseTrait<T>>()) {}
-
-  bool IsAwaited() {
-    MG_ASSERT(ptr_ != nullptr);
-    return trait_->IsAwaited(ptr_);
-  }
-
-  void TimeOut() {
-    MG_ASSERT(ptr_ != nullptr);
-    trait_->TimeOut(ptr_);
-    ptr_ = nullptr;
-  }
-
-  void Fill(OpaqueMessage &&opaque_message) {
-    MG_ASSERT(ptr_ != nullptr);
-    trait_->Fill(ptr_, std::move(opaque_message));
-    ptr_ = nullptr;
-  }
-
-  ~OpaquePromise() {
-    MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled");
-  }
-};
-
-}  // namespace memgraph::io::simulator

src/io/simulator/simulator.hpp

@@ -1,51 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <memory>
-#include <random>
-
-#include "io/address.hpp"
-#include "io/simulator/simulator_config.hpp"
-#include "io/simulator/simulator_handle.hpp"
-#include "io/simulator/simulator_transport.hpp"
-
-namespace memgraph::io::simulator {
-class Simulator {
-  std::mt19937 rng_;
-  std::shared_ptr<SimulatorHandle> simulator_handle_;
-  uint16_t auto_port_ = 0;
-
- public:
-  explicit Simulator(SimulatorConfig config)
-      : rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
-
-  void ShutDown() { simulator_handle_->ShutDown(); }
-
-  Io<SimulatorTransport> RegisterNew() {
-    Address address = Address::TestAddress(auto_port_++);
-    return Register(address);
-  }
-
-  Io<SimulatorTransport> Register(Address address) {
-    std::uniform_int_distribution<uint64_t> seed_distrib;
-    uint64_t seed = seed_distrib(rng_);
-    return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
-  }
-
-  void IncrementServerCountAndWaitForQuiescentState(Address address) {
-    simulator_handle_->IncrementServerCountAndWaitForQuiescentState(address);
-  }
-
-  SimulatorStats Stats() { return simulator_handle_->Stats(); }
-};
-};  // namespace memgraph::io::simulator

src/io/simulator/simulator_config.hpp

@@ -1,30 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <chrono>
-
-#include "io/time.hpp"
-
-namespace memgraph::io::simulator {
-
-using memgraph::io::Time;
-
-struct SimulatorConfig {
-  uint8_t drop_percent = 0;
-  bool perform_timeouts = false;
-  bool scramble_messages = true;
-  uint64_t rng_seed = 0;
-  Time start_time = Time::min();
-  Time abort_time = Time::max();
-};
-};  // namespace memgraph::io::simulator

src/io/simulator/simulator_handle.cpp

@@ -1,154 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "io/simulator/simulator_handle.hpp"
-#include "io/address.hpp"
-#include "io/errors.hpp"
-#include "io/simulator/simulator_config.hpp"
-#include "io/simulator/simulator_stats.hpp"
-#include "io/time.hpp"
-#include "io/transport.hpp"
-
-namespace memgraph::io::simulator {
-
-using memgraph::io::Duration;
-using memgraph::io::Time;
-
-void SimulatorHandle::ShutDown() {
-  std::unique_lock<std::mutex> lock(mu_);
-  should_shut_down_ = true;
-  cv_.notify_all();
-}
-
-bool SimulatorHandle::ShouldShutDown() const {
-  std::unique_lock<std::mutex> lock(mu_);
-  return should_shut_down_;
-}
-
-void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
-  std::unique_lock<std::mutex> lock(mu_);
-  server_addresses_.insert(address);
-
-  while (true) {
-    const size_t blocked_servers = BlockedServers();
-
-    const bool all_servers_blocked = blocked_servers == server_addresses_.size();
-
-    if (all_servers_blocked) {
-      return;
-    }
-
-    cv_.wait(lock);
-  }
-}
-
-size_t SimulatorHandle::BlockedServers() {
-  size_t blocked_servers = blocked_on_receive_;
-
-  for (auto &[promise_key, opaque_promise] : promises_) {
-    if (opaque_promise.promise.IsAwaited() && server_addresses_.contains(promise_key.requester_address)) {
-      blocked_servers++;
-    }
-  }
-
-  return blocked_servers;
-}
-
-bool SimulatorHandle::MaybeTickSimulator() {
-  std::unique_lock<std::mutex> lock(mu_);
-
-  const size_t blocked_servers = BlockedServers();
-
-  if (blocked_servers < server_addresses_.size()) {
-    // we only need to advance the simulator when all
-    // servers have reached a quiescent state, blocked
-    // on their own futures or receive methods.
-    return false;
-  }
-
-  stats_.simulator_ticks++;
-
-  cv_.notify_all();
-
-  TimeoutPromisesPastDeadline();
-
-  if (in_flight_.empty()) {
-    // return early here because there are no messages to schedule
-
-    // We tick the clock forward when all servers are blocked but
-    // there are no in-flight messages to schedule delivery of.
-    std::poisson_distribution<> time_distrib(50);
-    Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
-    cluster_wide_time_microseconds_ += clock_advance;
-
-    MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
-              "Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
-              "in an expected amount of time.");
-    return true;
-  }
-
-  if (config_.scramble_messages) {
-    // scramble messages
-    std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
-    const size_t swap_index = swap_distrib(rng_);
-    std::swap(in_flight_[swap_index], in_flight_.back());
-  }
-
-  auto [to_address, opaque_message] = std::move(in_flight_.back());
-  in_flight_.pop_back();
-
-  std::uniform_int_distribution<int> drop_distrib(0, 99);
-  const int drop_threshold = drop_distrib(rng_);
-  const bool should_drop = drop_threshold < config_.drop_percent;
-
-  if (should_drop) {
-    stats_.dropped_messages++;
-  }
-
-  PromiseKey promise_key{.requester_address = to_address,
-                         .request_id = opaque_message.request_id,
-                         .replier_address = opaque_message.from_address};
-
-  if (promises_.contains(promise_key)) {
-    // complete waiting promise if it's there
-    DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
-    promises_.erase(promise_key);
-
-    const bool normal_timeout = config_.perform_timeouts && (dop.deadline < cluster_wide_time_microseconds_);
-
-    if (should_drop || normal_timeout) {
-      stats_.timed_out_requests++;
-      dop.promise.TimeOut();
-    } else {
-      stats_.total_responses++;
-      dop.promise.Fill(std::move(opaque_message));
-    }
-  } else if (should_drop) {
-    // don't add it anywhere, let it drop
-  } else {
-    // add to can_receive_ if not
-    const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
-    om_vec->second.emplace_back(std::move(opaque_message));
-  }
-
-  return true;
-}
-
-Time SimulatorHandle::Now() const {
-  std::unique_lock<std::mutex> lock(mu_);
-  return cluster_wide_time_microseconds_;
-}
-
-SimulatorStats SimulatorHandle::Stats() {
-  std::unique_lock<std::mutex> lock(mu_);
-  return stats_;
-}
-}  // namespace memgraph::io::simulator

src/io/simulator/simulator_handle.hpp

@@ -1,206 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <any>
-#include <compare>
-#include <iostream>
-#include <map>
-#include <memory>
-#include <optional>
-#include <set>
-#include <utility>
-#include <variant>
-#include <vector>
-
-#include "io/address.hpp"
-#include "io/errors.hpp"
-#include "io/simulator/message_conversion.hpp"
-#include "io/simulator/simulator_config.hpp"
-#include "io/simulator/simulator_stats.hpp"
-#include "io/time.hpp"
-#include "io/transport.hpp"
-
-namespace memgraph::io::simulator {
-
-using memgraph::io::Duration;
-using memgraph::io::Time;
-
-struct PromiseKey {
-  Address requester_address;
-  uint64_t request_id;
-  // TODO(tyler) possibly remove replier_address from promise key
-  // once we want to support DSR.
-  Address replier_address;
-
- public:
-  friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
-    if (lhs.requester_address != rhs.requester_address) {
-      return lhs.requester_address < rhs.requester_address;
-    }
-
-    if (lhs.request_id != rhs.request_id) {
-      return lhs.request_id < rhs.request_id;
-    }
-
-    return lhs.replier_address < rhs.replier_address;
-  }
-};
-
-struct DeadlineAndOpaquePromise {
-  Time deadline;
-  OpaquePromise promise;
-};
-
-class SimulatorHandle {
-  mutable std::mutex mu_{};
-  mutable std::condition_variable cv_;
-
-  // messages that have not yet been scheduled or dropped
-  std::vector<std::pair<Address, OpaqueMessage>> in_flight_;
-
-  // the responses to requests that are being waited on
-  std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
-
-  // messages that are sent to servers that may later receive them
-  std::map<Address, std::vector<OpaqueMessage>> can_receive_;
-
-  Time cluster_wide_time_microseconds_;
-  bool should_shut_down_ = false;
-  SimulatorStats stats_;
-  size_t blocked_on_receive_ = 0;
-  std::set<Address> server_addresses_;
-  std::mt19937 rng_;
-  SimulatorConfig config_;
-
-  /// Returns the number of servers currently blocked on Receive, plus
-  /// the servers that are blocked on Futures that were created through
-  /// SimulatorTransport::Request.
-  ///
-  /// TODO(tyler) investigate whether avoiding consideration of Futures
-  /// increases determinism.
-  size_t BlockedServers();
-
-  void TimeoutPromisesPastDeadline() {
-    const Time now = cluster_wide_time_microseconds_;
-
-    for (auto &[promise_key, dop] : promises_) {
-      if (dop.deadline < now) {
-        spdlog::debug("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
-                      promise_key.replier_address.ToString());
-        std::move(dop).promise.TimeOut();
-        promises_.erase(promise_key);
-
-        stats_.timed_out_requests++;
-      }
-    }
-  }
-
- public:
-  explicit SimulatorHandle(SimulatorConfig config)
-      : cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
-
-  void IncrementServerCountAndWaitForQuiescentState(Address address);
-
-  /// This method causes most of the interesting simulation logic to happen, wrt network behavior.
-  /// It checks to see if all background "server" threads are blocked on new messages, and if so,
-  /// it will decide whether to drop, reorder, or deliver in-flight messages based on the SimulatorConfig
-  /// that was used to create the Simulator.
-  bool MaybeTickSimulator();
-
-  void ShutDown();
-
-  bool ShouldShutDown() const;
-
-  template <Message Request, Message Response>
-  void SubmitRequest(Address to_address, Address from_address, uint64_t request_id, Request &&request, Duration timeout,
-                     ResponsePromise<Response> &&promise) {
-    std::unique_lock<std::mutex> lock(mu_);
-
-    const Time deadline = cluster_wide_time_microseconds_ + timeout;
-
-    std::any message(request);
-    OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message)};
-    in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
-
-    PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
-    OpaquePromise opaque_promise(std::move(promise).ToUnique());
-    DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
-    promises_.emplace(std::move(promise_key), std::move(dop));
-
-    stats_.total_messages++;
-    stats_.total_requests++;
-
-    cv_.notify_all();
-  }
-
-  template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
-    std::unique_lock<std::mutex> lock(mu_);
-
-    blocked_on_receive_ += 1;
-
-    const Time deadline = cluster_wide_time_microseconds_ + timeout;
-
-    while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
-      if (can_receive_.contains(receiver)) {
-        std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
-        if (!can_rx.empty()) {
-          OpaqueMessage message = std::move(can_rx.back());
-          can_rx.pop_back();
-
-          // TODO(tyler) search for item in can_receive_ that matches the desired types, rather
-          // than asserting that the last item in can_rx matches.
-          auto m_opt = std::move(message).Take<Ms...>();
-
-          blocked_on_receive_ -= 1;
-
-          return std::move(m_opt).value();
-        }
-      }
-
-      lock.unlock();
-      bool made_progress = MaybeTickSimulator();
-      lock.lock();
-      if (!should_shut_down_ && !made_progress) {
-        cv_.wait(lock);
-      }
-    }
-
-    blocked_on_receive_ -= 1;
-
-    return TimedOut{};
-  }
-
-  template <Message M>
-  void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
-    std::unique_lock<std::mutex> lock(mu_);
-    std::any message_any(std::move(message));
-    OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message_any)};
-    in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
-
-    stats_.total_messages++;
-
-    cv_.notify_all();
-  }
-
-  Time Now() const;
-
-  template <class D = std::poisson_distribution<>, class Return = uint64_t>
-  Return Rand(D distrib) {
-    std::unique_lock<std::mutex> lock(mu_);
-    return distrib(rng_);
-  }
-
-  SimulatorStats Stats();
-};
-};  // namespace memgraph::io::simulator

src/io/simulator/simulator_stats.hpp

@@ -1,25 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <cstdint>
-
-namespace memgraph::io::simulator {
-struct SimulatorStats {
-  uint64_t total_messages = 0;
-  uint64_t dropped_messages = 0;
-  uint64_t timed_out_requests = 0;
-  uint64_t total_requests = 0;
-  uint64_t total_responses = 0;
-  uint64_t simulator_ticks = 0;
-};
-};  // namespace memgraph::io::simulator

src/io/simulator/simulator_transport.hpp

@@ -1,65 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <memory>
-#include <utility>
-
-#include "io/address.hpp"
-#include "io/simulator/simulator_handle.hpp"
-#include "io/time.hpp"
-
-namespace memgraph::io::simulator {
-
-using memgraph::io::Duration;
-using memgraph::io::Time;
-
-class SimulatorTransport {
-  std::shared_ptr<SimulatorHandle> simulator_handle_;
-  const Address address_;
-  std::mt19937 rng_;
-
- public:
-  SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
-      : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
-
-  template <Message Request, Message Response>
-  ResponseFuture<Response> Request(Address address, uint64_t request_id, Request request, Duration timeout) {
-    std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
-    auto [future, promise] =
-        memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(maybe_tick_simulator);
-
-    simulator_handle_->SubmitRequest(address, address_, request_id, std::move(request), timeout, std::move(promise));
-
-    return std::move(future);
-  }
-
-  template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Duration timeout) {
-    return simulator_handle_->template Receive<Ms...>(address_, timeout);
-  }
-
-  template <Message M>
-  void Send(Address address, uint64_t request_id, M message) {
-    return simulator_handle_->template Send<M>(address, address_, request_id, message);
-  }
-
-  Time Now() const { return simulator_handle_->Now(); }
-
-  bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
-
-  template <class D = std::poisson_distribution<>, class Return = uint64_t>
-  Return Rand(D distrib) {
-    return distrib(rng_);
-  }
-};
-};  // namespace memgraph::io::simulator

src/io/time.hpp

@@ -1,21 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <chrono>
-
-namespace memgraph::io {
-
-using Duration = std::chrono::microseconds;
-using Time = std::chrono::time_point<std::chrono::local_t, Duration>;
-
-}  // namespace memgraph::io

src/io/transport.hpp

@@ -1,133 +0,0 @@
-// Copyright 2022 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#pragma once
-
-#include <chrono>
-#include <concepts>
-#include <random>
-#include <variant>
-
-#include "io/address.hpp"
-#include "io/errors.hpp"
-#include "io/future.hpp"
-#include "io/time.hpp"
-#include "utils/result.hpp"
-
-namespace memgraph::io {
-
-using memgraph::utils::BasicResult;
-
-// TODO(tyler) ensure that Message continues to represent
-// reasonable constraints around message types over time,
-// as we adapt things to use Thrift-generated message types.
-template <typename T>
-concept Message = std::same_as<T, std::decay_t<T>>;
-
-template <Message M>
-struct ResponseEnvelope {
-  M message;
-  uint64_t request_id;
-  Address from_address;
-};
-
-template <Message M>
-using ResponseResult = BasicResult<TimedOut, ResponseEnvelope<M>>;
-
-template <Message M>
-using ResponseFuture = memgraph::io::Future<ResponseResult<M>>;
-
-template <Message M>
-using ResponsePromise = memgraph::io::Promise<ResponseResult<M>>;
-
-template <Message... Ms>
-struct RequestEnvelope {
-  std::variant<Ms...> message;
-  uint64_t request_id;
-  Address from_address;
-};
-
-template <Message... Ms>
-using RequestResult = BasicResult<TimedOut, RequestEnvelope<Ms...>>;
-
-template <typename I>
-class Io {
-  I implementation_;
-  Address address_;
-  uint64_t request_id_counter_ = 0;
-  Duration default_timeout_ = std::chrono::microseconds{50000};
-
- public:
-  Io(I io, Address address) : implementation_(io), address_(address) {}
-
-  /// Set the default timeout for all requests that are issued
-  /// without an explicit timeout set.
-  void SetDefaultTimeout(Duration timeout) { default_timeout_ = timeout; }
-
-  /// Returns the current default timeout for this Io instance.
-  Duration GetDefaultTimeout() { return default_timeout_; }
-
-  /// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
-  template <Message Request, Message Response>
-  ResponseFuture<Response> RequestWithTimeout(Address address, Request request, Duration timeout) {
-    const uint64_t request_id = ++request_id_counter_;
-    return implementation_.template Request<Request, Response>(address, request_id, request, timeout);
-  }
-
-  /// Issue a request that times out after the default timeout. This tends
-  /// to be used by clients.
-  template <Message Request, Message Response>
-  ResponseFuture<Response> Request(Address address, Request request) {
-    const uint64_t request_id = ++request_id_counter_;
-    const Duration timeout = default_timeout_;
-    return implementation_.template Request<Request, Response>(address, request_id, std::move(request), timeout);
-  }
-
-  /// Wait for an explicit number of microseconds for a request of one of the
-  /// provided types to arrive. This tends to be used by servers.
-  template <Message... Ms>
-  RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
-    return implementation_.template Receive<Ms...>(timeout);
-  }
-
-  /// Wait the default number of microseconds for a request of one of the
-  /// provided types to arrive. This tends to be used by servers.
-  template <Message... Ms>
-  requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
-    const Duration timeout = default_timeout_;
-    return implementation_.template Receive<Ms...>(timeout);
-  }
-
-  /// Send a message in a best-effort fashion. This is used for messaging where
-  /// responses are not necessarily expected, and for servers to respond to requests.
-  /// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
-  template <Message M>
-  void Send(Address address, uint64_t request_id, M message) {
-    return implementation_.template Send<M>(address, request_id, std::move(message));
-  }
-
-  /// The current system time. This time source should be preferred over any other,
-  /// because it lets us deterministically control clocks from tests for making
-  /// things like timeouts deterministic.
-  Time Now() const { return implementation_.Now(); }
-
-  /// Returns true if the system should shut-down.
-  bool ShouldShutDown() const { return implementation_.ShouldShutDown(); }
-
-  /// Returns a random number within the specified distribution.
-  template <class D = std::poisson_distribution<>, class Return = uint64_t>
-  Return Rand(D distrib) {
-    return implementation_.template Rand<D, Return>(distrib);
-  }
-
-  Address GetAddress() { return address_; }
-};
-};  // namespace memgraph::io

src/memgraph.cpp

@@ -26,16 +26,22 @@
 #include <thread>
 
 #include <fmt/format.h>
+#include <folly/init/Init.h>
+#include <folly/io/async/AsyncSocket.h>
+#include <folly/io/async/AsyncTransport.h>
+#include <folly/io/async/EventBase.h>
 #include <gflags/gflags.h>
 #include <spdlog/common.h>
 #include <spdlog/sinks/daily_file_sink.h>
 #include <spdlog/sinks/dist_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
+#include <thrift/lib/cpp2/async/HeaderClientChannel.h>
 
 #include "communication/bolt/v1/constants.hpp"
 #include "communication/websocket/auth.hpp"
 #include "communication/websocket/server.hpp"
 #include "helpers.hpp"
+#include "interface/gen-cpp2/Storage.h"
 #include "py/py.hpp"
 #include "query/auth_checker.hpp"
 #include "query/discard_value_stream.hpp"
@@ -216,11 +222,6 @@ DEFINE_bool(telemetry_enabled, false,
             "the database runtime (vertex and edge counts and resource usage) "
             "to allow for easier improvement of the product.");
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_bool(storage_restore_replicas_on_startup, true,
-            "Controls replicas should be restored automatically.");  // TODO(42jeremy) this must be removed once T0835
-                                                                     // is implemented.
-
 // Streams flags
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
 DEFINE_uint32(
@@ -1200,8 +1201,7 @@ int main(int argc, char **argv) {
                      .snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
                      .wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
                      .wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
-                     .snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
-                     .restore_replicas_on_startup = FLAGS_storage_restore_replicas_on_startup},
+                     .snapshot_on_exit = FLAGS_storage_snapshot_on_exit},
       .transaction = {.isolation_level = ParseIsolationLevel()}};
   if (FLAGS_storage_snapshot_interval_sec == 0) {
     if (FLAGS_storage_wal_enabled) {
@@ -1252,8 +1252,9 @@ int main(int argc, char **argv) {
     // the triggers
     auto storage_accessor = interpreter_context.db->Access();
     auto dba = memgraph::query::DbAccessor{&storage_accessor};
-    interpreter_context.trigger_store.RestoreTriggers(
-        &interpreter_context.ast_cache, &dba, interpreter_context.config.query, interpreter_context.auth_checker);
+    interpreter_context.trigger_store.RestoreTriggers(&interpreter_context.ast_cache, &dba,
+                                                      &interpreter_context.antlr_lock, interpreter_context.config.query,
+                                                      interpreter_context.auth_checker);
   }
 
   // As the Stream transformations are using modules, they have to be restored after the query modules are loaded.
@@ -1318,8 +1319,19 @@ int main(int argc, char **argv) {
   MG_ASSERT(server.Start(), "Couldn't start the Bolt server!");
   websocket_server.Start();
 
-  server.AwaitShutdown();
-  websocket_server.AwaitShutdown();
+  folly::init(&argc, &argv);
+  folly::EventBase base;
+  int port = 7779;  // The port on which server is listening
+
+  // Create a async client socket and connect it. Change
+  // the ip address to where the server is listening
+  auto socket(folly::AsyncSocket::newSocket(&base, "127.0.0.1", port));
+
+  // Create a HeaderClientChannel object which is used in creating
+  // client object
+  auto client_channel = apache::thrift::HeaderClientChannel::newChannel(std::move(socket));
+  // Create a client object
+  interface::storage::StorageAsyncClient client(std::move(client_channel));
 
   memgraph::query::procedure::gModuleRegistry.UnloadAllModules();
 

src/memory/CMakeLists.txt

@@ -9,5 +9,4 @@ target_link_libraries(mg-memory mg-utils fmt)
 
 if (ENABLE_JEMALLOC)
   target_link_libraries(mg-memory Jemalloc::Jemalloc)
-  target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
 endif()