Fix tests for unordered configs

This reverts commit 521a085b79.

.clang-tidy

@@ -58,7 +58,8 @@ Checks: '*,
          -readability-implicit-bool-conversion,
          -readability-magic-numbers,
          -readability-named-parameter,
-         -misc-no-recursion'
+         -misc-no-recursion,
+         -concurrency-mt-unsafe'
 WarningsAsErrors: ''
 HeaderFilterRegex: 'src/.*'
 AnalyzeTemporaryDtors: false

.github/workflows/daily_benchmark.yaml

@@ -42,6 +42,16 @@ jobs:
             --groups aggregation 1000_create unwind_create dense_expand match \
             --no-strict
 
+      - name: Get branch name (merge)
+        if: github.event_name != 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
+
+      - name: Get branch name (pull request)
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+
       - name: Upload macro benchmark results
         run: |
           cd tools/bench-graph-client
@@ -51,7 +61,8 @@ jobs:
           ./main.py --benchmark-name "macro_benchmark" \
                     --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
                     --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}"
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"
 
       - name: Run mgbench
         run: |
@@ -67,4 +78,5 @@ jobs:
           ./main.py --benchmark-name "mgbench" \
                     --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
                     --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}"
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/diff.yaml

@@ -1,13 +1,16 @@
 name: Diff
 
 on:
+  push:
+    branches:
+      - master
   workflow_dispatch:
   pull_request:
     paths-ignore:
-      - 'docs/**'
-      - '**/*.md'
-      - '.clang-format'
-      - 'CODEOWNERS'
+      - "docs/**"
+      - "**/*.md"
+      - ".clang-format"
+      - "CODEOWNERS"
 
 jobs:
   community_build:
@@ -374,6 +377,16 @@ jobs:
             --groups aggregation 1000_create unwind_create dense_expand match \
             --no-strict
 
+      - name: Get branch name (merge)
+        if: github.event_name != 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
+
+      - name: Get branch name (pull request)
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+
       - name: Upload macro benchmark results
         run: |
           cd tools/bench-graph-client
@@ -383,7 +396,8 @@ jobs:
           ./main.py --benchmark-name "macro_benchmark" \
                     --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
                     --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}"
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"
 
       - name: Run mgbench
         run: |
@@ -399,4 +413,5 @@ jobs:
           ./main.py --benchmark-name "mgbench" \
                     --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
                     --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}"
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"

CMakeLists.txt

@@ -66,7 +66,7 @@ if (MG_ENTERPRISE)
 else()
   set(get_version_offering "--open-source")
 endif()
-set(get_version_script "${CMAKE_SOURCE_DIR}/release/get_version.py")
+set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
 
 # Get version that should be used in the binary.
 execute_process(
@@ -76,7 +76,7 @@ execute_process(
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
                 "--memgraph-root-dir"
-                "${CMAKE_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
 if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
     message(FATAL_ERROR "Unable to get Memgraph version.")
@@ -93,7 +93,7 @@ execute_process(
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
                 "--memgraph-root-dir"
-                "${CMAKE_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
 if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
     message(FATAL_ERROR "Unable to get Memgraph DEB version.")
@@ -110,7 +110,7 @@ execute_process(
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
                 "--memgraph-root-dir"
-                "${CMAKE_SOURCE_DIR}"
+                "${CMAKE_CURRENT_SOURCE_DIR}"
 )
 if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
     message(FATAL_ERROR "Unable to get Memgraph RPM version.")
@@ -331,8 +331,13 @@ add_subdirectory(src)
 # Release configuration
 add_subdirectory(release)
 
-enable_testing()
-add_subdirectory(tests)
+option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
+message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
+
+if (MG_ENABLE_TESTING)
+  enable_testing()
+  add_subdirectory(tests)
+endif()
 
 if(TOOLS)
   add_subdirectory(tools)

README.md

@@ -115,6 +115,14 @@ understand what actions will and will not be tolerated.
 Read our [contributing guide](CONTRIBUTING.md) to learn about our development
 process and how to propose bug fixes and improvements.
 
+### Internals
+
+Read our
+[internal](https://memgraph.notion.site/Memgraph-Internals-12b69132d67a417898972927d6870bd2)
+docs to learn more about Memgraph's architecture, how to build the project from
+source and how to start contributing. All information related to the database,
+can be found in the aforementioned docs.
+
 ### :scroll: License
 
 Memgraph Community is available under the [BSL

environment/README.md

@@ -1,4 +0,0 @@
-# Memgraph Build and Run Environments
-
-Please continue in
-[Notion](https://www.notion.so/memgraph/Tools-05e0baafb78a49b386e0063b4833d23d).

environment/os/centos-7.sh

@@ -49,6 +49,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/centos-8.sh

@@ -48,6 +48,7 @@ MEMGRAPH_BUILD_DEPS=(
     which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
     sbcl # for custom Lisp C++ preprocessing
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-10.sh

@@ -46,6 +46,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-11.sh

@@ -24,6 +24,7 @@ TOOLCHAIN_RUN_DEPS=(
     zlib1g # zlib library used for all builds
     libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
     libcurl4 # for cmake
+    file # for CPack
     libreadline8 # for cmake and llvm
     libffi7 libxml2 # for llvm
 )
@@ -46,6 +47,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-9.sh

@@ -44,6 +44,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/ubuntu-18.04.sh

@@ -45,6 +45,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/ubuntu-20.04.sh

@@ -46,6 +46,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

libs/.gitignore

@@ -5,3 +5,4 @@
 !CMakeLists.txt
 !__main.cpp
 !jemalloc.cmake
+!pulsar.patch

libs/CMakeLists.txt

@@ -48,7 +48,7 @@ endfunction(import_library)
 # INSTALL_COMMAND arguments.
 function(add_external_project name)
   set(options NO_C_COMPILER)
-  set(one_value_kwargs SOURCE_DIR)
+  set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE)
   set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
       CONFIGURE_COMMAND)
   cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
@@ -56,11 +56,16 @@ function(add_external_project name)
   if (KW_SOURCE_DIR)
     set(source_dir ${KW_SOURCE_DIR})
   endif()
+  set(build_in_source 0)
+  if (KW_BUILD_IN_SOURCE)
+    set(build_in_source ${KW_BUILD_IN_SOURCE})
+  endif()
   if (NOT KW_NO_C_COMPILER)
     set(KW_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} ${KW_CMAKE_ARGS})
   endif()
   ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
                       PREFIX ${source_dir} SOURCE_DIR ${source_dir}
+                      BUILD_IN_SOURCE ${build_in_source}
                       CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
                       CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
                                  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -169,9 +174,12 @@ import_external_library(bzip2 STATIC
     INSTALL_COMMAND true)
 
 # Setup zlib
+set(ZLIB_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/zlib)
+set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/libz.a)
+set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
 import_external_library(zlib STATIC
-    ${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a
-    ${CMAKE_CURRENT_SOURCE_DIR}/zlib
+    ${ZLIB_LIBRARIES}
+    ${ZLIB_INCLUDE_DIRS}
     CMAKE_ARGS -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
     BUILD_COMMAND $(MAKE) zlibstatic)
 
@@ -231,6 +239,7 @@ import_external_library(librdkafka STATIC
   CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
              -DRDKAFKA_BUILD_EXAMPLES=OFF
              -DRDKAFKA_BUILD_TESTS=OFF
+             -DENABLE_LZ4_EXT=OFF
              -DCMAKE_INSTALL_LIBDIR=lib
              -DWITH_SSL=ON
              # If we want SASL, we need to install it on build machines
@@ -242,3 +251,36 @@ import_library(librdkafka++ STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include
 )
 target_link_libraries(librdkafka++ INTERFACE librdkafka)
+
+set(PROTOBUF_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/protobuf/lib)
+import_external_library(protobuf STATIC
+  ${PROTOBUF_ROOT}/lib/libprotobuf.a
+  ${PROTOBUF_ROOT}/include
+  BUILD_IN_SOURCE 1
+  CONFIGURE_COMMAND true)
+
+set(BOOST_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/boost/lib)
+
+import_external_library(pulsar STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/pulsar/pulsar-client-cpp/lib/libpulsarwithdeps.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include
+  BUILD_IN_SOURCE 1
+  CONFIGURE_COMMAND cmake pulsar-client-cpp
+                            -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install
+                            -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                            -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+                            -DBUILD_DYNAMIC_LIB=OFF
+                            -DBUILD_STATIC_LIB=ON
+                            -DBUILD_TESTS=OFF
+                            -DLINK_STATIC=ON
+                            -DPROTOC_PATH=${PROTOBUF_ROOT}/bin/protoc
+                            -DBOOST_ROOT=${BOOST_ROOT}
+                            -DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT}
+                            -DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include
+                            -DZLIB_LIBRARIES=${ZLIB_LIBRARIES}
+                            -DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}
+                            -DBUILD_PYTHON_WRAPPER=OFF
+                            -DBUILD_PERF_TOOLS=OFF
+                            -DUSE_LOG4CXX=OFF
+ BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
+add_dependencies(pulsar-proj protobuf zlib)

libs/setup.sh

@@ -14,39 +14,49 @@ clone () {
   local git_repo=$1
   local dir_name=$2
   local checkout_id=$3
-  shift 3
+  local shallow=$4
+  shift 4
   # Clone if there's no repo.
   if [[ ! -d "$dir_name" ]]; then
     echo "Cloning from $git_repo"
     # If the clone fails, it doesn't make sense to continue with the function
     # execution but the whole script should continue executing because we might
     # clone the same repo from a different source.
-    git clone "$git_repo" "$dir_name" || return 1
+
+    if [ "$shallow" = true ]; then
+      git clone --depth 1 --branch "$checkout_id" "$git_repo" "$dir_name" || return 1
+    else
+      git clone "$git_repo" "$dir_name" || return 1
+    fi
   fi
   pushd "$dir_name"
-  # Just fetch new commits from remote repository. Don't merge/pull them in, so
-  # that we don't clobber local modifications.
-  git fetch
   # Check whether we have any local changes which need to be preserved.
   local local_changes=true
   if git diff --no-ext-diff --quiet && git diff --no-ext-diff --cached --quiet; then
     local_changes=false
   fi
-  # Stash regardless of local_changes, so that a user gets a message on stdout.
-  git stash
-  # Checkout the primary commit (there's no need to pull/merge).
-  # The checkout fail should exit this script immediately because the target
-  # commit is not there and that will most likely create build-time errors.
-  git checkout "$checkout_id" || exit 1
-  # Apply any optional cherry pick fixes.
-  while [[ $# -ne 0 ]]; do
-    local cherry_pick_id=$1
-    shift
-    # The cherry-pick fail should exit this script immediately because the
-    # target commit is not there and that will most likely create build-time
-    # errors.
-    git cherry-pick -n "$cherry_pick_id" || exit 1
-  done
+
+  if [ "$shallow" = false ]; then
+    # Stash regardless of local_changes, so that a user gets a message on stdout.
+    git stash
+    # Just fetch new commits from remote repository. Don't merge/pull them in, so
+    # that we don't clobber local modifications.
+    git fetch
+    # Checkout the primary commit (there's no need to pull/merge).
+    # The checkout fail should exit this script immediately because the target
+    # commit is not there and that will most likely create build-time errors.
+    git checkout "$checkout_id" || exit 1
+    # Apply any optional cherry pick fixes.
+    while [[ $# -ne 0 ]]; do
+      local cherry_pick_id=$1
+      shift
+      # The cherry-pick fail should exit this script immediately because the
+      # target commit is not there and that will most likely create build-time
+      # errors.
+      git cherry-pick -n "$cherry_pick_id" || exit 1
+    done
+  fi
+
   # Reapply any local changes.
   if [[ $local_changes == true ]]; then
     git stash pop
@@ -70,12 +80,13 @@ repo_clone_try_double () {
     secondary_url="$2"
     folder_name="$3"
     ref="$4"
+    shallow="${5:-false}"
     echo "Cloning primary from $primary_url secondary from $secondary_url"
     if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
     if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
     if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
     if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
-    clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
+    clone "$primary_url" "$folder_name" "$ref" "$shallow" || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
     echo ""
 }
 
@@ -113,6 +124,9 @@ declare -A primary_urls=(
   ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
   ["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
+  ["protobuf"]="http://$local_cache_host/git/protobuf.git"
+  ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+  ["pulsar"]="https://github.com/apache/pulsar.git"
 )
 
 # The goal of secondary urls is to have links to the "source of truth" of
@@ -140,13 +154,16 @@ declare -A secondary_urls=(
   ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
   ["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="https://github.com/edenhill/librdkafka.git"
+  ["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
+  ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+  ["pulsar"]="https://github.com/apache/pulsar.git"
 )
 
 # antlr
 file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
 
-antlr4_tag="5e5b6d35b4183fd330102c40947b95c4b5c6abb5" # v4.9.2
-repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
+antlr4_tag="4.9.2" # v4.9.2
+repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
 # fix issue https://github.com/antlr/antlr4/issues/3194 - should update Antlr commit once the PR related to the issue gets merged
@@ -161,20 +178,20 @@ cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
 repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
 
 # fmt
-fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
-repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
+fmt_tag="7.1.3" # (2020-11-25)
+repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag" true
 
 # rapidcheck
 rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
 repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
 
 # google benchmark
-benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
-repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
+benchmark_tag="v1.1.0"
+repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
 
 # google test
-googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
-repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
+googletest_tag="release-1.8.0"
+repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
 
 # google flags
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
@@ -201,19 +218,19 @@ cd ..
 bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
 repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
 
-zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
-repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
+zlib_tag="v1.2.11" # v1.2.11.
+repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag" true
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
 
-rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
-repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
+rocksdb_tag="v6.14.6" # (2020-10-14)
+repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
 # remove shared library from install dependencies
 sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
 
 # mgclient
 mgclient_tag="v1.3.0" # (2021-09-23)
-repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
+repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag" true
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
 
 # pymgclient
@@ -222,10 +239,10 @@ repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient
 
 # mgconsole
 mgconsole_tag="v1.1.0" # (2021-10-07)
-repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag"
+repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
 
-spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
-repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
+spdlog_tag="v1.8.2" # (2020-12-01)
+repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
 
 jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
 repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
@@ -248,4 +265,27 @@ popd
 
 # librdkafka
 librdkafka_tag="v1.7.0" # (2021-05-06)
-repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag"
+repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag" true
+
+# protobuf
+protobuf_tag="v3.12.4"
+repo_clone_try_double "${primary_urls[protobuf]}" "${secondary_urls[protobuf]}" "protobuf" "$protobuf_tag" true
+pushd protobuf
+./autogen.sh && ./configure --prefix=$(pwd)/lib
+popd
+
+# boost
+file_get_try_double  "${primary_urls[boost]}" "${secondary_urls[boost]}"
+tar -xzf boost_1_77_0.tar.gz
+mv boost_1_77_0 boost
+pushd boost
+./bootstrap.sh --prefix=$(pwd)/lib --with-libraries="system,regex"
+./b2 -j$(nproc) install
+popd
+
+#pulsar
+pulsar_tag="v2.8.1"
+repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pulsar" "$pulsar_tag" true
+pushd pulsar
+git apply ../pulsar.patch
+popd

release/README.md

@@ -1,4 +0,0 @@
-# Memgraph Release
-
-Please continue in
-[Notion](https://www.notion.so/memgraph/Release-621733518c82456ebc119d2a025b5dd3).

release/debian/copyright

@@ -1 +1 @@
-../LICENSE.md
+../../LICENSE

release/get_version.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 import argparse
+import os
 import re
 import subprocess
 import sys
-import os
-
+import time
+from functools import wraps
 
 # This script is used to determine the current version of Memgraph. The script
 # determines the current version using `git` automatically. The user can also
@@ -93,16 +94,29 @@ import os
 #   https://fedoraproject.org/wiki/Package_Versioning_Examples
 
 
+def retry(retry_limit, timeout=100):
+    def inner_func(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            for _ in range(retry_limit):
+                try:
+                    return func(*args, **kwargs)
+                except Exception:
+                    time.sleep(timeout)
+            return func(*args, **kwargs)
+        return wrapper
+    return inner_func
+
+
+@retry(3)
 def get_output(*cmd, multiple=False):
     ret = subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
     if multiple:
-        return list(map(lambda x: x.strip(),
-                        ret.stdout.decode("utf-8").strip().split("\n")))
+        return list(map(lambda x: x.strip(), ret.stdout.decode("utf-8").strip().split("\n")))
     return ret.stdout.decode("utf-8").strip()
 
 
-def format_version(variant, version, offering, distance=None, shorthash=None,
-                   suffix=None):
+def format_version(variant, version, offering, distance=None, shorthash=None, suffix=None):
     if not distance:
         # This is a release version.
         if variant == "deb":
@@ -135,8 +149,7 @@ def format_version(variant, version, offering, distance=None, shorthash=None,
             return ret
         elif variant == "rpm":
             # <VERSION>_0.<DISTANCE>.<SHORTHASH>.<OFFERING>[.<SUFFIX>]
-            ret = "{}_0.{}.{}{}".format(
-                version, distance, shorthash, "." + offering if offering else "")
+            ret = "{}_0.{}.{}{}".format(version, distance, shorthash, "." + offering if offering else "")
             if suffix:
                 ret += "." + suffix
             return ret
@@ -149,27 +162,23 @@ def format_version(variant, version, offering, distance=None, shorthash=None,
 
 
 # Parse arguments.
-parser = argparse.ArgumentParser(
-    description="Get the current version of Memgraph.")
+parser = argparse.ArgumentParser(description="Get the current version of Memgraph.")
+parser.add_argument("--open-source", action="store_true", help="set the current offering to 'open-source'")
+parser.add_argument("version", help="manual version override, if supplied the version isn't " "determined using git")
+parser.add_argument("suffix", help="custom suffix for the current version being built")
 parser.add_argument(
-    "--open-source", action="store_true",
-    help="set the current offering to 'open-source'")
+    "--variant",
+    choices=("binary", "deb", "rpm"),
+    default="binary",
+    help="which variant of the version string should be generated",
+)
 parser.add_argument(
-    "version", help="manual version override, if supplied the version isn't "
-    "determined using git")
-parser.add_argument(
-    "suffix", help="custom suffix for the current version being built")
-parser.add_argument(
-    "--variant", choices=("binary", "deb", "rpm"), default="binary",
-    help="which variant of the version string should be generated")
-parser.add_argument(
-    "--memgraph-root-dir", help="The root directory of the checked out "
-    "Memgraph repository.", default=".")
+    "--memgraph-root-dir", help="The root directory of the checked out " "Memgraph repository.", default="."
+)
 args = parser.parse_args()
 
 if not os.path.isdir(args.memgraph_root_dir):
-    raise Exception("The root directory ({}) is not a valid directory".format(
-        args.memgraph_root_dir))
+    raise Exception("The root directory ({}) is not a valid directory".format(args.memgraph_root_dir))
 
 os.chdir(args.memgraph_root_dir)
 
@@ -179,8 +188,7 @@ offering = "open-source" if args.open_source else None
 if args.version:
     if not re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", args.version):
         raise Exception("Invalid version supplied '{}'!".format(args.version))
-    print(format_version(args.variant, args.version, offering,
-                         suffix=args.suffix), end="")
+    print(format_version(args.variant, args.version, offering, suffix=args.suffix), end="")
     sys.exit(0)
 
 # Within CI, after the regular checkout, master is sometimes (e.g. in the case
@@ -234,33 +242,28 @@ versions.sort(reverse=True)
 current_version = None
 for version in versions:
     version_tuple, branch, master_branch_merge = version
-    current_branch_merge = get_output(
-        "git", "merge-base", current_hash, branch)
-    master_current_merge = get_output(
-        "git", "merge-base", current_hash, "master")
+    current_branch_merge = get_output("git", "merge-base", current_hash, branch)
+    master_current_merge = get_output("git", "merge-base", current_hash, "master")
     # The first check checks whether this commit is a child of `master` and
     # the version branch was created before us.
     # The second check checks whether this commit is a child of the version
     # branch.
-    if master_branch_merge == current_branch_merge or \
-            master_branch_merge == master_current_merge:
+    if master_branch_merge == current_branch_merge or master_branch_merge == master_current_merge:
         current_version = version
         break
 
 # Determine current version.
 if current_version is None:
-    raise Exception("You are attempting to determine the version for a very "
-                    "old version of Memgraph!")
+    raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!")
 version, branch, master_branch_merge = current_version
-distance = int(get_output("git", "rev-list", "--count", "--first-parent",
-                          master_branch_merge + ".." + current_hash))
+distance = int(get_output("git", "rev-list", "--count", "--first-parent", master_branch_merge + ".." + current_hash))
 version_str = ".".join(map(str, version)) + ".0"
 if distance == 0:
-    print(format_version(args.variant, version_str, offering,
-                         suffix=args.suffix),
-          end="")
+    print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="")
 else:
-    print(format_version(args.variant, version_str, offering,
-                         distance=distance, shorthash=current_hash_short,
-                         suffix=args.suffix),
-          end="")
+    print(
+        format_version(
+            args.variant, version_str, offering, distance=distance, shorthash=current_hash_short, suffix=args.suffix
+        ),
+        end="",
+    )

release/package/run.sh

@@ -18,14 +18,21 @@ print_help () {
 
 make_package () {
     os="$1"
+
+    build_container="mgbuild_$os"
+    echo "Building Memgraph for $os on $build_container..."
+
     package_command=""
     if [[ "$os" =~ ^"centos".* ]]; then
+        docker exec "$build_container" bash -c "yum -y update"
         package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
     fi
     if [[ "$os" =~ ^"debian".* ]]; then
+        docker exec "$build_container" bash -c "apt update"
         package_command=" cpack -G DEB --config ../CPackConfig.cmake "
     fi
     if [[ "$os" =~ ^"ubuntu".* ]]; then
+        docker exec "$build_container" bash -c "apt update"
         package_command=" cpack -G DEB --config ../CPackConfig.cmake "
     fi
     telemetry_id_override_flag=""
@@ -39,8 +46,6 @@ make_package () {
           exit
         fi
     fi
-    build_container="mgbuild_$os"
-    echo "Building Memgraph for $os on $build_container..."
 
     echo "Copying project files..."
     # If master is not the current branch, fetch it, because the get_version

src/communication/bolt/v1/codes.hpp

@@ -31,6 +31,7 @@ enum class Signature : uint8_t {
   Begin = 0x11,
   Commit = 0x12,
   Rollback = 0x13,
+  Route = 0x66,
 
   Record = 0x71,
   Success = 0x70,

src/communication/bolt/v1/constants.hpp

@@ -28,7 +28,7 @@ static constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
  */
 static constexpr size_t kHandshakeSize = 20;
 
-static constexpr uint16_t kSupportedVersions[3] = {0x0100, 0x0400, 0x0401};
+static constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
 
 static constexpr int kPullAll = -1;
 static constexpr int kPullLast = -1;

src/communication/bolt/v1/states/executing.hpp

@@ -12,6 +12,7 @@
 #pragma once
 
 #include <map>
+#include <memory>
 #include <new>
 #include <string>
 
@@ -19,339 +20,69 @@
 #include "communication/bolt/v1/constants.hpp"
 #include "communication/bolt/v1/exceptions.hpp"
 #include "communication/bolt/v1/state.hpp"
+#include "communication/bolt/v1/states/handlers.hpp"
 #include "communication/bolt/v1/value.hpp"
-#include "communication/exceptions.hpp"
 #include "utils/likely.hpp"
 #include "utils/logging.hpp"
 #include "utils/message.hpp"
 
 namespace communication::bolt {
 
-// TODO (mferencevic): revise these error messages
-inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::exception &e) {
-  if (auto *verbose = dynamic_cast<const VerboseError *>(&e)) {
-    return {verbose->code(), verbose->what()};
-  }
-  if (dynamic_cast<const ClientError *>(&e)) {
-    // Clients expect 4 strings separated by dots. First being database name
-    // (for example: Neo, Memgraph...), second being either ClientError,
-    // TransientError or DatabaseError (or ClientNotification for warnings).
-    // ClientError means wrong query, do not retry. DatabaseError means
-    // something wrong in database, do not retry. TransientError means query
-    // failed, but if retried it may succeed, retry it.
-    //
-    // Third and fourth strings being namespace and specific error name.
-    // It is not really important what we put there since we don't expect
-    // any special handling of specific exceptions on client side, but we
-    // need to make sure that we don't accidentally return some exception
-    // name which clients handle in a special way. For example, if client
-    // receives *.TransientError.Transaction.Terminate it will not rerun
-    // query even though TransientError was returned, because of Neo's
-    // semantics of that error.
-    return {"Memgraph.ClientError.MemgraphError.MemgraphError", e.what()};
-  }
-  if (dynamic_cast<const utils::BasicException *>(&e)) {
-    // Exception not derived from QueryException was thrown which means that
-    // database probably aborted transaction because of some timeout,
-    // deadlock, serialization error or something similar. We return
-    // TransientError since retry of same transaction could succeed.
-    return {"Memgraph.TransientError.MemgraphError.MemgraphError", e.what()};
-  }
-  if (dynamic_cast<const std::bad_alloc *>(&e)) {
-    // std::bad_alloc was thrown, God knows in which state is database ->
-    // terminate.
-    LOG_FATAL("Memgraph is out of memory");
-  }
-  // All exceptions used in memgraph are derived from BasicException. Since
-  // we caught some other exception we don't know what is going on. Return
-  // DatabaseError, log real message and return generic string.
-  spdlog::error(
-      utils::MessageWithLink("Unknown exception occurred during query execution {}.", e.what(), "https://memgr.ph/unknown"));
-  return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
-          "An unknown exception occurred, this is unexpected. Real message "
-          "should be in database logs."};
-}
-
 template <typename TSession>
-inline State HandleFailure(TSession &session, const std::exception &e) {
-  spdlog::trace("Error message: {}", e.what());
-  if (const auto *p = dynamic_cast<const utils::StacktraceException *>(&e)) {
-    spdlog::trace("Error trace: {}", p->trace());
-  }
-  session.encoder_buffer_.Clear();
-  auto code_message = ExceptionToErrorMessage(e);
-  bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
-  if (!fail_sent) {
-    spdlog::trace("Couldn't send failure message!");
-    return State::Close;
-  }
-  return State::Error;
-}
-
-template <typename TSession>
-State HandleRun(TSession &session, State state, Marker marker) {
-  const std::map<std::string, Value> kEmptyFields = {{"fields", std::vector<Value>{}}};
-
-  const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct2 : Marker::TinyStruct3;
-  if (marker != expected_marker) {
-    spdlog::trace("Expected {} marker, but received 0x{:02X}!",
-                  session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  Value query, params, extra;
-  if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
-    spdlog::trace("Couldn't read query string!");
-    return State::Close;
-  }
-
-  if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
-    spdlog::trace("Couldn't read parameters!");
-    return State::Close;
-  }
-
-  if (session.version_.major == 4) {
-    if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
-      spdlog::trace("Couldn't read extra field!");
-    }
-  }
-
-  if (state != State::Idle) {
-    // Client could potentially recover if we move to error state, but there is
-    // no legitimate situation in which well working client would end up in this
-    // situation.
-    spdlog::trace("Unexpected RUN command!");
-    return State::Close;
-  }
-
-  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
-
-  spdlog::debug("[Run] '{}'", query.ValueString());
-
-  try {
-    // Interpret can throw.
-    auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
-    // Convert std::string to Value
-    std::vector<Value> vec;
-    std::map<std::string, Value> data;
-    vec.reserve(header.size());
-    for (auto &i : header) vec.emplace_back(std::move(i));
-    data.emplace("fields", std::move(vec));
-    // Send the header.
-    if (!session.encoder_.MessageSuccess(data)) {
-      spdlog::trace("Couldn't send query header!");
+State RunHandlerV1(Signature signature, TSession &session, State state, Marker marker) {
+  switch (signature) {
+    case Signature::Run:
+      return HandleRunV1<TSession>(session, state, marker);
+    case Signature::Pull:
+      return HandlePullV1<TSession>(session, state, marker);
+    case Signature::Discard:
+      return HandleDiscardV1<TSession>(session, state, marker);
+    case Signature::Reset:
+      return HandleReset<TSession>(session, marker);
+    default:
+      spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
       return State::Close;
-    }
-    return State::Result;
-  } catch (const std::exception &e) {
-    return HandleFailure(session, e);
   }
 }
 
-namespace detail {
-template <bool is_pull, typename TSession>
-State HandlePullDiscard(TSession &session, State state, Marker marker) {
-  const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct : Marker::TinyStruct1;
-  if (marker != expected_marker) {
-    spdlog::trace("Expected {} marker, but received 0x{:02X}!",
-                  session.version_.major == 1 ? "TinyStruct" : "TinyStruct1", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  if (state != State::Result) {
-    if constexpr (is_pull) {
-      spdlog::trace("Unexpected PULL!");
-    } else {
-      spdlog::trace("Unexpected DISCARD!");
-    }
-    // Same as `unexpected RUN` case.
-    return State::Close;
-  }
-
-  try {
-    std::optional<int> n;
-    std::optional<int> qid;
-
-    if (session.version_.major == 4) {
-      Value extra;
-      if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
-        spdlog::trace("Couldn't read extra field!");
-      }
-      const auto &extra_map = extra.ValueMap();
-      if (extra_map.count("n")) {
-        if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
-          n = n_value;
-        }
-      }
-
-      if (extra_map.count("qid")) {
-        if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
-          qid = qid_value;
-        }
+template <typename TSession, int bolt_minor = 0>
+State RunHandlerV4(Signature signature, TSession &session, State state, Marker marker) {
+  switch (signature) {
+    case Signature::Run:
+      return HandleRunV4<TSession>(session, state, marker);
+    case Signature::Pull:
+      return HandlePullV4<TSession>(session, state, marker);
+    case Signature::Discard:
+      return HandleDiscardV4<TSession>(session, state, marker);
+    case Signature::Reset:
+      return HandleReset<TSession>(session, marker);
+    case Signature::Begin:
+      return HandleBegin<TSession>(session, state, marker);
+    case Signature::Commit:
+      return HandleCommit<TSession>(session, state, marker);
+    case Signature::Goodbye:
+      return HandleGoodbye<TSession>();
+    case Signature::Rollback:
+      return HandleRollback<TSession>(session, state, marker);
+    case Signature::Noop: {
+      if constexpr (bolt_minor >= 1) {
+        return HandleNoop<TSession>(state);
+      } else {
+        spdlog::trace("Supported only in bolt v4.1");
+        return State::Close;
       }
     }
-
-    std::map<std::string, Value> summary;
-    if constexpr (is_pull) {
-      // Pull can throw.
-      summary = session.Pull(&session.encoder_, n, qid);
-    } else {
-      summary = session.Discard(n, qid);
+    case Signature::Route: {
+      if constexpr (bolt_minor >= 3) {
+        if (signature == Signature::Route) return HandleRoute<TSession>(session);
+      } else {
+        spdlog::trace("Supported only in bolt v4.3");
+        return State::Close;
+      }
     }
-
-    if (!session.encoder_.MessageSuccess(summary)) {
-      spdlog::trace("Couldn't send query summary!");
+    default:
+      spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
       return State::Close;
-    }
-
-    if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
-      return State::Result;
-    }
-
-    return State::Idle;
-  } catch (const std::exception &e) {
-    return HandleFailure(session, e);
-  }
-}
-}  // namespace detail
-
-template <typename Session>
-State HandlePull(Session &session, State state, Marker marker) {
-  return detail::HandlePullDiscard<true>(session, state, marker);
-}
-
-template <typename Session>
-State HandleDiscard(Session &session, State state, Marker marker) {
-  return detail::HandlePullDiscard<false>(session, state, marker);
-}
-
-template <typename Session>
-State HandleReset(Session &session, State, Marker marker) {
-  // IMPORTANT: This implementation of the Bolt RESET command isn't fully
-  // compliant to the protocol definition. In the protocol it is defined
-  // that this command should immediately stop any running commands and
-  // reset the session to a clean state. That means that we should always
-  // make a look-ahead for the RESET command before processing anything.
-  // Our implementation, for now, does everything in a blocking fashion
-  // so we cannot simply "kill" a transaction while it is running. So
-  // now this command only resets the session to a clean state. It
-  // does not IGNORE running and pending commands as it should.
-  if (marker != Marker::TinyStruct) {
-    spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  // Clear all pending data and send a success message.
-  session.encoder_buffer_.Clear();
-
-  if (!session.encoder_.MessageSuccess()) {
-    spdlog::trace("Couldn't send success message!");
-    return State::Close;
-  }
-
-  session.Abort();
-
-  return State::Idle;
-}
-
-template <typename Session>
-State HandleBegin(Session &session, State state, Marker marker) {
-  if (session.version_.major == 1) {
-    spdlog::trace("BEGIN messsage not supported in Bolt v1!");
-    return State::Close;
-  }
-
-  if (marker != Marker::TinyStruct1) {
-    spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  Value extra;
-  if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
-    spdlog::trace("Couldn't read extra fields!");
-    return State::Close;
-  }
-
-  if (state != State::Idle) {
-    spdlog::trace("Unexpected BEGIN command!");
-    return State::Close;
-  }
-
-  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
-
-  if (!session.encoder_.MessageSuccess({})) {
-    spdlog::trace("Couldn't send success message!");
-    return State::Close;
-  }
-
-  try {
-    session.BeginTransaction();
-  } catch (const std::exception &e) {
-    return HandleFailure(session, e);
-  }
-
-  return State::Idle;
-}
-
-template <typename Session>
-State HandleCommit(Session &session, State state, Marker marker) {
-  if (session.version_.major == 1) {
-    spdlog::trace("COMMIT messsage not supported in Bolt v1!");
-    return State::Close;
-  }
-
-  if (marker != Marker::TinyStruct) {
-    spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  if (state != State::Idle) {
-    spdlog::trace("Unexpected COMMIT command!");
-    return State::Close;
-  }
-
-  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
-
-  try {
-    if (!session.encoder_.MessageSuccess({})) {
-      spdlog::trace("Couldn't send success message!");
-      return State::Close;
-    }
-    session.CommitTransaction();
-    return State::Idle;
-  } catch (const std::exception &e) {
-    return HandleFailure(session, e);
-  }
-}
-
-template <typename Session>
-State HandleRollback(Session &session, State state, Marker marker) {
-  if (session.version_.major == 1) {
-    spdlog::trace("ROLLBACK messsage not supported in Bolt v1!");
-    return State::Close;
-  }
-
-  if (marker != Marker::TinyStruct) {
-    spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
-    return State::Close;
-  }
-
-  if (state != State::Idle) {
-    spdlog::trace("Unexpected ROLLBACK command!");
-    return State::Close;
-  }
-
-  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
-
-  try {
-    if (!session.encoder_.MessageSuccess({})) {
-      spdlog::trace("Couldn't send success message!");
-      return State::Close;
-    }
-    session.RollbackTransaction();
-    return State::Idle;
-  } catch (const std::exception &e) {
-    return HandleFailure(session, e);
   }
 }
 
@@ -361,8 +92,8 @@ State HandleRollback(Session &session, State state, Marker marker) {
  * It executes: RUN, PULL_ALL, DISCARD_ALL & RESET.
  * @param session the session that should be used for the run
  */
-template <typename Session>
-State StateExecutingRun(Session &session, State state) {
+template <typename TSession>
+State StateExecutingRun(TSession &session, State state) {
   Marker marker;
   Signature signature;
   if (!session.decoder_.ReadMessageHeader(&signature, &marker)) {
@@ -370,30 +101,21 @@ State StateExecutingRun(Session &session, State state) {
     return State::Close;
   }
 
-  if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
-    spdlog::trace("Received NOOP message");
-    return state;
-  }
-
-  if (signature == Signature::Run) {
-    return HandleRun(session, state, marker);
-  } else if (signature == Signature::Pull) {
-    return HandlePull(session, state, marker);
-  } else if (signature == Signature::Discard) {
-    return HandleDiscard(session, state, marker);
-  } else if (signature == Signature::Begin) {
-    return HandleBegin(session, state, marker);
-  } else if (signature == Signature::Commit) {
-    return HandleCommit(session, state, marker);
-  } else if (signature == Signature::Rollback) {
-    return HandleRollback(session, state, marker);
-  } else if (signature == Signature::Reset) {
-    return HandleReset(session, state, marker);
-  } else if (signature == Signature::Goodbye && session.version_.major != 1) {
-    throw SessionClosedException("Closing connection.");
-  } else {
-    spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
-    return State::Close;
+  switch (session.version_.major) {
+    case 1:
+      return RunHandlerV1(signature, session, state, marker);
+    case 4: {
+      if (session.version_.minor >= 3) {
+        return RunHandlerV4<TSession, 3>(signature, session, state, marker);
+      }
+      if (session.version_.minor >= 1) {
+        return RunHandlerV4<TSession, 1>(signature, session, state, marker);
+      }
+      return RunHandlerV4<TSession>(signature, session, state, marker);
+    }
+    default:
+      spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
+      return State::Close;
   }
 }
 }  // namespace communication::bolt

src/communication/bolt/v1/states/handlers.hpp

@@ -0,0 +1,418 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "communication/bolt/v1/codes.hpp"
+#include "communication/bolt/v1/constants.hpp"
+#include "communication/bolt/v1/state.hpp"
+#include "communication/bolt/v1/value.hpp"
+#include "communication/exceptions.hpp"
+#include "utils/logging.hpp"
+#include "utils/message.hpp"
+
+namespace communication::bolt {
+// TODO: Revise these error messages
+inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::exception &e) {
+  if (const auto *verbose = dynamic_cast<const VerboseError *>(&e)) {
+    return {verbose->code(), verbose->what()};
+  }
+  if (dynamic_cast<const ClientError *>(&e)) {
+    // Clients expect 4 strings separated by dots. First being database name
+    // (for example: Neo, Memgraph...), second being either ClientError,
+    // TransientError or DatabaseError (or ClientNotification for warnings).
+    // ClientError means wrong query, do not retry. DatabaseError means
+    // something wrong in database, do not retry. TransientError means query
+    // failed, but if retried it may succeed, retry it.
+    //
+    // Third and fourth strings being namespace and specific error name.
+    // It is not really important what we put there since we don't expect
+    // any special handling of specific exceptions on client side, but we
+    // need to make sure that we don't accidentally return some exception
+    // name which clients handle in a special way. For example, if client
+    // receives *.TransientError.Transaction.Terminate it will not rerun
+    // query even though TransientError was returned, because of Neo's
+    // semantics of that error.
+    return {"Memgraph.ClientError.MemgraphError.MemgraphError", e.what()};
+  }
+  if (dynamic_cast<const utils::BasicException *>(&e)) {
+    // Exception not derived from QueryException was thrown which means that
+    // database probably aborted transaction because of some timeout,
+    // deadlock, serialization error or something similar. We return
+    // TransientError since retry of same transaction could succeed.
+    return {"Memgraph.TransientError.MemgraphError.MemgraphError", e.what()};
+  }
+  if (dynamic_cast<const std::bad_alloc *>(&e)) {
+    // std::bad_alloc was thrown, God knows in which state is database ->
+    // terminate.
+    LOG_FATAL("Memgraph is out of memory");
+  }
+  // All exceptions used in memgraph are derived from BasicException. Since
+  // we caught some other exception we don't know what is going on. Return
+  // DatabaseError, log real message and return generic string.
+  spdlog::error(utils::MessageWithLink("Unknown exception occurred during query execution {}.", e.what(),
+                                       "https://memgr.ph/unknown"));
+  return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
+          "An unknown exception occurred, this is unexpected. Real message "
+          "should be in database logs."};
+}
+
+namespace details {
+
+template <typename TSession>
+State HandleRun(TSession &session, const State state, const Value &query, const Value &params) {
+  if (state != State::Idle) {
+    // Client could potentially recover if we move to error state, but there is
+    // no legitimate situation in which well working client would end up in this
+    // situation.
+    spdlog::trace("Unexpected RUN command!");
+    return State::Close;
+  }
+
+  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
+
+  spdlog::debug("[Run] '{}'", query.ValueString());
+
+  try {
+    // Interpret can throw.
+    const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
+    // Convert std::string to Value
+    std::vector<Value> vec;
+    std::map<std::string, Value> data;
+    vec.reserve(header.size());
+    for (auto &i : header) vec.emplace_back(std::move(i));
+    data.emplace("fields", std::move(vec));
+    // Send the header.
+    if (!session.encoder_.MessageSuccess(data)) {
+      spdlog::trace("Couldn't send query header!");
+      return State::Close;
+    }
+    return State::Result;
+  } catch (const std::exception &e) {
+    return HandleFailure(session, e);
+  }
+}
+
+template <bool is_pull, typename TSession>
+State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<int> qid) {
+  try {
+    std::map<std::string, Value> summary;
+    if constexpr (is_pull) {
+      // Pull can throw.
+      summary = session.Pull(&session.encoder_, n, qid);
+    } else {
+      summary = session.Discard(n, qid);
+    }
+
+    if (!session.encoder_.MessageSuccess(summary)) {
+      spdlog::trace("Couldn't send query summary!");
+      return State::Close;
+    }
+
+    if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
+      return State::Result;
+    }
+
+    return State::Idle;
+  } catch (const std::exception &e) {
+    return HandleFailure(session, e);
+  }
+}
+
+template <bool is_pull, typename TSession>
+State HandlePullDiscardV1(TSession &session, const State state, const Marker marker) {
+  const auto expected_marker = Marker::TinyStruct;
+  if (marker != expected_marker) {
+    spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  if (state != State::Result) {
+    if constexpr (is_pull) {
+      spdlog::trace("Unexpected PULL!");
+    } else {
+      spdlog::trace("Unexpected DISCARD!");
+    }
+    // Same as `unexpected RUN` case.
+    return State::Close;
+  }
+
+  return HandlePullDiscard<is_pull, TSession>(session, std::nullopt, std::nullopt);
+}
+
+template <bool is_pull, typename TSession>
+State HandlePullDiscardV4(TSession &session, const State state, const Marker marker) {
+  const auto expected_marker = Marker::TinyStruct1;
+  if (marker != expected_marker) {
+    spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct1", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  if (state != State::Result) {
+    if constexpr (is_pull) {
+      spdlog::trace("Unexpected PULL!");
+    } else {
+      spdlog::trace("Unexpected DISCARD!");
+    }
+    // Same as `unexpected RUN` case.
+    return State::Close;
+  }
+  std::optional<int> n;
+  std::optional<int> qid;
+  Value extra;
+  if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
+    spdlog::trace("Couldn't read extra field!");
+  }
+  const auto &extra_map = extra.ValueMap();
+  if (extra_map.count("n")) {
+    if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
+      n = n_value;
+    }
+  }
+
+  if (extra_map.count("qid")) {
+    if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
+      qid = qid_value;
+    }
+  }
+  return HandlePullDiscard<is_pull, TSession>(session, n, qid);
+}
+}  // namespace details
+
+template <typename TSession>
+inline State HandleFailure(TSession &session, const std::exception &e) {
+  spdlog::trace("Error message: {}", e.what());
+  if (const auto *p = dynamic_cast<const utils::StacktraceException *>(&e)) {
+    spdlog::trace("Error trace: {}", p->trace());
+  }
+  session.encoder_buffer_.Clear();
+  auto code_message = ExceptionToErrorMessage(e);
+  bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
+  if (!fail_sent) {
+    spdlog::trace("Couldn't send failure message!");
+    return State::Close;
+  }
+  return State::Error;
+}
+
+template <typename TSession>
+State HandleRunV1(TSession &session, const State state, const Marker marker) {
+  const auto expected_marker = Marker::TinyStruct2;
+  if (marker != expected_marker) {
+    spdlog::trace("Expected {} marker, but received 0x{:02X}!",
+                  session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+  Value query, params;
+  if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
+    spdlog::trace("Couldn't read query string!");
+    return State::Close;
+  }
+
+  if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
+    spdlog::trace("Couldn't read parameters!");
+    return State::Close;
+  }
+
+  return details::HandleRun(session, state, query, params);
+}
+
+template <typename TSession>
+State HandleRunV4(TSession &session, const State state, const Marker marker) {
+  const auto expected_marker = Marker::TinyStruct3;
+  if (marker != expected_marker) {
+    spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct3", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+  Value query, params, extra;
+  if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
+    spdlog::trace("Couldn't read query string!");
+    return State::Close;
+  }
+
+  if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
+    spdlog::trace("Couldn't read parameters!");
+    return State::Close;
+  }
+
+  // Even though this part seems unnecessary it is needed to move the buffer
+  if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
+    spdlog::trace("Couldn't read extra field!");
+  }
+
+  return details::HandleRun(session, state, query, params);
+}
+
+template <typename TSession>
+State HandlePullV1(TSession &session, const State state, const Marker marker) {
+  return details::HandlePullDiscardV1<true>(session, state, marker);
+}
+
+template <typename TSession>
+State HandlePullV4(TSession &session, const State state, const Marker marker) {
+  return details::HandlePullDiscardV4<true>(session, state, marker);
+}
+
+template <typename TSession>
+State HandleDiscardV1(TSession &session, const State state, const Marker marker) {
+  return details::HandlePullDiscardV1<false>(session, state, marker);
+}
+
+template <typename TSession>
+State HandleDiscardV4(TSession &session, const State state, const Marker marker) {
+  return details::HandlePullDiscardV4<false>(session, state, marker);
+}
+
+template <typename TSession>
+State HandleReset(TSession &session, const Marker marker) {
+  // IMPORTANT: This implementation of the Bolt RESET command isn't fully
+  // compliant to the protocol definition. In the protocol it is defined
+  // that this command should immediately stop any running commands and
+  // reset the session to a clean state. That means that we should always
+  // make a look-ahead for the RESET command before processing anything.
+  // Our implementation, for now, does everything in a blocking fashion
+  // so we cannot simply "kill" a transaction while it is running. So
+  // now this command only resets the session to a clean state. It
+  // does not IGNORE running and pending commands as it should.
+  if (marker != Marker::TinyStruct) {
+    spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  // Clear all pending data and send a success message.
+  session.encoder_buffer_.Clear();
+
+  if (!session.encoder_.MessageSuccess()) {
+    spdlog::trace("Couldn't send success message!");
+    return State::Close;
+  }
+
+  session.Abort();
+
+  return State::Idle;
+}
+
+template <typename TSession>
+State HandleBegin(TSession &session, const State state, const Marker marker) {
+  if (marker != Marker::TinyStruct1) {
+    spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  Value extra;
+  if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
+    spdlog::trace("Couldn't read extra fields!");
+    return State::Close;
+  }
+
+  if (state != State::Idle) {
+    spdlog::trace("Unexpected BEGIN command!");
+    return State::Close;
+  }
+
+  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
+
+  if (!session.encoder_.MessageSuccess({})) {
+    spdlog::trace("Couldn't send success message!");
+    return State::Close;
+  }
+
+  try {
+    session.BeginTransaction();
+  } catch (const std::exception &e) {
+    return HandleFailure(session, e);
+  }
+
+  return State::Idle;
+}
+
+template <typename TSession>
+State HandleCommit(TSession &session, const State state, const Marker marker) {
+  if (marker != Marker::TinyStruct) {
+    spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  if (state != State::Idle) {
+    spdlog::trace("Unexpected COMMIT command!");
+    return State::Close;
+  }
+
+  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
+
+  try {
+    if (!session.encoder_.MessageSuccess({})) {
+      spdlog::trace("Couldn't send success message!");
+      return State::Close;
+    }
+    session.CommitTransaction();
+    return State::Idle;
+  } catch (const std::exception &e) {
+    return HandleFailure(session, e);
+  }
+}
+
+template <typename TSession>
+State HandleRollback(TSession &session, const State state, const Marker marker) {
+  if (marker != Marker::TinyStruct) {
+    spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
+    return State::Close;
+  }
+
+  if (state != State::Idle) {
+    spdlog::trace("Unexpected ROLLBACK command!");
+    return State::Close;
+  }
+
+  DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
+
+  try {
+    if (!session.encoder_.MessageSuccess({})) {
+      spdlog::trace("Couldn't send success message!");
+      return State::Close;
+    }
+    session.RollbackTransaction();
+    return State::Idle;
+  } catch (const std::exception &e) {
+    return HandleFailure(session, e);
+  }
+}
+
+template <typename TSession>
+State HandleNoop(const State state) {
+  spdlog::trace("Received NOOP message");
+  return state;
+}
+
+template <typename TSession>
+State HandleGoodbye() {
+  throw SessionClosedException("Closing connection.");
+}
+
+template <typename TSession>
+State HandleRoute(TSession &session) {
+  // Route message is not implemented since it is neo4j specific, therefore we
+  // will receive it an inform user that there is no implementation.
+  session.encoder_buffer_.Clear();
+  bool fail_sent =
+      session.encoder_.MessageFailure({{"code", 66}, {"message", "Route message not supported in Memgraph!"}});
+  if (!fail_sent) {
+    spdlog::trace("Couldn't send failure message!");
+    return State::Close;
+  }
+  return State::Error;
+}
+}  // namespace communication::bolt

src/communication/bolt/v1/states/handshake.hpp

@@ -12,6 +12,9 @@
 #pragma once
 
 #include <fmt/format.h>
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
 
 #include "communication/bolt/v1/codes.hpp"
 #include "communication/bolt/v1/constants.hpp"
@@ -21,6 +24,35 @@
 
 namespace communication::bolt {
 
+inline bool CopyProtocolInformationIfSupported(uint16_t version, uint8_t *protocol) {
+  const auto *supported_version = std::find(std::begin(kSupportedVersions), std::end(kSupportedVersions), version);
+  if (supported_version != std::end(kSupportedVersions)) {
+    std::memcpy(protocol, &version, sizeof(version));
+    return true;
+  }
+  return false;
+}
+
+inline bool CopyProtocolInformationIfSupportedWithOffset(auto data_position, uint8_t *protocol) {
+  struct bolt_range_version {
+    uint8_t offset;
+    uint8_t minor;
+    uint8_t major;
+  } bolt_range_version;
+  std::memcpy(&bolt_range_version, data_position, sizeof(bolt_range_version));
+  if (bolt_range_version.major == 0 || bolt_range_version.minor == 0) return false;
+  bolt_range_version.offset = std::min(bolt_range_version.offset, bolt_range_version.minor);
+
+  for (uint8_t i{0U}; i <= bolt_range_version.offset; i++) {
+    uint8_t current_minor = bolt_range_version.minor - i;
+    if (CopyProtocolInformationIfSupported(static_cast<uint16_t>((bolt_range_version.major << 8U) + current_minor),
+                                           protocol)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Handshake state run function
  * This function runs everything to make a Bolt handshake with the client.
@@ -29,7 +61,7 @@ namespace communication::bolt {
 template <typename TSession>
 State StateHandshakeRun(TSession &session) {
   auto precmp = std::memcmp(session.input_stream_.data(), kPreamble, sizeof(kPreamble));
-  if (UNLIKELY(precmp != 0)) {
+  if (precmp != 0) [[unlikely]] {
     spdlog::trace("Received a wrong preamble!");
     return State::Close;
   }
@@ -37,22 +69,22 @@ State StateHandshakeRun(TSession &session) {
   DMG_ASSERT(session.input_stream_.size() >= kHandshakeSize, "Wrong size of the handshake data!");
 
   auto dataPosition = session.input_stream_.data() + sizeof(kPreamble);
-
   uint8_t protocol[4] = {0x00};
-  for (int i = 0; i < 4 && !protocol[3]; ++i) {
-    dataPosition += 2;  // version is defined only by the last 2 bytes
 
-    uint16_t version = 0;
+  for (int i = 0; i < 4 && !protocol[3]; ++i) {
+    // If there is an offset defined (e.g. 0x00 0x03 0x03 0x04) the second byte
+    // That would enable the client to pick between 4.0 and 4.3 versions
+    // as per changes in handshake bolt protocol in v4.3
+    if (CopyProtocolInformationIfSupportedWithOffset(dataPosition + 1, protocol + 2)) break;
+
+    dataPosition += 2;  // version is defined only by the last 2 bytes
+    uint16_t version{0};
     std::memcpy(&version, dataPosition, sizeof(version));
     if (!version) {
       break;
     }
-
-    for (const auto supportedVersion : kSupportedVersions) {
-      if (supportedVersion == version) {
-        std::memcpy(protocol + 2, &version, sizeof(version));
-        break;
-      }
+    if (CopyProtocolInformationIfSupported(version, protocol + 2)) {
+      break;
     }
 
     dataPosition += 2;

src/communication/bolt/v1/states/init.hpp

@@ -12,6 +12,7 @@
 #pragma once
 
 #include <fmt/format.h>
+#include <optional>
 
 #include "communication/bolt/v1/codes.hpp"
 #include "communication/bolt/v1/state.hpp"
@@ -22,10 +23,45 @@
 
 namespace communication::bolt {
 
-namespace detail {
+namespace details {
 template <typename TSession>
-std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
-  if (UNLIKELY(marker != Marker::TinyStruct2)) {
+std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
+  // Get authentication data.
+  auto &data = metadata.ValueMap();
+  if (!data.count("scheme")) {
+    spdlog::warn("The client didn't supply authentication information!");
+    return State::Close;
+  }
+  std::string username;
+  std::string password;
+  if (data["scheme"].ValueString() == "basic") {
+    if (!data.count("principal") || !data.count("credentials")) {
+      spdlog::warn("The client didn't supply authentication information!");
+      return State::Close;
+    }
+    username = data["principal"].ValueString();
+    password = data["credentials"].ValueString();
+  } else if (data["scheme"].ValueString() != "none") {
+    spdlog::warn("Unsupported authentication scheme: {}", data["scheme"].ValueString());
+    return State::Close;
+  }
+
+  // Authenticate the user.
+  if (!session.Authenticate(username, password)) {
+    if (!session.encoder_.MessageFailure(
+            {{"code", "Memgraph.ClientError.Security.Unauthenticated"}, {"message", "Authentication failure"}})) {
+      spdlog::trace("Couldn't send failure message to the client!");
+    }
+    // Throw an exception to indicate to the network stack that the session
+    // should be closed and cleaned up.
+    throw SessionClosedException("The client is not authenticated!");
+  }
+  return std::nullopt;
+}
+
+template <typename TSession>
+std::optional<Value> GetMetadataV1(TSession &session, const Marker marker) {
+  if (marker != Marker::TinyStruct2) [[unlikely]] {
     spdlog::trace("Expected TinyStruct2 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
     spdlog::trace(
         "The client sent malformed data, but we are continuing "
@@ -53,8 +89,8 @@ std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
 }
 
 template <typename TSession>
-std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
-  if (UNLIKELY(marker != Marker::TinyStruct1)) {
+std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
+  if (marker != Marker::TinyStruct1) [[unlikely]] {
     spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
     spdlog::trace(
         "The client sent malformed data, but we are continuing "
@@ -80,15 +116,79 @@ std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
 
   return metadata;
 }
-}  // namespace detail
+
+template <typename TSession>
+State SendSuccessMessage(TSession &session) {
+  // Neo4j's Java driver 4.1.1+ requires connection_id.
+  // The only usage in the mentioned version is for logging purposes.
+  // Because it's not critical for the regular usage of the driver
+  // we send a hardcoded value for now.
+  std::map<std::string, Value> metadata{{"connection_id", "bolt-1"}};
+  if (auto server_name = session.GetServerNameForInit(); server_name) {
+    metadata.insert({"server", *server_name});
+  }
+  bool success_sent = session.encoder_.MessageSuccess(metadata);
+  if (!success_sent) {
+    spdlog::trace("Couldn't send success message to the client!");
+    return State::Close;
+  }
+
+  return State::Idle;
+}
+
+template <typename TSession>
+State StateInitRunV1(TSession &session, const Marker marker, const Signature signature) {
+  if (signature != Signature::Init) [[unlikely]] {
+    spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
+    return State::Close;
+  }
+
+  auto maybeMetadata = GetMetadataV1(session, marker);
+
+  if (!maybeMetadata) {
+    return State::Close;
+  }
+  if (auto result = AuthenticateUser(session, *maybeMetadata)) {
+    return result.value();
+  }
+
+  return SendSuccessMessage(session);
+}
+
+template <typename TSession, int bolt_minor = 0>
+State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
+  if constexpr (bolt_minor > 0) {
+    if (signature == Signature::Noop) [[unlikely]] {
+      SPDLOG_DEBUG("Received NOOP message");
+      return State::Init;
+    }
+  }
+
+  if (signature != Signature::Init) [[unlikely]] {
+    spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
+    return State::Close;
+  }
+
+  auto maybeMetadata = GetMetadataV4(session, marker);
+
+  if (!maybeMetadata) {
+    return State::Close;
+  }
+  if (auto result = AuthenticateUser(session, *maybeMetadata)) {
+    return result.value();
+  }
+
+  return SendSuccessMessage(session);
+}
+}  // namespace details
 
 /**
  * Init state run function.
  * This function runs everything to initialize a Bolt session with the client.
  * @param session the session that should be used for the run.
  */
-template <typename Session>
-State StateInitRun(Session &session) {
+template <typename TSession>
+State StateInitRun(TSession &session) {
   DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
 
   Marker marker;
@@ -98,72 +198,18 @@ State StateInitRun(Session &session) {
     return State::Close;
   }
 
-  if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
-    SPDLOG_DEBUG("Received NOOP message");
-    return State::Init;
-  }
-
-  if (UNLIKELY(signature != Signature::Init)) {
-    spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
-    return State::Close;
-  }
-
-  auto maybeMetadata =
-      session.version_.major == 1 ? detail::StateInitRunV1(session, marker) : detail::StateInitRunV4(session, marker);
-
-  if (!maybeMetadata) {
-    return State::Close;
-  }
-
-  // Get authentication data.
-  std::string username;
-  std::string password;
-  auto &data = maybeMetadata->ValueMap();
-  if (!data.count("scheme")) {
-    spdlog::warn("The client didn't supply authentication information!");
-    return State::Close;
-  }
-  if (data["scheme"].ValueString() == "basic") {
-    if (!data.count("principal") || !data.count("credentials")) {
-      spdlog::warn("The client didn't supply authentication information!");
-      return State::Close;
+  switch (session.version_.major) {
+    case 1: {
+      return details::StateInitRunV1<TSession>(session, marker, signature);
     }
-    username = data["principal"].ValueString();
-    password = data["credentials"].ValueString();
-  } else if (data["scheme"].ValueString() != "none") {
-    spdlog::warn("Unsupported authentication scheme: {}", data["scheme"].ValueString());
-    return State::Close;
-  }
-
-  // Authenticate the user.
-  if (!session.Authenticate(username, password)) {
-    if (!session.encoder_.MessageFailure(
-            {{"code", "Memgraph.ClientError.Security.Unauthenticated"}, {"message", "Authentication failure"}})) {
-      spdlog::trace("Couldn't send failure message to the client!");
-    }
-    // Throw an exception to indicate to the network stack that the session
-    // should be closed and cleaned up.
-    throw SessionClosedException("The client is not authenticated!");
-  }
-
-  // Return success.
-  {
-    bool success_sent = false;
-    // Neo4j's Java driver 4.1.1+ requires connection_id.
-    // The only usage in the mentioned version is for logging purposes.
-    // Because it's not critical for the regular usage of the driver
-    // we send a hardcoded value for now.
-    std::map<std::string, Value> metadata{{"connection_id", "bolt-1"}};
-    if (auto server_name = session.GetServerNameForInit(); server_name) {
-      metadata.insert({"server", *server_name});
-    }
-    success_sent = session.encoder_.MessageSuccess(metadata);
-    if (!success_sent) {
-      spdlog::trace("Couldn't send success message to the client!");
-      return State::Close;
+    case 4: {
+      if (session.version_.minor > 0) {
+        return details::StateInitRunV4<TSession, 1>(session, marker, signature);
+      }
+      return details::StateInitRunV4<TSession>(session, marker, signature);
     }
   }
-
-  return State::Idle;
+  spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
+  return State::Close;
 }
 }  // namespace communication::bolt

src/integrations/CMakeLists.txt

@@ -1 +1,2 @@
 add_subdirectory(kafka)
+add_subdirectory(pulsar)

src/integrations/constants.hpp

@@ -0,0 +1,22 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+#include <chrono>
+
+namespace integrations {
+constexpr std::chrono::milliseconds kDefaultBatchInterval{100};
+constexpr int64_t kDefaultBatchSize = 1000;
+constexpr int64_t kDefaultCheckBatchLimit = 1;
+constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
+constexpr std::chrono::milliseconds kMinimumInterval{1};
+constexpr int64_t kMinimumSize{1};
+}  // namespace integrations

src/integrations/kafka/consumer.cpp

@@ -19,6 +19,8 @@
 
 #include <librdkafka/rdkafkacpp.h>
 #include <spdlog/spdlog.h>
+
+#include "integrations/constants.hpp"
 #include "integrations/kafka/exceptions.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
@@ -27,13 +29,6 @@
 
 namespace integrations::kafka {
 
-constexpr std::chrono::milliseconds kDefaultBatchInterval{100};
-constexpr int64_t kDefaultBatchSize = 1000;
-constexpr int64_t kDefaultCheckBatchLimit = 1;
-constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
-constexpr std::chrono::milliseconds kMinimumInterval{1};
-constexpr int64_t kMinimumSize{1};
-
 namespace {
 utils::BasicResult<std::string, std::vector<Message>> GetBatch(RdKafka::KafkaConsumer &consumer,
                                                                const ConsumerInfo &info,
@@ -109,14 +104,14 @@ int64_t Message::Timestamp() const {
   return rd_kafka_message_timestamp(c_message, nullptr);
 }
 
-Consumer::Consumer(const std::string &bootstrap_servers, ConsumerInfo info, ConsumerFunction consumer_function)
+Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
     : info_{std::move(info)}, consumer_function_(std::move(consumer_function)) {
   MG_ASSERT(consumer_function_, "Empty consumer function for Kafka consumer");
   // NOLINTNEXTLINE (modernize-use-nullptr)
-  if (info.batch_interval.value_or(kMinimumInterval) < kMinimumInterval) {
+  if (info_.batch_interval.value_or(kMinimumInterval) < kMinimumInterval) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, "Batch interval has to be positive!");
   }
-  if (info.batch_size.value_or(kMinimumSize) < kMinimumSize) {
+  if (info_.batch_size.value_or(kMinimumSize) < kMinimumSize) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, "Batch size has to be positive!");
   }
 
@@ -139,7 +134,7 @@ Consumer::Consumer(const std::string &bootstrap_servers, ConsumerInfo info, Cons
     throw ConsumerFailedToInitializeException(info_.consumer_name, error);
   }
 
-  if (conf->set("bootstrap.servers", bootstrap_servers, error) != RdKafka::Conf::CONF_OK) {
+  if (conf->set("bootstrap.servers", info_.bootstrap_servers, error) != RdKafka::Conf::CONF_OK) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, error);
   }
 

src/integrations/kafka/consumer.hpp

@@ -79,6 +79,7 @@ struct ConsumerInfo {
   std::string consumer_name;
   std::vector<std::string> topics;
   std::string consumer_group;
+  std::string bootstrap_servers;
   std::optional<std::chrono::milliseconds> batch_interval;
   std::optional<int64_t> batch_size;
 };
@@ -93,7 +94,7 @@ class Consumer final : public RdKafka::EventCb {
   ///
   /// @throws ConsumerFailedToInitializeException if the consumer can't connect
   ///         to the Kafka endpoint.
-  Consumer(const std::string &bootstrap_servers, ConsumerInfo info, ConsumerFunction consumer_function);
+  Consumer(ConsumerInfo info, ConsumerFunction consumer_function);
   ~Consumer() override;
 
   Consumer(const Consumer &other) = delete;

src/integrations/pulsar/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(integrations_pulsar_src_files
+    consumer.cpp
+)
+
+find_package(CURL REQUIRED)
+
+add_library(mg-integrations-pulsar STATIC ${integrations_pulsar_src_files})
+target_link_libraries(mg-integrations-pulsar mg-utils pulsar Threads::Threads ${CURL_LIBRARIES})

src/integrations/pulsar/consumer.cpp

@@ -0,0 +1,291 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "integrations/pulsar/consumer.hpp"
+
+#include <fmt/format.h>
+#include <pulsar/Client.h>
+#include <pulsar/InitialPosition.h>
+
+#include <chrono>
+#include <thread>
+
+#include "integrations/constants.hpp"
+#include "integrations/pulsar/exceptions.hpp"
+#include "utils/concepts.hpp"
+#include "utils/logging.hpp"
+#include "utils/on_scope_exit.hpp"
+#include "utils/result.hpp"
+#include "utils/thread.hpp"
+
+namespace integrations::pulsar {
+
+namespace {
+
+template <typename T>
+concept PulsarConsumer = utils::SameAsAnyOf<T, pulsar_client::Consumer, pulsar_client::Reader>;
+
+pulsar_client::Result ConsumeMessage(pulsar_client::Consumer &consumer, pulsar_client::Message &message,
+                                     int remaining_timeout_in_ms) {
+  return consumer.receive(message, remaining_timeout_in_ms);
+}
+
+pulsar_client::Result ConsumeMessage(pulsar_client::Reader &reader, pulsar_client::Message &message,
+                                     int remaining_timeout_in_ms) {
+  return reader.readNext(message, remaining_timeout_in_ms);
+}
+
+template <PulsarConsumer TConsumer>
+utils::BasicResult<std::string, std::vector<Message>> GetBatch(TConsumer &consumer, const ConsumerInfo &info,
+                                                               std::atomic<bool> &is_running) {
+  std::vector<Message> batch{};
+
+  const auto batch_size = info.batch_size.value_or(kDefaultBatchSize);
+  batch.reserve(batch_size);
+
+  auto remaining_timeout_in_ms = info.batch_interval.value_or(kDefaultBatchInterval).count();
+  auto start = std::chrono::steady_clock::now();
+
+  for (int64_t i = 0; remaining_timeout_in_ms > 0 && i < batch_size && is_running.load(); ++i) {
+    pulsar_client::Message message;
+    const auto result = ConsumeMessage(consumer, message, remaining_timeout_in_ms);
+    switch (result) {
+      case pulsar_client::Result::ResultTimeout:
+        return std::move(batch);
+      case pulsar_client::Result::ResultOk:
+        batch.emplace_back(Message{std::move(message)});
+        break;
+      default:
+        spdlog::warn(fmt::format("Unexpected error while consuming message from consumer {}, error: {}",
+                                 info.consumer_name, result));
+        return {pulsar_client::strResult(result)};
+    }
+
+    auto now = std::chrono::steady_clock::now();
+    auto took = std::chrono::duration_cast<std::chrono::milliseconds>(now - start);
+    remaining_timeout_in_ms = remaining_timeout_in_ms - took.count();
+    start = now;
+  }
+
+  return std::move(batch);
+}
+
+class SpdlogLogger : public pulsar_client::Logger {
+  bool isEnabled(Level /*level*/) override { return spdlog::should_log(spdlog::level::trace); }
+
+  void log(Level /*level*/, int /*line*/, const std::string &message) override {
+    spdlog::trace("[Pulsar] {}", message);
+  }
+};
+
+class SpdlogLoggerFactory : public pulsar_client::LoggerFactory {
+  pulsar_client::Logger *getLogger(const std::string & /*file_name*/) override {
+    if (!logger_) {
+      logger_ = std::make_unique<SpdlogLogger>();
+    }
+    return logger_.get();
+  }
+
+ private:
+  std::unique_ptr<SpdlogLogger> logger_;
+};
+
+pulsar_client::Client CreateClient(const std::string &service_url) {
+  pulsar_client::ClientConfiguration conf;
+  conf.setLogger(new SpdlogLoggerFactory);
+  return {service_url, conf};
+}
+}  // namespace
+
+Message::Message(pulsar_client::Message &&message) : message_{std::move(message)} {}
+
+std::span<const char> Message::Payload() const {
+  return {static_cast<const char *>(message_.getData()), message_.getLength()};
+}
+
+Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
+    : info_{std::move(info)},
+      client_{CreateClient(info_.service_url)},
+      consumer_function_{std::move(consumer_function)} {
+  pulsar_client::ConsumerConfiguration config;
+  config.setSubscriptionInitialPosition(pulsar_client::InitialPositionLatest);
+  config.setConsumerType(pulsar_client::ConsumerType::ConsumerExclusive);
+  if (pulsar_client::Result result = client_.subscribe(info_.topics, info_.consumer_name, config, consumer_);
+      result != pulsar_client::ResultOk) {
+    throw ConsumerFailedToInitializeException(info_.consumer_name, pulsar_client::strResult(result));
+  }
+}
+Consumer::~Consumer() {
+  StopIfRunning();
+  consumer_.close();
+  client_.close();
+}
+
+bool Consumer::IsRunning() const { return is_running_; }
+
+const ConsumerInfo &Consumer::Info() const { return info_; }
+
+void Consumer::Start() {
+  if (is_running_) {
+    throw ConsumerRunningException(info_.consumer_name);
+  }
+
+  StartConsuming();
+}
+
+void Consumer::Stop() {
+  if (!is_running_) {
+    throw ConsumerStoppedException(info_.consumer_name);
+  }
+  StopConsuming();
+}
+
+void Consumer::StopIfRunning() {
+  if (is_running_) {
+    StopConsuming();
+  }
+
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
+                     const ConsumerFunction &check_consumer_function) const {
+  // NOLINTNEXTLINE (modernize-use-nullptr)
+  if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Timeout has to be positive!");
+  }
+  if (limit_batches.value_or(kMinimumSize) < kMinimumSize) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Batch limit has to be positive!");
+  }
+  // The implementation of this function is questionable: it is const qualified, though it changes the inner state of
+  // PulsarConsumer. Though it changes the inner state, it saves the current assignment for future Check/Start calls to
+  // restore the current state, so the changes made by this function shouldn't be visible for the users of the class. It
+  // also passes a non const reference of PulsarConsumer to GetBatch function. That means the object is bitwise const
+  // (PulsarConsumer is stored in unique_ptr) and internally mostly synchronized. Mostly, because as Start/Stop requires
+  // exclusive access to consumer, so we don't have to deal with simultaneous calls to those functions. The only concern
+  // in this function is to prevent executing this function on multiple threads simultaneously.
+  if (is_running_.exchange(true)) {
+    throw ConsumerRunningException(info_.consumer_name);
+  }
+
+  utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
+
+  const auto num_of_batches = limit_batches.value_or(kDefaultCheckBatchLimit);
+  const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
+  const auto start = std::chrono::steady_clock::now();
+
+  if (info_.topics.size() != 1) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Check cannot be used for multiple topics");
+  }
+
+  std::vector<std::string> partitions;
+  const auto &topic = info_.topics.front();
+  client_.getPartitionsForTopic(topic, partitions);
+  if (partitions.size() > 1) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Check cannot be used for topics with multiple partitions");
+  }
+  pulsar_client::Reader reader;
+  client_.createReader(topic, last_message_id_, {}, reader);
+  for (int64_t i = 0; i < num_of_batches;) {
+    const auto now = std::chrono::steady_clock::now();
+    // NOLINTNEXTLINE (modernize-use-nullptr)
+    if (now - start >= timeout_to_use) {
+      throw ConsumerCheckFailedException(info_.consumer_name, "Timeout reached");
+    }
+
+    auto maybe_batch = GetBatch(reader, info_, is_running_);
+
+    if (maybe_batch.HasError()) {
+      throw ConsumerCheckFailedException(info_.consumer_name, maybe_batch.GetError());
+    }
+
+    const auto &batch = maybe_batch.GetValue();
+
+    if (batch.empty()) {
+      continue;
+    }
+    ++i;
+
+    try {
+      check_consumer_function(batch);
+    } catch (const std::exception &e) {
+      spdlog::warn("Pulsar consumer {} check failed with error {}", info_.consumer_name, e.what());
+      throw ConsumerCheckFailedException(info_.consumer_name, e.what());
+    }
+  }
+  reader.close();
+}
+
+void Consumer::StartConsuming() {
+  MG_ASSERT(!is_running_, "Cannot start already running consumer!");
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+
+  is_running_.store(true);
+
+  thread_ = std::thread([this] {
+    constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
+    const auto full_thread_name = "Cons#" + info_.consumer_name;
+
+    utils::ThreadSetName(full_thread_name.substr(0, kMaxThreadNameSize));
+
+    while (is_running_) {
+      auto maybe_batch = GetBatch(consumer_, info_, is_running_);
+
+      if (maybe_batch.HasError()) {
+        spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
+                     maybe_batch.GetError());
+        break;
+      }
+
+      const auto &batch = maybe_batch.GetValue();
+
+      if (batch.empty()) {
+        continue;
+      }
+
+      spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
+
+      try {
+        consumer_function_(batch);
+
+        if (std::any_of(batch.begin(), batch.end(), [&](const auto &message) {
+              if (const auto result = consumer_.acknowledge(message.message_); result != pulsar_client::ResultOk) {
+                spdlog::warn("Acknowledging a message of consumer {} failed: {}", info_.consumer_name, result);
+                return true;
+              }
+              last_message_id_ = message.message_.getMessageId();
+              return false;
+            })) {
+          break;
+        }
+      } catch (const std::exception &e) {
+        spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
+        break;
+      }
+
+      spdlog::info("Pulsar consumer {} finished processing", info_.consumer_name);
+    }
+    is_running_.store(false);
+  });
+}
+
+void Consumer::StopConsuming() {
+  is_running_.store(false);
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+}  // namespace integrations::pulsar

src/integrations/pulsar/consumer.hpp

@@ -0,0 +1,81 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+#include <atomic>
+#include <optional>
+#include <span>
+#include <thread>
+
+#include <pulsar/Client.h>
+
+namespace integrations::pulsar {
+
+namespace pulsar_client = ::pulsar;
+
+class Consumer;
+
+class Message final {
+ public:
+  explicit Message(pulsar_client::Message &&message);
+
+  std::span<const char> Payload() const;
+
+ private:
+  pulsar_client::Message message_;
+
+  friend Consumer;
+};
+
+using ConsumerFunction = std::function<void(const std::vector<Message> &)>;
+
+struct ConsumerInfo {
+  std::optional<int64_t> batch_size;
+  std::optional<std::chrono::milliseconds> batch_interval;
+  std::vector<std::string> topics;
+  std::string consumer_name;
+  std::string service_url;
+};
+
+class Consumer final {
+ public:
+  Consumer(ConsumerInfo info, ConsumerFunction consumer_function);
+  ~Consumer();
+
+  Consumer(const Consumer &) = delete;
+  Consumer(Consumer &&) noexcept = delete;
+  Consumer &operator=(const Consumer &) = delete;
+  Consumer &operator=(Consumer &&) = delete;
+
+  bool IsRunning() const;
+  void Start();
+  void Stop();
+  void StopIfRunning();
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
+             const ConsumerFunction &check_consumer_function) const;
+
+  const ConsumerInfo &Info() const;
+
+ private:
+  void StartConsuming();
+  void StopConsuming();
+
+  ConsumerInfo info_;
+  mutable pulsar_client::Client client_;
+  pulsar_client::Consumer consumer_;
+  ConsumerFunction consumer_function_;
+
+  mutable std::atomic<bool> is_running_{false};
+  pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};
+  std::thread thread_;
+};
+}  // namespace integrations::pulsar

src/integrations/pulsar/exceptions.hpp

@@ -0,0 +1,58 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <string>
+
+#include "utils/exceptions.hpp"
+
+namespace integrations::pulsar {
+class PulsarStreamException : public utils::BasicException {
+  using utils::BasicException::BasicException;
+};
+
+class ConsumerFailedToInitializeException : public PulsarStreamException {
+ public:
+  ConsumerFailedToInitializeException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Failed to initialize Pulsar consumer {} : {}", consumer_name, error) {}
+};
+
+class ConsumerRunningException : public PulsarStreamException {
+ public:
+  explicit ConsumerRunningException(const std::string &consumer_name)
+      : PulsarStreamException("Pulsar consumer {} is already running", consumer_name) {}
+};
+
+class ConsumerStoppedException : public PulsarStreamException {
+ public:
+  explicit ConsumerStoppedException(const std::string &consumer_name)
+      : PulsarStreamException("Pulsar consumer {} is already stopped", consumer_name) {}
+};
+
+class ConsumerCheckFailedException : public PulsarStreamException {
+ public:
+  explicit ConsumerCheckFailedException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Pulsar consumer {} check failed: {}", consumer_name, error) {}
+};
+
+class ConsumerStartFailedException : public PulsarStreamException {
+ public:
+  explicit ConsumerStartFailedException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Starting Pulsar consumer {} failed: {}", consumer_name, error) {}
+};
+
+class TopicNotFoundException : public PulsarStreamException {
+ public:
+  TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
+      : PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
+};
+}  // namespace integrations::pulsar

src/lisp/CMakeLists.txt

@@ -2,19 +2,19 @@
 
 # Don't forget to repeat this list below in `define_add_lcp`.
 set(lcp_src_files
-    ${CMAKE_SOURCE_DIR}/src/lisp/lcp.asd
-    ${CMAKE_SOURCE_DIR}/src/lisp/compile-lcp
-    ${CMAKE_SOURCE_DIR}/src/lisp/package.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/names.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/types.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/clone.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/code-gen.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/slk.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/debug.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/test.lisp
-    ${CMAKE_SOURCE_DIR}/src/lisp/util.lisp
-    ${CMAKE_SOURCE_DIR}/tools/lcp)
+    ${PROJECT_SOURCE_DIR}/src/lisp/lcp.asd
+    ${PROJECT_SOURCE_DIR}/src/lisp/compile-lcp
+    ${PROJECT_SOURCE_DIR}/src/lisp/package.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/names.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/types.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/clone.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/code-gen.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/slk.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/lcp.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/debug.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/test.lisp
+    ${PROJECT_SOURCE_DIR}/src/lisp/util.lisp
+    ${PROJECT_SOURCE_DIR}/tools/lcp)
 
 # Make `lcp_src_files` a persistent (cache) variable so that
 # tests/unit/CMakeLists.txt can see it.
@@ -65,21 +65,21 @@ macro(define_add_lcp name main_src_files generated_lcp_files)
     # Repeat the `lcp_src_files` because this is a macro and the variable is
     # not visible when invoked in another file.
     set(lcp_src_files
-        ${CMAKE_SOURCE_DIR}/src/lisp/lcp.asd
-        ${CMAKE_SOURCE_DIR}/src/lisp/compile-lcp
-        ${CMAKE_SOURCE_DIR}/src/lisp/package.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/names.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/types.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/clone.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/code-gen.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/slk.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/lcp.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/debug.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/test.lisp
-        ${CMAKE_SOURCE_DIR}/src/lisp/util.lisp
-        ${CMAKE_SOURCE_DIR}/tools/lcp)
+        ${PROJECT_SOURCE_DIR}/src/lisp/lcp.asd
+        ${PROJECT_SOURCE_DIR}/src/lisp/compile-lcp
+        ${PROJECT_SOURCE_DIR}/src/lisp/package.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/names.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/types.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/clone.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/code-gen.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/slk.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/lcp.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/debug.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/test.lisp
+        ${PROJECT_SOURCE_DIR}/src/lisp/util.lisp
+        ${PROJECT_SOURCE_DIR}/tools/lcp)
     add_custom_command(OUTPUT ${h_file} ${cpp_file}
-      COMMAND ${CMAKE_SOURCE_DIR}/tools/lcp ${lcp_file} ${slk_serialize}
+      COMMAND ${PROJECT_SOURCE_DIR}/tools/lcp ${lcp_file} ${slk_serialize}
       VERBATIM
       DEPENDS ${lcp_src_files} lcp ${lcp_file}
       WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})

src/memgraph.cpp

@@ -188,7 +188,7 @@ DEFINE_bool(telemetry_enabled, false,
 
 // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
 DEFINE_string(kafka_bootstrap_servers, "",
-              "List of Kafka brokers as a comma separated list of broker host or host:port.");
+              "List of default Kafka brokers as a comma separated list of broker host or host:port.");
 
 // Audit logging flags.
 #ifdef MG_ENTERPRISE
@@ -999,8 +999,8 @@ int main(int argc, char **argv) {
       auto gil = py::EnsureGIL();
       auto maybe_exc = py::AppendToSysPath(py_support_dir.c_str());
       if (maybe_exc) {
-        spdlog::error(
-            utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc, "https://memgr.ph/python"));
+        spdlog::error(utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc,
+                                             "https://memgr.ph/python"));
       }
     } else {
       spdlog::error(utils::MessageWithLink("Unable to load support for embedded Python: missing directory {}.",
@@ -1119,11 +1119,11 @@ int main(int argc, char **argv) {
   }
   storage::Storage db(db_config);
 
-  query::InterpreterContext interpreter_context{
-      &db,
-      {.query = {.allow_load_csv = FLAGS_allow_load_csv}, .execution_timeout_sec = FLAGS_query_execution_timeout_sec},
-      FLAGS_data_directory,
-      FLAGS_kafka_bootstrap_servers};
+  query::InterpreterContext interpreter_context{&db,
+                                                {.query = {.allow_load_csv = FLAGS_allow_load_csv},
+                                                 .execution_timeout_sec = FLAGS_query_execution_timeout_sec,
+                                                 .default_kafka_bootstrap_servers = FLAGS_kafka_bootstrap_servers},
+                                                FLAGS_data_directory};
 #ifdef MG_ENTERPRISE
   SessionData session_data{&db, &interpreter_context, &auth, &audit_log};
 #else
@@ -1133,9 +1133,6 @@ int main(int argc, char **argv) {
   query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories);
   query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories();
 
-  // As the Stream transformations are using modules, they have to be restored after the query modules are loaded.
-  interpreter_context.streams.RestoreStreams();
-
   AuthQueryHandler auth_handler(&auth, FLAGS_auth_user_or_role_name_regex);
   AuthChecker auth_checker{&auth};
   interpreter_context.auth = &auth_handler;
@@ -1151,6 +1148,9 @@ int main(int argc, char **argv) {
                                                       interpreter_context.auth_checker);
   }
 
+  // As the Stream transformations are using modules, they have to be restored after the query modules are loaded.
+  interpreter_context.streams.RestoreStreams();
+
   ServerContext context;
   std::string service_name = "Bolt";
   if (!FLAGS_bolt_key_file.empty() && !FLAGS_bolt_cert_file.empty()) {

src/memory/new_delete.cpp

@@ -19,13 +19,12 @@
 #include <cstdlib>
 #endif
 
-#include "utils/likely.hpp"
 #include "utils/memory_tracker.hpp"
 
 namespace {
 void *newImpl(const std::size_t size) {
   auto *ptr = malloc(size);
-  if (LIKELY(ptr != nullptr)) {
+  if (ptr != nullptr) [[likely]] {
     return ptr;
   }
 
@@ -34,7 +33,7 @@ void *newImpl(const std::size_t size) {
 
 void *newImpl(const std::size_t size, const std::align_val_t align) {
   auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
-  if (LIKELY(ptr != nullptr)) {
+  if (ptr != nullptr) [[likely]] {
     return ptr;
   }
 
@@ -47,14 +46,22 @@ void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept
 }
 
 #if USE_JEMALLOC
-void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
+void deleteImpl(void *ptr) noexcept {
+  if (ptr == nullptr) [[unlikely]] {
+    return;
+  }
+  dallocx(ptr, 0);
+}
 
 void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
+  if (ptr == nullptr) [[unlikely]] {
+    return;
+  }
   dallocx(ptr, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
 }
 
 void deleteSized(void *ptr, const std::size_t size) noexcept {
-  if (UNLIKELY(ptr == nullptr)) {
+  if (ptr == nullptr) [[unlikely]] {
     return;
   }
 
@@ -62,7 +69,7 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
 }
 
 void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
-  if (UNLIKELY(ptr == nullptr)) {
+  if (ptr == nullptr) [[unlikely]] {
     return;
   }
 
@@ -81,7 +88,7 @@ void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t
 
 void TrackMemory(std::size_t size) {
 #if USE_JEMALLOC
-  if (LIKELY(size != 0)) {
+  if (size != 0) [[likely]] {
     size = nallocx(size, 0);
   }
 #endif
@@ -90,7 +97,7 @@ void TrackMemory(std::size_t size) {
 
 void TrackMemory(std::size_t size, const std::align_val_t align) {
 #if USE_JEMALLOC
-  if (LIKELY(size != 0)) {
+  if (size != 0) [[likely]] {
     size = nallocx(size, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
   }
 #endif
@@ -120,7 +127,7 @@ bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
 void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
   try {
 #if USE_JEMALLOC
-    if (LIKELY(ptr != nullptr)) {
+    if (ptr != nullptr) [[likely]] {
       utils::total_memory_tracker.Free(sallocx(ptr, 0));
     }
 #else
@@ -138,7 +145,7 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size
 void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
   try {
 #if USE_JEMALLOC
-    if (LIKELY(ptr != nullptr)) {
+    if (ptr != nullptr) [[likely]] {
       utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align)));  // NOLINT(hicpp-signed-bitwise)
     }
 #else
@@ -176,28 +183,28 @@ void *operator new[](const std::size_t size, const std::align_val_t align) {
 }
 
 void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size))) {
+  if (TrackMemoryNoExcept(size)) [[likely]] {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
 void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size))) {
+  if (TrackMemoryNoExcept(size)) [[likely]] {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
 void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+  if (TrackMemoryNoExcept(size, align)) [[likely]] {
     return newNoExcept(size, align);
   }
   return nullptr;
 }
 
 void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+  if (TrackMemoryNoExcept(size, align)) [[likely]] {
     return newNoExcept(size, align);
   }
   return nullptr;

src/query/CMakeLists.txt

@@ -32,7 +32,9 @@ set(mg_query_sources
     procedure/module.cpp
     procedure/py_module.cpp
     serialization/property_value.cpp
-    streams.cpp
+    stream/streams.cpp
+    stream/sources.cpp
+    stream/common.cpp
     trigger.cpp
     trigger_context.cpp
     typed_value.cpp)
@@ -41,7 +43,7 @@ add_library(mg-query STATIC ${mg_query_sources})
 add_dependencies(mg-query generate_lcp_query)
 target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(mg-query dl cppitertools)
-target_link_libraries(mg-query mg-integrations-kafka mg-storage-v2 mg-utils mg-kvstore mg-memory)
+target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-utils mg-kvstore mg-memory)
 if("${MG_PYTHON_VERSION}" STREQUAL "")
     find_package(Python3 3.5 REQUIRED COMPONENTS Development)
 else()

src/query/config.hpp

@@ -10,6 +10,7 @@
 // licenses/APL.txt.
 
 #pragma once
+#include <string>
 
 namespace query {
 struct InterpreterConfig {
@@ -19,5 +20,7 @@ struct InterpreterConfig {
 
   // The default execution timeout is 10 minutes.
   double execution_timeout_sec{600.0};
+
+  std::string default_kafka_bootstrap_servers;
 };
 }  // namespace query

src/query/frontend/ast/ast.lcp

@@ -2498,22 +2498,48 @@ cpp<#
   (:serialize (:slk))
   (:clone))
 
+(defun clone-variant-topic-names (source destination)
+  #>cpp
+    if (auto *topic_expression = std::get_if<Expression*>(&${source})) {
+      if (*topic_expression == nullptr) {
+        ${destination} = nullptr;
+      } else {
+        ${destination} = (*topic_expression)->Clone(storage);
+      }
+    } else {
+      ${destination} = std::get<std::vector<std::string>>(${source});
+    }
+  cpp<#)
+
 (lcp:define-class stream-query (query)
   ((action "Action" :scope :public)
+   (type "Type" :scope :public)
    (stream_name "std::string" :scope :public)
-   (topic_names "std::vector<std::string>" :scope :public)
+
+   (batch_limit "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+   (timeout "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+
    (transform_name "std::string" :scope :public)
-   (consumer_group "std::string" :scope :public)
    (batch_interval "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression"))
    (batch_size "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression"))
-   (batch_limit "Expression *" :initval "nullptr" :scope :public
+
+   (topic_names "std::variant<Expression*, std::vector<std::string>>" :initval "nullptr" 
+             :clone #'clone-variant-topic-names
+             :scope :public)
+   (consumer_group "std::string" :scope :public)
+   (bootstrap_servers "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression"))
-   (timeout "Expression *" :initval "nullptr" :scope :public
+
+   (service_url "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression")))
 
@@ -2521,6 +2547,9 @@ cpp<#
     (lcp:define-enum action
         (create-stream drop-stream start-stream stop-stream start-all-streams stop-all-streams show-streams check-stream)
       (:serialize))
+    (lcp:define-enum type
+        (kafka pulsar)
+      (:serialize))
     #>cpp
     StreamQuery() = default;
 

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -18,6 +18,7 @@
 // of the same name, EOF.
 // This hides the definition of the macro which causes
 // the compilation to fail.
+#include "query/frontend/ast/ast.hpp"
 #include "query/frontend/ast/ast_visitor.hpp"
 #include "query/procedure/module.hpp"
 //////////////////////////////////////////////////////
@@ -39,6 +40,7 @@
 #include "query/exceptions.hpp"
 #include "query/frontend/parsing.hpp"
 #include "query/interpret/awesome_memgraph_functions.hpp"
+#include "query/stream/common.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
 #include "utils/string.hpp"
@@ -492,39 +494,272 @@ antlrcpp::Any CypherMainVisitor::visitStreamQuery(MemgraphCypher::StreamQueryCon
 }
 
 antlrcpp::Any CypherMainVisitor::visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) {
+  MG_ASSERT(ctx->children.size() == 1, "CreateStreamQuery should have exactly one child!");
+  auto *stream_query = ctx->children[0]->accept(this).as<StreamQuery *>();
+  query_ = stream_query;
+  return stream_query;
+}
+
+std::vector<std::string> TopicNamesFromSymbols(
+    antlr4::tree::ParseTreeVisitor &visitor,
+    const std::vector<MemgraphCypher::SymbolicNameWithDotsAndMinusContext *> &topic_name_symbols) {
+  MG_ASSERT(!topic_name_symbols.empty());
+  std::vector<std::string> topic_names;
+  topic_names.reserve(topic_names.size());
+  std::transform(topic_name_symbols.begin(), topic_name_symbols.end(), std::back_inserter(topic_names),
+                 [&visitor](auto *topic_name) { return JoinSymbolicNamesWithDotsAndMinus(visitor, *topic_name); });
+  return topic_names;
+}
+
+template <bool required, typename... ValueTypes>
+void MapConfig(auto &memory, const auto &enum_key, auto &destination) {
+  const auto key = static_cast<uint8_t>(enum_key);
+  if (!memory.contains(key)) {
+    if constexpr (required) {
+      throw SemanticException("Config {} is required.", ToString(enum_key));
+    } else {
+      return;
+    }
+  }
+
+  std::visit(
+      [&]<typename T>(T &&value) {
+        using ValueType = std::decay_t<T>;
+        if constexpr (utils::SameAsAnyOf<ValueType, ValueTypes...>) {
+          destination = std::forward<T>(value);
+        } else {
+          LOG_FATAL("Invalid type mapped");
+        }
+      },
+      std::move(memory[key]));
+}
+
+enum class CommonStreamConfigKey : uint8_t { TRANSFORM, BATCH_INTERVAL, BATCH_SIZE, END };
+
+constexpr std::array all_common_stream_config_keys{
+    CommonStreamConfigKey::TRANSFORM, CommonStreamConfigKey::BATCH_INTERVAL, CommonStreamConfigKey::BATCH_SIZE};
+
+std::string_view ToString(const CommonStreamConfigKey key) {
+  switch (key) {
+    case CommonStreamConfigKey::TRANSFORM:
+      return "TRANSFORM";
+    case CommonStreamConfigKey::BATCH_INTERVAL:
+      return "BATCH_INTERVAL";
+    case CommonStreamConfigKey::BATCH_SIZE:
+      return "BATCH_SIZE";
+    case CommonStreamConfigKey::END:
+      LOG_FATAL("Invalid config key used");
+  }
+}
+
+#define CONCAT_HELPER(a, b) a##b
+#define CONCAT(a, b) CONCAT_HELPER(a, b)
+
+#define GENERATE_STREAM_CONFIG_KEY_ENUM(stream, first_config, ...)   \
+  enum class CONCAT(stream, ConfigKey) : uint8_t {                   \
+    first_config = static_cast<uint8_t>(CommonStreamConfigKey::END), \
+    __VA_ARGS__                                                      \
+  };
+
+GENERATE_STREAM_CONFIG_KEY_ENUM(Kafka, TOPICS, CONSUMER_GROUP, BOOTSTRAP_SERVERS);
+
+constexpr std::array all_kafka_config_keys{KafkaConfigKey::TOPICS, KafkaConfigKey::CONSUMER_GROUP,
+                                           KafkaConfigKey::BOOTSTRAP_SERVERS};
+
+std::string_view ToString(const KafkaConfigKey key) {
+  switch (key) {
+    case KafkaConfigKey::TOPICS:
+      return "TOPICS";
+    case KafkaConfigKey::CONSUMER_GROUP:
+      return "CONSUMER_GROUP";
+    case KafkaConfigKey::BOOTSTRAP_SERVERS:
+      return "BOOTSTRAP_SERVERS";
+  }
+}
+
+void MapCommonStreamConfigs(auto &memory, StreamQuery &stream_query) {
+  for (const auto key : all_common_stream_config_keys) {
+    switch (key) {
+      case CommonStreamConfigKey::TRANSFORM:
+        MapConfig<true, std::string>(memory, CommonStreamConfigKey::TRANSFORM, stream_query.transform_name_);
+        break;
+      case CommonStreamConfigKey::BATCH_INTERVAL:
+        MapConfig<false, Expression *>(memory, CommonStreamConfigKey::BATCH_INTERVAL, stream_query.batch_interval_);
+        break;
+      case CommonStreamConfigKey::BATCH_SIZE:
+        MapConfig<false, Expression *>(memory, CommonStreamConfigKey::BATCH_SIZE, stream_query.batch_size_);
+        break;
+      case CommonStreamConfigKey::END:
+        LOG_FATAL("Invalid config key used");
+    }
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) {
   auto *stream_query = storage_->Create<StreamQuery>();
   stream_query->action_ = StreamQuery::Action::CREATE_STREAM;
+  stream_query->type_ = StreamQuery::Type::KAFKA;
   stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as<std::string>();
 
-  auto *topic_names_ctx = ctx->topicNames();
-  MG_ASSERT(topic_names_ctx != nullptr);
-  auto topic_names = topic_names_ctx->symbolicNameWithDotsAndMinus();
-  MG_ASSERT(!topic_names.empty());
-  stream_query->topic_names_.reserve(topic_names.size());
-  std::transform(topic_names.begin(), topic_names.end(), std::back_inserter(stream_query->topic_names_),
-                 [this](auto *topic_name) { return JoinSymbolicNamesWithDotsAndMinus(*this, *topic_name); });
+  for (auto *create_config_ctx : ctx->kafkaCreateStreamConfig()) {
+    create_config_ctx->accept(this);
+  }
 
-  stream_query->transform_name_ = JoinSymbolicNames(this, ctx->transformationName->symbolicName());
+  for (const auto key : all_kafka_config_keys) {
+    switch (key) {
+      case KafkaConfigKey::TOPICS:
+        MapConfig<true, std::vector<std::string>>(memory_, KafkaConfigKey::TOPICS, stream_query->topic_names_);
+        break;
+      case KafkaConfigKey::CONSUMER_GROUP:
+        MapConfig<false, std::string>(memory_, KafkaConfigKey::CONSUMER_GROUP, stream_query->consumer_group_);
+        break;
+      case KafkaConfigKey::BOOTSTRAP_SERVERS:
+        MapConfig<false, Expression *>(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS, stream_query->bootstrap_servers_);
+        break;
+    }
+  }
+
+  MapCommonStreamConfigs(memory_, *stream_query);
+
+  return stream_query;
+}
+
+void ThrowIfExists(auto &map, const auto &enum_key) {
+  const auto key = static_cast<uint8_t>(enum_key);
+  if (map.contains(key)) {
+    throw SemanticException("{} defined multiple times in the query", ToString(enum_key));
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) {
+  if (ctx->commonCreateStreamConfig()) {
+    return ctx->commonCreateStreamConfig()->accept(this);
+  }
+
+  if (ctx->TOPICS()) {
+    ThrowIfExists(memory_, KafkaConfigKey::TOPICS);
+    auto *topic_names_ctx = ctx->topicNames();
+    MG_ASSERT(topic_names_ctx != nullptr);
+    const auto topic_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
+    memory_[topic_key] = TopicNamesFromSymbols(*this, topic_names_ctx->symbolicNameWithDotsAndMinus());
+    return {};
+  }
 
   if (ctx->CONSUMER_GROUP()) {
-    stream_query->consumer_group_ = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
+    ThrowIfExists(memory_, KafkaConfigKey::CONSUMER_GROUP);
+    const auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
+    memory_[consumer_group_key] = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
+    return {};
+  }
+
+  MG_ASSERT(ctx->BOOTSTRAP_SERVERS());
+  ThrowIfExists(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS);
+  if (!ctx->bootstrapServers->StringLiteral()) {
+    throw SemanticException("Bootstrap servers should be a string!");
+  }
+
+  const auto bootstrap_servers_key = static_cast<uint8_t>(KafkaConfigKey::BOOTSTRAP_SERVERS);
+  memory_[bootstrap_servers_key] = ctx->bootstrapServers->accept(this).as<Expression *>();
+  return {};
+}  // namespace query::frontend
+
+GENERATE_STREAM_CONFIG_KEY_ENUM(Pulsar, TOPICS, SERVICE_URL);
+
+constexpr std::array all_pulsar_config_keys{PulsarConfigKey::TOPICS, PulsarConfigKey::SERVICE_URL};
+
+std::string_view ToString(const PulsarConfigKey key) {
+  switch (key) {
+    case PulsarConfigKey::TOPICS:
+      return "TOPICS";
+    case PulsarConfigKey::SERVICE_URL:
+      return "SERVICE_URL";
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) {
+  auto *stream_query = storage_->Create<StreamQuery>();
+  stream_query->action_ = StreamQuery::Action::CREATE_STREAM;
+  stream_query->type_ = StreamQuery::Type::PULSAR;
+  stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as<std::string>();
+
+  for (auto *create_config_ctx : ctx->pulsarCreateStreamConfig()) {
+    create_config_ctx->accept(this);
+  }
+
+  for (const auto key : all_pulsar_config_keys) {
+    switch (key) {
+      case PulsarConfigKey::TOPICS:
+        MapConfig<true, std::vector<std::string>, Expression *>(memory_, PulsarConfigKey::TOPICS,
+                                                                stream_query->topic_names_);
+        break;
+      case PulsarConfigKey::SERVICE_URL:
+        MapConfig<true, Expression *>(memory_, PulsarConfigKey::SERVICE_URL, stream_query->service_url_);
+        break;
+    }
+  }
+
+  MapCommonStreamConfigs(memory_, *stream_query);
+
+  return stream_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) {
+  if (ctx->commonCreateStreamConfig()) {
+    return ctx->commonCreateStreamConfig()->accept(this);
+  }
+
+  if (ctx->TOPICS()) {
+    ThrowIfExists(memory_, PulsarConfigKey::TOPICS);
+    auto *pulsar_topic_names_ctx = ctx->pulsarTopicNames();
+    MG_ASSERT(pulsar_topic_names_ctx != nullptr);
+    const auto topics_key = static_cast<uint8_t>(PulsarConfigKey::TOPICS);
+    if (auto *topic_names_ctx = pulsar_topic_names_ctx->topicNames()) {
+      memory_[topics_key] = TopicNamesFromSymbols(*this, topic_names_ctx->symbolicNameWithDotsAndMinus());
+    } else {
+      if (!pulsar_topic_names_ctx->literal()->StringLiteral()) {
+        throw SemanticException("Topic names should be defined in a string");
+      }
+      memory_[topics_key] = pulsar_topic_names_ctx->accept(this).as<Expression *>();
+    }
+    return {};
+  }
+
+  MG_ASSERT(ctx->SERVICE_URL());
+  ThrowIfExists(memory_, PulsarConfigKey::SERVICE_URL);
+  if (!ctx->serviceUrl->StringLiteral()) {
+    throw SemanticException("Service url should be a string!");
+  }
+  const auto service_url_key = static_cast<uint8_t>(PulsarConfigKey::SERVICE_URL);
+  memory_[service_url_key] = ctx->serviceUrl->accept(this).as<Expression *>();
+  return {};
+}
+
+antlrcpp::Any CypherMainVisitor::visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) {
+  if (ctx->TRANSFORM()) {
+    ThrowIfExists(memory_, CommonStreamConfigKey::TRANSFORM);
+    const auto transform_key = static_cast<uint8_t>(CommonStreamConfigKey::TRANSFORM);
+    memory_[transform_key] = JoinSymbolicNames(this, ctx->transformationName->symbolicName());
+    return {};
   }
 
   if (ctx->BATCH_INTERVAL()) {
+    ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_INTERVAL);
     if (!ctx->batchInterval->numberLiteral() || !ctx->batchInterval->numberLiteral()->integerLiteral()) {
       throw SemanticException("Batch interval should be an integer literal!");
     }
-    stream_query->batch_interval_ = ctx->batchInterval->accept(this);
+    const auto batch_interval_key = static_cast<uint8_t>(CommonStreamConfigKey::BATCH_INTERVAL);
+    memory_[batch_interval_key] = ctx->batchInterval->accept(this).as<Expression *>();
+    return {};
   }
 
-  if (ctx->BATCH_SIZE()) {
-    if (!ctx->batchSize->numberLiteral() || !ctx->batchSize->numberLiteral()->integerLiteral()) {
-      throw SemanticException("Batch size should be an integer literal!");
-    }
-    stream_query->batch_size_ = ctx->batchSize->accept(this);
+  MG_ASSERT(ctx->BATCH_SIZE());
+  ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_SIZE);
+  if (!ctx->batchSize->numberLiteral() || !ctx->batchSize->numberLiteral()->integerLiteral()) {
+    throw SemanticException("Batch size should be an integer literal!");
   }
-
-  return stream_query;
+  const auto batch_size_key = static_cast<uint8_t>(CommonStreamConfigKey::BATCH_SIZE);
+  memory_[batch_size_key] = ctx->batchSize->accept(this).as<Expression *>();
+  return {};
 }
 
 antlrcpp::Any CypherMainVisitor::visitDropStream(MemgraphCypher::DropStreamContext *ctx) {

src/query/frontend/ast/cypher_main_visitor.hpp

@@ -269,6 +269,31 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
    */
   antlrcpp::Any visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) override;
 
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) override;
+
   /**
    * @return StreamQuery*
    */
@@ -824,6 +849,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
   ParsingContext context_;
   AstStorage *storage_;
 
+  std::unordered_map<uint8_t, std::variant<Expression *, std::string, std::vector<std::string>>> memory_;
   // Set of identifiers from queries.
   std::unordered_set<std::string> users_identifiers;
   // Identifiers that user didn't name.

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -29,6 +29,7 @@ memgraphCypherKeyword : cypherKeyword
                       | BATCH_LIMIT
                       | BATCH_SIZE
                       | BEFORE
+                      | BOOTSTRAP_SERVERS
                       | CHECK
                       | CLEAR
                       | COMMIT
@@ -296,12 +297,29 @@ symbolicNameWithDotsAndMinus: symbolicNameWithMinus ( DOT symbolicNameWithMinus
 
 topicNames : symbolicNameWithDotsAndMinus ( COMMA symbolicNameWithDotsAndMinus )* ;
 
-createStream : CREATE STREAM streamName
-               TOPICS topicNames
-               TRANSFORM transformationName=procedureName
-               ( CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus ) ?
-               ( BATCH_INTERVAL batchInterval=literal ) ?
-               ( BATCH_SIZE batchSize=literal ) ? ;
+commonCreateStreamConfig : TRANSFORM transformationName=procedureName
+                         | BATCH_INTERVAL batchInterval=literal
+                         | BATCH_SIZE batchSize=literal
+                         ;
+
+createStream : kafkaCreateStream | pulsarCreateStream ;
+
+kafkaCreateStreamConfig : TOPICS topicNames
+                        | CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus
+                        | BOOTSTRAP_SERVERS bootstrapServers=literal
+                        | commonCreateStreamConfig
+                        ;
+
+kafkaCreateStream : CREATE KAFKA STREAM streamName ( kafkaCreateStreamConfig ) * ;
+
+pulsarTopicNames : topicNames | literal ;
+
+pulsarCreateStreamConfig : TOPICS pulsarTopicNames
+                         | SERVICE_URL serviceUrl=literal
+                         | commonCreateStreamConfig
+                         ;
+
+pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig ) * ;
 
 dropStream : DROP STREAM streamName ;
 

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -25,81 +25,85 @@ import CypherLexer ;
 
 UNDERSCORE : '_' ;
 
-AFTER          : A F T E R ;
-ALTER          : A L T E R ;
-ASYNC          : A S Y N C ;
-AUTH           : A U T H ;
-BAD            : B A D ;
-BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
-BATCH_LIMIT    : B A T C H UNDERSCORE L I M I T ;
-BATCH_SIZE     : B A T C H UNDERSCORE S I Z E ;
-BEFORE         : B E F O R E ;
-CHECK          : C H E C K ;
-CLEAR          : C L E A R ;
-COMMIT         : C O M M I T ;
-COMMITTED      : C O M M I T T E D ;
-CONFIG         : C O N F I G ;
-CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ;
-CSV            : C S V ;
-DATA           : D A T A ;
-DELIMITER      : D E L I M I T E R ;
-DATABASE       : D A T A B A S E ;
-DENY           : D E N Y ;
-DIRECTORY      : D I R E C T O R Y ;
-DROP           : D R O P ;
-DUMP           : D U M P ;
-DURABILITY     : D U R A B I L I T Y ;
-EXECUTE        : E X E C U T E ;
-FOR            : F O R ;
-FREE           : F R E E ;
-FREE_MEMORY    : F R E E UNDERSCORE M E M O R Y ;
-FROM           : F R O M ;
-GLOBAL         : G L O B A L ;
-GRANT          : G R A N T ;
-GRANTS         : G R A N T S ;
-HEADER         : H E A D E R ;
-IDENTIFIED     : I D E N T I F I E D ;
-IGNORE         : I G N O R E ;
-ISOLATION      : I S O L A T I O N ;
-LEVEL          : L E V E L ;
-LOAD           : L O A D ;
-LOCK           : L O C K ;
-MAIN           : M A I N ;
-MODE           : M O D E ;
-NEXT           : N E X T ;
-NO             : N O ;
-PASSWORD       : P A S S W O R D ;
-PORT           : P O R T ;
-PRIVILEGES     : P R I V I L E G E S ;
-READ           : R E A D ;
-READ_FILE      : R E A D UNDERSCORE F I L E ;
-REGISTER       : R E G I S T E R ;
-REPLICA        : R E P L I C A ;
-REPLICAS       : R E P L I C A S ;
-REPLICATION    : R E P L I C A T I O N ;
-REVOKE         : R E V O K E ;
-ROLE           : R O L E ;
-ROLES          : R O L E S ;
-QUOTE          : Q U O T E ;
-SESSION        : S E S S I O N ;
-SETTING        : S E T T I N G ;
-SETTINGS       : S E T T I N G S ;
-SNAPSHOT       : S N A P S H O T ;
-START          : S T A R T ;
-STATS          : S T A T S ;
-STOP           : S T O P ;
-STREAM         : S T R E A M ;
-STREAMS        : S T R E A M S ;
-SYNC           : S Y N C ;
-TIMEOUT        : T I M E O U T ;
-TO             : T O ;
-TOPICS         : T O P I C S;
-TRANSACTION    : T R A N S A C T I O N ;
-TRANSFORM      : T R A N S F O R M ;
-TRIGGER        : T R I G G E R ;
-TRIGGERS       : T R I G G E R S ;
-UNCOMMITTED    : U N C O M M I T T E D ;
-UNLOCK         : U N L O C K ;
-UPDATE         : U P D A T E ;
-USER           : U S E R ;
-USERS          : U S E R S ;
+AFTER               : A F T E R ;
+ALTER               : A L T E R ;
+ASYNC               : A S Y N C ;
+AUTH                : A U T H ;
+BAD                 : B A D ;
+BATCH_INTERVAL      : B A T C H UNDERSCORE I N T E R V A L ;
+BATCH_LIMIT         : B A T C H UNDERSCORE L I M I T ;
+BATCH_SIZE          : B A T C H UNDERSCORE S I Z E ;
+BEFORE              : B E F O R E ;
+BOOTSTRAP_SERVERS   : B O O T S T R A P UNDERSCORE S E R V E R S ;
+CHECK               : C H E C K ;
+CLEAR               : C L E A R ;
+COMMIT              : C O M M I T ;
+COMMITTED           : C O M M I T T E D ;
+CONFIG              : C O N F I G ;
+CONSUMER_GROUP      : C O N S U M E R UNDERSCORE G R O U P ;
+CSV                 : C S V ;
+DATA                : D A T A ;
+DELIMITER           : D E L I M I T E R ;
+DATABASE            : D A T A B A S E ;
+DENY                : D E N Y ;
+DIRECTORY           : D I R E C T O R Y ;
+DROP                : D R O P ;
+DUMP                : D U M P ;
+DURABILITY          : D U R A B I L I T Y ;
+EXECUTE             : E X E C U T E ;
+FOR                 : F O R ;
+FREE                : F R E E ;
+FREE_MEMORY         : F R E E UNDERSCORE M E M O R Y ;
+FROM                : F R O M ;
+GLOBAL              : G L O B A L ;
+GRANT               : G R A N T ;
+GRANTS              : G R A N T S ;
+HEADER              : H E A D E R ;
+IDENTIFIED          : I D E N T I F I E D ;
+IGNORE              : I G N O R E ;
+ISOLATION           : I S O L A T I O N ;
+KAFKA               : K A F K A ;
+LEVEL               : L E V E L ;
+LOAD                : L O A D ;
+LOCK                : L O C K ;
+MAIN                : M A I N ;
+MODE                : M O D E ;
+NEXT                : N E X T ;
+NO                  : N O ;
+PASSWORD            : P A S S W O R D ;
+PORT                : P O R T ;
+PRIVILEGES          : P R I V I L E G E S ;
+PULSAR              : P U L S A R ;
+READ                : R E A D ;
+READ_FILE           : R E A D UNDERSCORE F I L E ;
+REGISTER            : R E G I S T E R ;
+REPLICA             : R E P L I C A ;
+REPLICAS            : R E P L I C A S ;
+REPLICATION         : R E P L I C A T I O N ;
+REVOKE              : R E V O K E ;
+ROLE                : R O L E ;
+ROLES               : R O L E S ;
+QUOTE               : Q U O T E ;
+SERVICE_URL         : S E R V I C E UNDERSCORE U R L ;
+SESSION             : S E S S I O N ;
+SETTING             : S E T T I N G ;
+SETTINGS            : S E T T I N G S ;
+SNAPSHOT            : S N A P S H O T ;
+START               : S T A R T ;
+STATS               : S T A T S ;
+STOP                : S T O P ;
+STREAM              : S T R E A M ;
+STREAMS             : S T R E A M S ;
+SYNC                : S Y N C ;
+TIMEOUT             : T I M E O U T ;
+TO                  : T O ;
+TOPICS              : T O P I C S;
+TRANSACTION         : T R A N S A C T I O N ;
+TRANSFORM           : T R A N S F O R M ;
+TRIGGER             : T R I G G E R ;
+TRIGGERS            : T R I G G E R S ;
+UNCOMMITTED         : U N C O M M I T T E D ;
+UNLOCK              : U N L O C K ;
+UPDATE              : U P D A T E ;
+USER                : U S E R ;
+USERS               : U S E R S ;

src/query/frontend/stripped_lexer_constants.hpp

@@ -89,61 +89,120 @@ class Trie {
 
 const int kBitsetSize = 65536;
 
-const trie::Trie kKeywords = {"union",       "all",
-                              "optional",    "match",
-                              "unwind",      "as",
-                              "merge",       "on",
-                              "create",      "set",
-                              "detach",      "delete",
-                              "remove",      "with",
-                              "distinct",    "return",
-                              "order",       "by",
-                              "skip",        "limit",
-                              "ascending",   "asc",
-                              "descending",  "desc",
-                              "where",       "or",
-                              "xor",         "and",
-                              "not",         "in",
-                              "starts",      "ends",
-                              "contains",    "is",
-                              "null",        "case",
-                              "when",        "then",
-                              "else",        "end",
-                              "count",       "filter",
-                              "extract",     "any",
-                              "none",        "single",
-                              "true",        "false",
-                              "reduce",      "coalesce",
-                              "user",        "password",
-                              "alter",       "drop",
-                              "show",        "stats",
-                              "unique",      "explain",
-                              "profile",     "storage",
-                              "index",       "info",
-                              "exists",      "assert",
-                              "constraint",  "node",
-                              "key",         "dump",
-                              "database",    "call",
-                              "yield",       "memory",
-                              "mb",          "kb",
-                              "unlimited",   "free",
-                              "procedure",   "query",
-                              "free_memory", "read_file",
-                              "lock_path",   "after",
-                              "before",      "execute",
-                              "transaction", "trigger",
-                              "triggers",    "update",
-                              "comitted",    "uncomitted",
-                              "global",      "isolation",
-                              "level",       "next",
-                              "read",        "session",
-                              "snapshot",    "transaction",
-                              "batch_limit", "batch_interval",
-                              "batch_size",  "consumer_group",
-                              "start",       "stream",
-                              "streams",     "transform",
-                              "topics",      "check",
-                              "setting",     "settings"};
+const trie::Trie kKeywords = {"union",
+                              "all",
+                              "optional",
+                              "match",
+                              "unwind",
+                              "as",
+                              "merge",
+                              "on",
+                              "create",
+                              "set",
+                              "detach",
+                              "delete",
+                              "remove",
+                              "with",
+                              "distinct",
+                              "return",
+                              "order",
+                              "by",
+                              "skip",
+                              "limit",
+                              "ascending",
+                              "asc",
+                              "descending",
+                              "desc",
+                              "where",
+                              "or",
+                              "xor",
+                              "and",
+                              "not",
+                              "in",
+                              "starts",
+                              "ends",
+                              "contains",
+                              "is",
+                              "null",
+                              "case",
+                              "when",
+                              "then",
+                              "else",
+                              "end",
+                              "count",
+                              "filter",
+                              "extract",
+                              "any",
+                              "none",
+                              "single",
+                              "true",
+                              "false",
+                              "reduce",
+                              "coalesce",
+                              "user",
+                              "password",
+                              "alter",
+                              "drop",
+                              "show",
+                              "stats",
+                              "unique",
+                              "explain",
+                              "profile",
+                              "storage",
+                              "index",
+                              "info",
+                              "exists",
+                              "assert",
+                              "constraint",
+                              "node",
+                              "key",
+                              "dump",
+                              "database",
+                              "call",
+                              "yield",
+                              "memory",
+                              "mb",
+                              "kb",
+                              "unlimited",
+                              "free",
+                              "procedure",
+                              "query",
+                              "free_memory",
+                              "read_file",
+                              "lock_path",
+                              "after",
+                              "before",
+                              "execute",
+                              "transaction",
+                              "trigger",
+                              "triggers",
+                              "update",
+                              "comitted",
+                              "uncomitted",
+                              "global",
+                              "isolation",
+                              "level",
+                              "next",
+                              "read",
+                              "session",
+                              "snapshot",
+                              "transaction",
+                              "batch_limit",
+                              "batch_interval",
+                              "batch_size",
+                              "consumer_group",
+                              "start",
+                              "stream",
+                              "streams",
+                              "transform",
+                              "topics",
+                              "check",
+                              "setting",
+                              "settings",
+                              "bootstrap_servers",
+                              "kafka",
+                              "pulsar",
+                              "service_url"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

src/query/interpreter.cpp

@@ -34,7 +34,7 @@
 #include "query/plan/planner.hpp"
 #include "query/plan/profile.hpp"
 #include "query/plan/vertex_count_cache.hpp"
-#include "query/streams.hpp"
+#include "query/stream/common.hpp"
 #include "query/trigger.hpp"
 #include "query/typed_value.hpp"
 #include "storage/v2/property_value.hpp"
@@ -52,6 +52,7 @@
 #include "utils/settings.hpp"
 #include "utils/string.hpp"
 #include "utils/tsc.hpp"
+#include "utils/variant.hpp"
 
 namespace EventCounter {
 extern Event ReadQuery;
@@ -86,7 +87,8 @@ void UpdateTypeCount(const plan::ReadWriteTypeChecker::RWType type) {
 
 struct Callback {
   std::vector<std::string> header;
-  std::function<std::vector<std::vector<TypedValue>>()> fn;
+  using CallbackFunction = std::function<std::vector<std::vector<TypedValue>>()>;
+  CallbackFunction fn;
   bool should_abort_query{false};
 };
 
@@ -106,6 +108,17 @@ std::optional<TResult> GetOptionalValue(query::Expression *expression, Expressio
   return {};
 };
 
+std::optional<std::string> GetOptionalStringValue(query::Expression *expression, ExpressionEvaluator &evaluator) {
+  if (expression != nullptr) {
+    auto value = expression->Accept(evaluator);
+    MG_ASSERT(value.IsNull() || value.IsString());
+    if (value.IsString()) {
+      return {std::string(value.ValueString().begin(), value.ValueString().end())};
+    }
+  }
+  return {};
+};
+
 class ReplQueryHandler final : public query::ReplicationQueryHandler {
  public:
   explicit ReplQueryHandler(storage::Storage *db) : db_(db) {}
@@ -500,6 +513,76 @@ std::optional<std::string> StringPointerToOptional(const std::string *str) {
   return str == nullptr ? std::nullopt : std::make_optional(*str);
 }
 
+CommonStreamInfo GetCommonStreamInfo(StreamQuery *stream_query, ExpressionEvaluator &evaluator) {
+  return {.batch_interval = GetOptionalValue<std::chrono::milliseconds>(stream_query->batch_interval_, evaluator),
+          .batch_size = GetOptionalValue<int64_t>(stream_query->batch_size_, evaluator),
+          .transformation_name = stream_query->transform_name_};
+}
+
+std::vector<std::string> EvaluateTopicNames(ExpressionEvaluator &evaluator,
+                                            std::variant<Expression *, std::vector<std::string>> topic_variant) {
+  return std::visit(utils::Overloaded{[&](Expression *expression) {
+                                        auto topic_names = expression->Accept(evaluator);
+                                        MG_ASSERT(topic_names.IsString());
+                                        return utils::Split(topic_names.ValueString(), ",");
+                                      },
+                                      [&](std::vector<std::string> topic_names) { return topic_names; }},
+                    std::move(topic_variant));
+}
+
+Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator,
+                                                  InterpreterContext *interpreter_context,
+                                                  const std::string *username) {
+  constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
+  std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup
+                                                                   : stream_query->consumer_group_};
+
+  auto bootstrap = GetOptionalStringValue(stream_query->bootstrap_servers_, evaluator);
+  if (bootstrap && bootstrap->empty()) {
+    throw SemanticException("Bootstrap servers must not be an empty string!");
+  }
+  auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
+
+  return [interpreter_context, stream_name = stream_query->stream_name_,
+          topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
+          consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info),
+          bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username)]() mutable {
+    std::string bootstrap = bootstrap_servers
+                                ? std::move(*bootstrap_servers)
+                                : std::string{interpreter_context->config.default_kafka_bootstrap_servers};
+    interpreter_context->streams.Create<query::KafkaStream>(stream_name,
+                                                            {.common_info = std::move(common_stream_info),
+                                                             .topics = std::move(topic_names),
+                                                             .consumer_group = std::move(consumer_group),
+                                                             .bootstrap_servers = std::move(bootstrap)},
+                                                            std::move(owner));
+
+    return std::vector<std::vector<TypedValue>>{};
+  };
+}
+
+Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator,
+                                                   InterpreterContext *interpreter_context,
+                                                   const std::string *username) {
+  auto service_url = GetOptionalStringValue(stream_query->service_url_, evaluator);
+  if (service_url && service_url->empty()) {
+    throw SemanticException("Service URL must not be an empty string!");
+  }
+  auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
+  return [interpreter_context, stream_name = stream_query->stream_name_,
+          topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
+          common_stream_info = std::move(common_stream_info), service_url = std::move(service_url),
+          owner = StringPointerToOptional(username)]() mutable {
+    interpreter_context->streams.Create<query::PulsarStream>(stream_name,
+                                                             {.common_info = std::move(common_stream_info),
+                                                              .topics = std::move(topic_names),
+                                                              .service_url = std::move(*service_url)},
+                                                             std::move(owner));
+
+    return std::vector<std::vector<TypedValue>>{};
+  };
+}
+
 Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &parameters,
                            InterpreterContext *interpreter_context, DbAccessor *db_accessor,
                            const std::string *username) {
@@ -516,25 +599,15 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
   switch (stream_query->action_) {
     case StreamQuery::Action::CREATE_STREAM: {
       EventCounter::IncrementCounter(EventCounter::StreamsCreated);
-      constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
-      std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup
-                                                                       : stream_query->consumer_group_};
+      switch (stream_query->type_) {
+        case StreamQuery::Type::KAFKA:
+          callback.fn = GetKafkaCreateCallback(stream_query, evaluator, interpreter_context, username);
+          break;
+        case StreamQuery::Type::PULSAR:
+          callback.fn = GetPulsarCreateCallback(stream_query, evaluator, interpreter_context, username);
+          break;
+      }
 
-      callback.fn =
-          [interpreter_context, stream_name = stream_query->stream_name_, topic_names = stream_query->topic_names_,
-           consumer_group = std::move(consumer_group),
-           batch_interval = GetOptionalValue<std::chrono::milliseconds>(stream_query->batch_interval_, evaluator),
-           batch_size = GetOptionalValue<int64_t>(stream_query->batch_size_, evaluator),
-           transformation_name = stream_query->transform_name_, owner = StringPointerToOptional(username)]() mutable {
-            interpreter_context->streams.Create(stream_name,
-                                                query::StreamInfo{.topics = std::move(topic_names),
-                                                                  .consumer_group = std::move(consumer_group),
-                                                                  .batch_interval = batch_interval,
-                                                                  .batch_size = batch_size,
-                                                                  .transformation_name = std::move(transformation_name),
-                                                                  .owner = std::move(owner)});
-            return std::vector<std::vector<TypedValue>>{};
-          };
       return callback;
     }
     case StreamQuery::Action::START_STREAM: {
@@ -573,25 +646,12 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
       return callback;
     }
     case StreamQuery::Action::SHOW_STREAMS: {
-      callback.header = {"name",  "topics",    "consumer_group", "batch_interval", "batch_size", "transformation_name",
-                         "owner", "is running"};
+      callback.header = {"name", "batch_interval", "batch_size", "transformation_name", "owner", "is running"};
       callback.fn = [interpreter_context]() {
         auto streams_status = interpreter_context->streams.GetStreamInfo();
         std::vector<std::vector<TypedValue>> results;
         results.reserve(streams_status.size());
-        auto topics_as_typed_topics = [](const auto &topics) {
-          std::vector<TypedValue> typed_topics;
-          typed_topics.reserve(topics.size());
-          for (const auto &elem : topics) {
-            typed_topics.emplace_back(elem);
-          }
-          return typed_topics;
-        };
-
-        auto stream_info_as_typed_stream_info_emplace_in = [topics_as_typed_topics](auto &typed_status,
-                                                                                    const auto &stream_info) {
-          typed_status.emplace_back(topics_as_typed_topics(stream_info.topics));
-          typed_status.emplace_back(stream_info.consumer_group);
+        auto stream_info_as_typed_stream_info_emplace_in = [](auto &typed_status, const auto &stream_info) {
           if (stream_info.batch_interval.has_value()) {
             typed_status.emplace_back(stream_info.batch_interval->count());
           } else {
@@ -603,18 +663,18 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
             typed_status.emplace_back();
           }
           typed_status.emplace_back(stream_info.transformation_name);
-          if (stream_info.owner.has_value()) {
-            typed_status.emplace_back(*stream_info.owner);
-          } else {
-            typed_status.emplace_back();
-          }
         };
 
         for (const auto &status : streams_status) {
           std::vector<TypedValue> typed_status;
-          typed_status.reserve(7);
+          typed_status.reserve(8);
           typed_status.emplace_back(status.name);
           stream_info_as_typed_stream_info_emplace_in(typed_status, status.info);
+          if (status.owner.has_value()) {
+            typed_status.emplace_back(*status.owner);
+          } else {
+            typed_status.emplace_back();
+          }
           typed_status.emplace_back(status.is_running);
           results.push_back(std::move(typed_status));
         }
@@ -871,11 +931,8 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
 }  // namespace
 
 InterpreterContext::InterpreterContext(storage::Storage *db, const InterpreterConfig config,
-                                       const std::filesystem::path &data_directory, std::string kafka_bootstrap_servers)
-    : db(db),
-      trigger_store(data_directory / "triggers"),
-      config(config),
-      streams{this, std::move(kafka_bootstrap_servers), data_directory / "streams"} {}
+                                       const std::filesystem::path &data_directory)
+    : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory / "streams"} {}
 
 Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
   MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");

src/query/interpreter.hpp

@@ -26,7 +26,7 @@
 #include "query/plan/operator.hpp"
 #include "query/plan/read_write_type_checker.hpp"
 #include "query/stream.hpp"
-#include "query/streams.hpp"
+#include "query/stream/streams.hpp"
 #include "query/trigger.hpp"
 #include "query/typed_value.hpp"
 #include "storage/v2/isolation_level.hpp"
@@ -164,7 +164,7 @@ struct PreparedQuery {
  */
 struct InterpreterContext {
   explicit InterpreterContext(storage::Storage *db, InterpreterConfig config,
-                              const std::filesystem::path &data_directory, std::string kafka_bootstrap_servers);
+                              const std::filesystem::path &data_directory);
 
   storage::Storage *db;
 

src/query/plan/operator.cpp

@@ -49,6 +49,7 @@
 #include "utils/pmr/vector.hpp"
 #include "utils/readable_size.hpp"
 #include "utils/string.hpp"
+#include "utils/temporal.hpp"
 
 // macro for the default implementation of LogicalOperator::Accept
 // that accepts the visitor and visits it's input_ operator
@@ -1445,7 +1446,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor {
     // satisfy the "where" condition. if so, places them in the priority
     // queue.
     auto expand_pair = [this, &evaluator, &frame, &create_state](const EdgeAccessor &edge, const VertexAccessor &vertex,
-                                                                 double weight, int64_t depth) {
+                                                                 const TypedValue &total_weight, int64_t depth) {
       auto *memory = evaluator.GetMemoryResource();
       if (self_.filter_lambda_.expression) {
         frame[self_.filter_lambda_.inner_edge_symbol] = edge;
@@ -1457,27 +1458,48 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor {
       frame[self_.weight_lambda_->inner_edge_symbol] = edge;
       frame[self_.weight_lambda_->inner_node_symbol] = vertex;
 
-      TypedValue typed_weight = self_.weight_lambda_->expression->Accept(evaluator);
+      TypedValue current_weight = self_.weight_lambda_->expression->Accept(evaluator);
 
-      if (!typed_weight.IsNumeric()) {
-        throw QueryRuntimeException("Calculated weight must be numeric, got {}.", typed_weight.type());
+      if (!current_weight.IsNumeric() && !current_weight.IsDuration()) {
+        throw QueryRuntimeException("Calculated weight must be numeric or a Duration, got {}.", current_weight.type());
       }
-      if ((typed_weight < TypedValue(0, memory)).ValueBool()) {
+
+      const auto is_valid_numeric = [&] {
+        return current_weight.IsNumeric() && (current_weight >= TypedValue(0, memory)).ValueBool();
+      };
+
+      const auto is_valid_duration = [&] {
+        return current_weight.IsDuration() && (current_weight >= TypedValue(utils::Duration(0), memory)).ValueBool();
+      };
+
+      if (!is_valid_numeric() && !is_valid_duration()) {
         throw QueryRuntimeException("Calculated weight must be non-negative!");
       }
 
       auto next_state = create_state(vertex, depth);
-      auto next_weight = TypedValue(weight, memory) + typed_weight;
-      auto found_it = total_cost_.find(next_state);
-      if (found_it != total_cost_.end() && found_it->second.ValueDouble() <= next_weight.ValueDouble()) return;
 
-      pq_.push({next_weight.ValueDouble(), depth + 1, vertex, edge});
+      TypedValue next_weight = std::invoke([&] {
+        if (total_weight.IsNull()) {
+          return current_weight;
+        }
+
+        ValidateWeightTypes(current_weight, total_weight);
+
+        return TypedValue(current_weight, memory) + total_weight;
+      });
+
+      auto found_it = total_cost_.find(next_state);
+      if (found_it != total_cost_.end() && (found_it->second.IsNull() || (found_it->second <= next_weight).ValueBool()))
+        return;
+
+      pq_.push({next_weight, depth + 1, vertex, edge});
     };
 
     // Populates the priority queue structure with expansions
     // from the given vertex. skips expansions that don't satisfy
     // the "where" condition.
-    auto expand_from_vertex = [this, &expand_pair](const VertexAccessor &vertex, double weight, int64_t depth) {
+    auto expand_from_vertex = [this, &expand_pair](const VertexAccessor &vertex, const TypedValue &weight,
+                                                   int64_t depth) {
       if (self_.common_.direction != EdgeAtom::Direction::IN) {
         auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types));
         for (const auto &edge : out_edges) {
@@ -1522,7 +1544,7 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor {
         total_cost_.clear();
         yielded_vertices_.clear();
 
-        pq_.push({0.0, 0, vertex, std::nullopt});
+        pq_.push({TypedValue(), 0, vertex, std::nullopt});
         // We are adding the starting vertex to the set of yielded vertices
         // because we don't want to yield paths that end with the starting
         // vertex.
@@ -1622,17 +1644,38 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor {
   // Keeps track of vertices for which we yielded a path already.
   utils::pmr::unordered_set<VertexAccessor> yielded_vertices_;
 
+  static void ValidateWeightTypes(const TypedValue &lhs, const TypedValue &rhs) {
+    if (!((lhs.IsNumeric() && lhs.IsNumeric()) || (rhs.IsDuration() && rhs.IsDuration()))) {
+      throw QueryRuntimeException(utils::MessageWithLink(
+          "All weights should be of the same type, either numeric or a Duration. Please update the weight "
+          "expression or the filter expression.",
+          "https://memgr.ph/wsp"));
+    }
+  }
+
   // Priority queue comparator. Keep lowest weight on top of the queue.
   class PriorityQueueComparator {
    public:
-    bool operator()(const std::tuple<double, int64_t, VertexAccessor, std::optional<EdgeAccessor>> &lhs,
-                    const std::tuple<double, int64_t, VertexAccessor, std::optional<EdgeAccessor>> &rhs) {
-      return std::get<0>(lhs) > std::get<0>(rhs);
+    bool operator()(const std::tuple<TypedValue, int64_t, VertexAccessor, std::optional<EdgeAccessor>> &lhs,
+                    const std::tuple<TypedValue, int64_t, VertexAccessor, std::optional<EdgeAccessor>> &rhs) {
+      const auto &lhs_weight = std::get<0>(lhs);
+      const auto &rhs_weight = std::get<0>(rhs);
+      // Null defines minimum value for all types
+      if (lhs_weight.IsNull()) {
+        return false;
+      }
+
+      if (rhs_weight.IsNull()) {
+        return true;
+      }
+
+      ValidateWeightTypes(lhs_weight, rhs_weight);
+      return (lhs_weight > rhs_weight).ValueBool();
     }
   };
 
-  std::priority_queue<std::tuple<double, int64_t, VertexAccessor, std::optional<EdgeAccessor>>,
-                      utils::pmr::vector<std::tuple<double, int64_t, VertexAccessor, std::optional<EdgeAccessor>>>,
+  std::priority_queue<std::tuple<TypedValue, int64_t, VertexAccessor, std::optional<EdgeAccessor>>,
+                      utils::pmr::vector<std::tuple<TypedValue, int64_t, VertexAccessor, std::optional<EdgeAccessor>>>,
                       PriorityQueueComparator>
       pq_;
 

src/query/procedure/mg_procedure_impl.cpp

@@ -1218,7 +1218,7 @@ mgp_error mgp_local_time_get_hour(mgp_local_time *local_time, int *hour) {
   return WrapExceptions([local_time] { return local_time->local_time.hour; }, hour);
 }
 
-mgp_error mgp_local_time_get_minue(mgp_local_time *local_time, int *minute) {
+mgp_error mgp_local_time_get_minute(mgp_local_time *local_time, int *minute) {
   return WrapExceptions([local_time] { return local_time->local_time.minute; }, minute);
 }
 
@@ -2493,27 +2493,109 @@ bool IsValidIdentifierName(const char *name) {
 }  // namespace query::procedure
 
 mgp_error mgp_message_payload(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->Payload().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Payload().data();
+              } else if constexpr (std::same_as<MessageType, mgp_message::PulsarMessage>) {
+                return msg.Payload().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_payload_size(mgp_message *message, size_t *result) {
-  return WrapExceptions([message] { return message->msg->Payload().size(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> size_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Payload().size();
+              } else if constexpr (std::same_as<MessageType, mgp_message::PulsarMessage>) {
+                return msg.Payload().size();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_topic_name(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->TopicName().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->TopicName().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_key(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->Key().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Key().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_key_size(mgp_message *message, size_t *result) {
-  return WrapExceptions([message] { return message->msg->Key().size(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> size_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Key().size();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_timestamp(mgp_message *message, int64_t *result) {
-  return WrapExceptions([message] { return message->msg->Timestamp(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> int64_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Timestamp();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_messages_size(mgp_messages *messages, size_t *result) {

src/query/procedure/mg_procedure_impl.hpp

@@ -20,6 +20,7 @@
 #include <ostream>
 
 #include "integrations/kafka/consumer.hpp"
+#include "integrations/pulsar/consumer.hpp"
 #include "query/context.hpp"
 #include "query/db_accessor.hpp"
 #include "query/procedure/cypher_type_ptr.hpp"
@@ -790,7 +791,12 @@ bool IsValidIdentifierName(const char *name);
 }  // namespace query::procedure
 
 struct mgp_message {
-  const integrations::kafka::Message *msg;
+  explicit mgp_message(const integrations::kafka::Message &message) : msg{&message} {}
+  explicit mgp_message(const integrations::pulsar::Message &message) : msg{message} {}
+
+  using KafkaMessage = const integrations::kafka::Message *;
+  using PulsarMessage = integrations::pulsar::Message;
+  std::variant<KafkaMessage, PulsarMessage> msg;
 };
 
 struct mgp_messages {

src/query/procedure/module.cpp

@@ -325,15 +325,16 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
   dlerror();  // Clear any existing error.
   handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL);
   if (!handle_) {
-    // NOLINTNEXTLINE(concurrency-mt-unsafe)
-    spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
     return false;
   }
   // Get required mgp_init_module
   init_fn_ = reinterpret_cast<int (*)(mgp_module *, mgp_memory *)>(dlsym(handle_, "mgp_init_module"));
   char *dl_errored = dlerror();
   if (!init_fn_ || dl_errored) {
-    spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules"));
     dlclose(handle_);
     handle_ = nullptr;
     return false;
@@ -384,8 +385,8 @@ bool SharedLibraryModule::Close() {
     spdlog::warn("When closing module {}; mgp_shutdown_module returned {}", file_path_, shutdown_res);
   }
   if (dlclose(handle_) != 0) {
-    // NOLINTNEXTLINE(concurrency-mt-unsafe)
-    spdlog::error(utils::MessageWithLink("Failed to close module {}; {}.", file_path_, dlerror(), "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Failed to close module {}; {}.", file_path_, dlerror(), "https://memgr.ph/modules"));
     return false;
   }
   spdlog::info("Closed module {}", file_path_);
@@ -444,7 +445,8 @@ bool PythonModule::Load(const std::filesystem::path &file_path) {
   auto gil = py::EnsureGIL();
   auto maybe_exc = py::AppendToSysPath(file_path.parent_path().c_str());
   if (maybe_exc) {
-    spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, *maybe_exc, "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Unable to load module {}; {}.", file_path, *maybe_exc, "https://memgr.ph/modules"));
     return false;
   }
   bool succ = true;
@@ -469,7 +471,8 @@ bool PythonModule::Load(const std::filesystem::path &file_path) {
     return true;
   }
   auto exc_info = py::FetchError().value();
-  spdlog::error(utils::MessageWithLink("Unable to load module {}; {}.", file_path, exc_info, "https://memgr.ph/modules"));
+  spdlog::error(
+      utils::MessageWithLink("Unable to load module {}; {}.", file_path, exc_info, "https://memgr.ph/modules"));
   return false;
 }
 
@@ -534,7 +537,8 @@ bool ModuleRegistry::RegisterModule(const std::string_view &name, std::unique_pt
   MG_ASSERT(!name.empty(), "Module name cannot be empty");
   MG_ASSERT(module, "Tried to register an invalid module");
   if (modules_.find(name) != modules_.end()) {
-    spdlog::error(utils::MessageWithLink("Unable to overwrite an already loaded module {}.", name, "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Unable to overwrite an already loaded module {}.", name, "https://memgr.ph/modules"));
     return false;
   }
   modules_.emplace(name, std::move(module));
@@ -564,7 +568,8 @@ void ModuleRegistry::SetModulesDirectory(std::vector<std::filesystem::path> modu
 
 bool ModuleRegistry::LoadModuleIfFound(const std::filesystem::path &modules_dir, const std::string_view name) {
   if (!utils::DirExists(modules_dir)) {
-    spdlog::error(utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
     return false;
   }
   for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) {
@@ -601,7 +606,8 @@ bool ModuleRegistry::LoadOrReloadModuleFromName(const std::string_view name) {
 void ModuleRegistry::LoadModulesFromDirectory(const std::filesystem::path &modules_dir) {
   if (modules_dir.empty()) return;
   if (!utils::DirExists(modules_dir)) {
-    spdlog::error(utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
+    spdlog::error(
+        utils::MessageWithLink("Module directory {} doesn't exist.", modules_dir, "https://memgr.ph/modules"));
     return;
   }
   for (const auto &entry : std::filesystem::directory_iterator(modules_dir)) {

src/query/stream/common.cpp

@@ -0,0 +1,55 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/common.hpp"
+
+#include <json/json.hpp>
+
+namespace query {
+namespace {
+const std::string kBatchIntervalKey{"batch_interval"};
+const std::string kBatchSizeKey{"batch_size"};
+const std::string kTransformationName{"transformation_name"};
+}  // namespace
+
+void to_json(nlohmann::json &data, CommonStreamInfo &&common_info) {
+  if (common_info.batch_interval) {
+    data[kBatchIntervalKey] = common_info.batch_interval->count();
+  } else {
+    data[kBatchIntervalKey] = nullptr;
+  }
+
+  if (common_info.batch_size) {
+    data[kBatchSizeKey] = *common_info.batch_size;
+  } else {
+    data[kBatchSizeKey] = nullptr;
+  }
+
+  data[kTransformationName] = common_info.transformation_name;
+}
+
+void from_json(const nlohmann::json &data, CommonStreamInfo &common_info) {
+  if (const auto batch_interval = data.at(kBatchIntervalKey); !batch_interval.is_null()) {
+    using BatchInterval = typename decltype(common_info.batch_interval)::value_type;
+    common_info.batch_interval = BatchInterval{batch_interval.get<typename BatchInterval::rep>()};
+  } else {
+    common_info.batch_interval = {};
+  }
+
+  if (const auto batch_size = data.at(kBatchSizeKey); !batch_size.is_null()) {
+    common_info.batch_size = batch_size.get<typename decltype(common_info.batch_size)::value_type>();
+  } else {
+    common_info.batch_size = {};
+  }
+
+  data.at(kTransformationName).get_to(common_info.transformation_name);
+}
+}  // namespace query

src/query/stream/common.hpp

@@ -0,0 +1,73 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <optional>
+#include <string>
+
+#include <json/json.hpp>
+
+#include "query/procedure/mg_procedure_impl.hpp"
+
+namespace query {
+
+template <typename TMessage>
+using ConsumerFunction = std::function<void(const std::vector<TMessage> &)>;
+
+struct CommonStreamInfo {
+  std::optional<std::chrono::milliseconds> batch_interval;
+  std::optional<int64_t> batch_size;
+  std::string transformation_name;
+};
+
+template <typename T>
+concept ConvertableToJson = requires(T value, nlohmann::json data) {
+  { to_json(data, std::move(value)) } -> std::same_as<void>;
+  { from_json(data, value) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept ConvertableToMgpMessage = requires(T value) {
+  mgp_message{value};
+};
+
+template <typename TStream>
+concept Stream = requires(TStream stream) {
+  typename TStream::StreamInfo;
+  typename TStream::Message;
+  TStream{std::string{""}, typename TStream::StreamInfo{}, ConsumerFunction<typename TStream::Message>{}};
+  { stream.Start() } -> std::same_as<void>;
+  { stream.Stop() } -> std::same_as<void>;
+  { stream.IsRunning() } -> std::same_as<bool>;
+  {
+    stream.Check(std::optional<std::chrono::milliseconds>{}, std::optional<int64_t>{},
+                 ConsumerFunction<typename TStream::Message>{})
+    } -> std::same_as<void>;
+  { typename TStream::StreamInfo{}.common_info } -> std::same_as<CommonStreamInfo>;
+
+  requires ConvertableToMgpMessage<typename TStream::Message>;
+  requires ConvertableToJson<typename TStream::StreamInfo>;
+};
+
+enum class StreamSourceType : uint8_t { KAFKA, PULSAR };
+
+template <Stream T>
+StreamSourceType StreamType(const T & /*stream*/);
+
+const std::string kCommonInfoKey = "common_info";
+
+void to_json(nlohmann::json &data, CommonStreamInfo &&info);
+void from_json(const nlohmann::json &data, CommonStreamInfo &common_info);
+}  // namespace query

src/query/stream/sources.cpp

@@ -0,0 +1,113 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/sources.hpp"
+
+#include <json/json.hpp>
+
+namespace query {
+KafkaStream::KafkaStream(std::string stream_name, StreamInfo stream_info,
+                         ConsumerFunction<integrations::kafka::Message> consumer_function) {
+  integrations::kafka::ConsumerInfo consumer_info{
+      .consumer_name = std::move(stream_name),
+      .topics = std::move(stream_info.topics),
+      .consumer_group = std::move(stream_info.consumer_group),
+      .bootstrap_servers = std::move(stream_info.bootstrap_servers),
+      .batch_interval = stream_info.common_info.batch_interval,
+      .batch_size = stream_info.common_info.batch_size,
+  };
+  consumer_.emplace(std::move(consumer_info), std::move(consumer_function));
+};
+
+KafkaStream::StreamInfo KafkaStream::Info(std::string transformation_name) const {
+  const auto &info = consumer_->Info();
+  return {{.batch_interval = info.batch_interval,
+           .batch_size = info.batch_size,
+           .transformation_name = std::move(transformation_name)},
+          .topics = info.topics,
+          .consumer_group = info.consumer_group,
+          .bootstrap_servers = info.bootstrap_servers};
+}
+
+void KafkaStream::Start() { consumer_->Start(); }
+void KafkaStream::Stop() { consumer_->Stop(); }
+bool KafkaStream::IsRunning() const { return consumer_->IsRunning(); }
+
+void KafkaStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+                        const ConsumerFunction<integrations::kafka::Message> &consumer_function) const {
+  consumer_->Check(timeout, batch_limit, consumer_function);
+}
+
+namespace {
+const std::string kTopicsKey{"topics"};
+const std::string kConsumerGroupKey{"consumer_group"};
+const std::string kBoostrapServers{"bootstrap_servers"};
+}  // namespace
+
+void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info) {
+  data[kCommonInfoKey] = std::move(info.common_info);
+  data[kTopicsKey] = std::move(info.topics);
+  data[kConsumerGroupKey] = info.consumer_group;
+  data[kBoostrapServers] = std::move(info.bootstrap_servers);
+}
+
+void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info) {
+  data.at(kCommonInfoKey).get_to(info.common_info);
+  data.at(kTopicsKey).get_to(info.topics);
+  data.at(kConsumerGroupKey).get_to(info.consumer_group);
+  data.at(kBoostrapServers).get_to(info.bootstrap_servers);
+}
+
+PulsarStream::PulsarStream(std::string stream_name, StreamInfo stream_info,
+                           ConsumerFunction<integrations::pulsar::Message> consumer_function) {
+  integrations::pulsar::ConsumerInfo consumer_info{.batch_size = stream_info.common_info.batch_size,
+                                                   .batch_interval = stream_info.common_info.batch_interval,
+                                                   .topics = std::move(stream_info.topics),
+                                                   .consumer_name = std::move(stream_name),
+                                                   .service_url = std::move(stream_info.service_url)};
+
+  consumer_.emplace(std::move(consumer_info), std::move(consumer_function));
+};
+
+PulsarStream::StreamInfo PulsarStream::Info(std::string transformation_name) const {
+  const auto &info = consumer_->Info();
+  return {{.batch_interval = info.batch_interval,
+           .batch_size = info.batch_size,
+           .transformation_name = std::move(transformation_name)},
+          .topics = info.topics,
+          .service_url = info.service_url};
+}
+
+void PulsarStream::Start() { consumer_->Start(); }
+void PulsarStream::Stop() { consumer_->Stop(); }
+bool PulsarStream::IsRunning() const { return consumer_->IsRunning(); }
+
+void PulsarStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+                         const ConsumerFunction<Message> &consumer_function) const {
+  consumer_->Check(timeout, batch_limit, consumer_function);
+}
+
+namespace {
+const std::string kServiceUrl{"service_url"};
+}  // namespace
+
+void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info) {
+  data[kCommonInfoKey] = std::move(info.common_info);
+  data[kTopicsKey] = std::move(info.topics);
+  data[kServiceUrl] = std::move(info.service_url);
+}
+
+void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info) {
+  data.at(kCommonInfoKey).get_to(info.common_info);
+  data.at(kTopicsKey).get_to(info.topics);
+  data.at(kServiceUrl).get_to(info.service_url);
+}
+}  // namespace query

src/query/stream/sources.hpp

@@ -0,0 +1,89 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "query/stream/common.hpp"
+
+#include "integrations/kafka/consumer.hpp"
+#include "integrations/pulsar/consumer.hpp"
+
+namespace query {
+
+struct KafkaStream {
+  struct StreamInfo {
+    CommonStreamInfo common_info;
+    std::vector<std::string> topics;
+    std::string consumer_group;
+    std::string bootstrap_servers;
+  };
+
+  using Message = integrations::kafka::Message;
+
+  KafkaStream(std::string stream_name, StreamInfo stream_info,
+              ConsumerFunction<integrations::kafka::Message> consumer_function);
+
+  StreamInfo Info(std::string transformation_name) const;
+
+  void Start();
+  void Stop();
+  bool IsRunning() const;
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+             const ConsumerFunction<Message> &consumer_function) const;
+
+ private:
+  using Consumer = integrations::kafka::Consumer;
+  std::optional<Consumer> consumer_;
+};
+
+void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info);
+void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info);
+
+template <>
+inline StreamSourceType StreamType(const KafkaStream & /*stream*/) {
+  return StreamSourceType::KAFKA;
+}
+
+struct PulsarStream {
+  struct StreamInfo {
+    CommonStreamInfo common_info;
+    std::vector<std::string> topics;
+    std::string service_url;
+  };
+
+  using Message = integrations::pulsar::Message;
+
+  PulsarStream(std::string stream_name, StreamInfo stream_info, ConsumerFunction<Message> consumer_function);
+
+  StreamInfo Info(std::string transformation_name) const;
+
+  void Start();
+  void Stop();
+  bool IsRunning() const;
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+             const ConsumerFunction<Message> &consumer_function) const;
+
+ private:
+  using Consumer = integrations::pulsar::Consumer;
+  std::optional<Consumer> consumer_;
+};
+
+void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info);
+void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info);
+
+template <>
+inline StreamSourceType StreamType(const PulsarStream & /*stream*/) {
+  return StreamSourceType::PULSAR;
+}
+
+}  // namespace query

src/query/stream/streams.cpp

@@ -0,0 +1,434 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/streams.hpp"
+
+#include <shared_mutex>
+#include <string_view>
+#include <utility>
+
+#include <spdlog/spdlog.h>
+#include <json/json.hpp>
+
+#include "query/db_accessor.hpp"
+#include "query/discard_value_stream.hpp"
+#include "query/interpreter.hpp"
+#include "query/procedure/mg_procedure_impl.hpp"
+#include "query/procedure/module.hpp"
+#include "query/typed_value.hpp"
+#include "utils/event_counter.hpp"
+#include "utils/memory.hpp"
+#include "utils/on_scope_exit.hpp"
+#include "utils/pmr/string.hpp"
+
+namespace EventCounter {
+extern const Event MessagesConsumed;
+}  // namespace EventCounter
+
+namespace query {
+namespace {
+constexpr auto kExpectedTransformationResultSize = 2;
+const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()};
+const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()};
+
+const std::map<std::string, storage::PropertyValue> empty_parameters{};
+
+auto GetStream(auto &map, const std::string &stream_name) {
+  if (auto it = map.find(stream_name); it != map.end()) {
+    return it;
+  }
+  throw StreamsException("Couldn't find stream '{}'", stream_name);
+}
+
+std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformationResult(
+    utils::pmr::map<utils::pmr::string, TypedValue> &&values, const std::string_view transformation_name,
+    const std::string_view stream_name) {
+  if (values.size() != kExpectedTransformationResultSize) {
+    throw StreamsException(
+        "Transformation '{}' in stream '{}' did not yield all fields (query, parameters) as required.",
+        transformation_name, stream_name);
+  }
+
+  auto get_value = [&](const utils::pmr::string &field_name) mutable -> TypedValue & {
+    auto it = values.find(field_name);
+    if (it == values.end()) {
+      throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.",
+                             transformation_name, stream_name, field_name};
+    };
+    return it->second;
+  };
+
+  auto &query_value = get_value(query_param_name);
+  MG_ASSERT(query_value.IsString());
+  auto &params_value = get_value(params_param_name);
+  MG_ASSERT(params_value.IsNull() || params_value.IsMap());
+  return {std::move(query_value), std::move(params_value)};
+}
+
+template <typename TMessage>
+void CallCustomTransformation(const std::string &transformation_name, const std::vector<TMessage> &messages,
+                              mgp_result &result, storage::Storage::Accessor &storage_accessor,
+                              utils::MemoryResource &memory_resource, const std::string &stream_name) {
+  DbAccessor db_accessor{&storage_accessor};
+  {
+    auto maybe_transformation =
+        procedure::FindTransformation(procedure::gModuleRegistry, transformation_name, utils::NewDeleteResource());
+
+    if (!maybe_transformation) {
+      throw StreamsException("Couldn't find transformation {} for stream '{}'", transformation_name, stream_name);
+    };
+    const auto &trans = *maybe_transformation->second;
+    mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}};
+    std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages),
+                   [](const TMessage &message) { return mgp_message{message}; });
+    mgp_graph graph{&db_accessor, storage::View::OLD, nullptr};
+    mgp_memory memory{&memory_resource};
+    result.rows.clear();
+    result.error_msg.reset();
+    result.signature = &trans.results;
+
+    MG_ASSERT(result.signature->size() == kExpectedTransformationResultSize);
+    MG_ASSERT(result.signature->contains(query_param_name));
+    MG_ASSERT(result.signature->contains(params_param_name));
+
+    spdlog::trace("Calling transformation in stream '{}'", stream_name);
+    trans.cb(&mgp_messages, &graph, &result, &memory);
+  }
+  if (result.error_msg.has_value()) {
+    throw StreamsException(result.error_msg->c_str());
+  }
+}
+
+template <Stream TStream>
+StreamStatus<TStream> CreateStatus(std::string stream_name, std::string transformation_name,
+                                   std::optional<std::string> owner, const TStream &stream) {
+  return {.name = std::move(stream_name),
+          .type = StreamType(stream),
+          .is_running = stream.IsRunning(),
+          .info = stream.Info(std::move(transformation_name)),
+          .owner = std::move(owner)};
+}
+
+// nlohmann::json doesn't support string_view access yet
+const std::string kStreamName{"name"};
+const std::string kIsRunningKey{"is_running"};
+const std::string kOwner{"owner"};
+const std::string kType{"type"};
+}  // namespace
+
+template <Stream TStream>
+void to_json(nlohmann::json &data, StreamStatus<TStream> &&status) {
+  data[kStreamName] = std::move(status.name);
+  data[kType] = status.type;
+  data[kIsRunningKey] = status.is_running;
+
+  if (status.owner.has_value()) {
+    data[kOwner] = std::move(*status.owner);
+  } else {
+    data[kOwner] = nullptr;
+  }
+
+  to_json(data, std::move(status.info));
+}
+
+template <Stream TStream>
+void from_json(const nlohmann::json &data, StreamStatus<TStream> &status) {
+  data.at(kStreamName).get_to(status.name);
+  data.at(kIsRunningKey).get_to(status.is_running);
+
+  if (const auto &owner = data.at(kOwner); !owner.is_null()) {
+    status.owner = owner.get<typename decltype(status.owner)::value_type>();
+  } else {
+    status.owner = {};
+  }
+
+  from_json(data, status.info);
+}
+
+Streams::Streams(InterpreterContext *interpreter_context, std::filesystem::path directory)
+    : interpreter_context_(interpreter_context), storage_(std::move(directory)) {}
+
+template <Stream TStream>
+void Streams::Create(const std::string &stream_name, typename TStream::StreamInfo info,
+                     std::optional<std::string> owner) {
+  auto locked_streams = streams_.Lock();
+  auto it = CreateConsumer<TStream>(*locked_streams, stream_name, std::move(info), std::move(owner));
+
+  try {
+    std::visit(
+        [&](auto &&stream_data) {
+          const auto stream_source_ptr = stream_data.stream_source->ReadLock();
+          Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+        },
+        it->second);
+  } catch (...) {
+    locked_streams->erase(it);
+    throw;
+  }
+}
+
+template void Streams::Create<KafkaStream>(const std::string &stream_name, KafkaStream::StreamInfo info,
+                                           std::optional<std::string> owner);
+template void Streams::Create<PulsarStream>(const std::string &stream_name, PulsarStream::StreamInfo info,
+                                            std::optional<std::string> owner);
+
+template <Stream TStream>
+Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std::string &stream_name,
+                                                      typename TStream::StreamInfo stream_info,
+                                                      std::optional<std::string> owner) {
+  if (map.contains(stream_name)) {
+    throw StreamsException{"Stream already exists with name '{}'", stream_name};
+  }
+
+  auto *memory_resource = utils::NewDeleteResource();
+
+  auto consumer_function =
+      [interpreter_context = interpreter_context_, memory_resource, stream_name,
+       transformation_name = stream_info.common_info.transformation_name, owner = owner,
+       interpreter = std::make_shared<Interpreter>(interpreter_context_),
+       result = mgp_result{nullptr, memory_resource}](const std::vector<typename TStream::Message> &messages) mutable {
+        auto accessor = interpreter_context->db->Access();
+        EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
+        CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
+
+        DiscardValueResultStream stream;
+
+        spdlog::trace("Start transaction in stream '{}'", stream_name);
+        utils::OnScopeExit cleanup{[&interpreter, &result]() {
+          result.rows.clear();
+          interpreter->Abort();
+        }};
+        interpreter->BeginTransaction();
+
+        const static std::map<std::string, storage::PropertyValue> empty_parameters{};
+
+        for (auto &row : result.rows) {
+          spdlog::trace("Processing row in stream '{}'", stream_name);
+          auto [query_value, params_value] =
+              ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
+          storage::PropertyValue params_prop{params_value};
+
+          std::string query{query_value.ValueString()};
+          spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name);
+          auto prepare_result =
+              interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr);
+          if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges)) {
+            throw StreamsException{
+                "Couldn't execute query '{}' for stream '{}' becuase the owner is not authorized to execute the "
+                "query!",
+                query, stream_name};
+          }
+          interpreter->PullAll(&stream);
+        }
+
+        spdlog::trace("Commit transaction in stream '{}'", stream_name);
+        interpreter->CommitTransaction();
+        result.rows.clear();
+      };
+
+  auto insert_result = map.try_emplace(
+      stream_name, StreamData<TStream>{std::move(stream_info.common_info.transformation_name), std::move(owner),
+                                       std::make_unique<SynchronizedStreamSource<TStream>>(
+                                           stream_name, std::move(stream_info), std::move(consumer_function))});
+  MG_ASSERT(insert_result.second, "Unexpected error during storing consumer '{}'", stream_name);
+  return insert_result.first;
+}
+
+void Streams::RestoreStreams() {
+  spdlog::info("Loading streams...");
+  auto locked_streams_map = streams_.Lock();
+  MG_ASSERT(locked_streams_map->empty(), "Cannot restore streams when some streams already exist!");
+
+  for (const auto &[stream_name, stream_data] : storage_) {
+    const auto get_failed_message = [&stream_name = stream_name](const std::string_view message,
+                                                                 const std::string_view nested_message) {
+      return fmt::format("Failed to load stream '{}', because: {} caused by {}", stream_name, message, nested_message);
+    };
+
+    const auto create_consumer = [&, &stream_name = stream_name, this]<typename T>(StreamStatus<T> status,
+                                                                                   auto &&stream_json_data) {
+      try {
+        stream_json_data.get_to(status);
+      } catch (const nlohmann::json::type_error &exception) {
+        spdlog::warn(get_failed_message("invalid type conversion", exception.what()));
+        return;
+      } catch (const nlohmann::json::out_of_range &exception) {
+        spdlog::warn(get_failed_message("non existing field", exception.what()));
+        return;
+      }
+      MG_ASSERT(status.name == stream_name, "Expected stream name is '{}', but got '{}'", status.name, stream_name);
+
+      try {
+        auto it = CreateConsumer<T>(*locked_streams_map, stream_name, std::move(status.info), std::move(status.owner));
+        if (status.is_running) {
+          std::visit(
+              [&](auto &&stream_data) {
+                auto stream_source_ptr = stream_data.stream_source->Lock();
+                stream_source_ptr->Start();
+              },
+              it->second);
+        }
+        spdlog::info("Stream '{}' is loaded", stream_name);
+      } catch (const utils::BasicException &exception) {
+        spdlog::warn(get_failed_message("unexpected error", exception.what()));
+      }
+    };
+
+    auto stream_json_data = nlohmann::json::parse(stream_data);
+    const auto stream_type = static_cast<StreamSourceType>(stream_json_data.at("type"));
+
+    switch (stream_type) {
+      case StreamSourceType::KAFKA:
+        create_consumer(StreamStatus<KafkaStream>{}, std::move(stream_json_data));
+        break;
+      case StreamSourceType::PULSAR:
+        create_consumer(StreamStatus<PulsarStream>{}, std::move(stream_json_data));
+        break;
+    }
+  }
+}
+
+void Streams::Drop(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+
+  auto it = GetStream(*locked_streams, stream_name);
+
+  // streams_ is write locked, which means there is no access to it outside of this function, thus only the Test
+  // function can be executing with the consumer, nothing else.
+  // By acquiring the write lock here for the consumer, we make sure there is
+  // no running Test function for this consumer, therefore it can be erased.
+  std::visit([&](auto &&stream_data) { stream_data.stream_source->Lock(); }, it->second);
+
+  locked_streams->erase(it);
+  if (!storage_.Delete(stream_name)) {
+    throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
+  }
+
+  // TODO(antaljanosbenjamin) Release the transformation
+}
+
+void Streams::Start(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+  auto it = GetStream(*locked_streams, stream_name);
+
+  std::visit(
+      [&, this](auto &&stream_data) {
+        auto stream_source_ptr = stream_data.stream_source->Lock();
+        stream_source_ptr->Start();
+        Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+      },
+      it->second);
+}
+
+void Streams::Stop(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+  auto it = GetStream(*locked_streams, stream_name);
+
+  std::visit(
+      [&, this](auto &&stream_data) {
+        auto stream_source_ptr = stream_data.stream_source->Lock();
+        stream_source_ptr->Stop();
+
+        Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+      },
+      it->second);
+}
+
+void Streams::StartAll() {
+  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
+    std::visit(
+        [&stream_name = stream_name, this](auto &&stream_data) {
+          auto locked_stream_source = stream_data.stream_source->Lock();
+          if (!locked_stream_source->IsRunning()) {
+            locked_stream_source->Start();
+            Persist(
+                CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source));
+          }
+        },
+        stream_data);
+  }
+}
+
+void Streams::StopAll() {
+  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
+    std::visit(
+        [&stream_name = stream_name, this](auto &&stream_data) {
+          auto locked_stream_source = stream_data.stream_source->Lock();
+          if (locked_stream_source->IsRunning()) {
+            locked_stream_source->Stop();
+            Persist(
+                CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source));
+          }
+        },
+        stream_data);
+  }
+}
+
+std::vector<StreamStatus<>> Streams::GetStreamInfo() const {
+  std::vector<StreamStatus<>> result;
+  {
+    for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) {
+      std::visit(
+          [&, &stream_name = stream_name](auto &&stream_data) {
+            auto locked_stream_source = stream_data.stream_source->ReadLock();
+            auto info = locked_stream_source->Info(stream_data.transformation_name);
+            result.emplace_back(StreamStatus<>{stream_name, StreamType(*locked_stream_source),
+                                               locked_stream_source->IsRunning(), std::move(info.common_info),
+                                               stream_data.owner});
+          },
+          stream_data);
+    }
+  }
+  return result;
+}
+
+TransformationResult Streams::Check(const std::string &stream_name, std::optional<std::chrono::milliseconds> timeout,
+                                    std::optional<int64_t> batch_limit) const {
+  std::optional locked_streams{streams_.ReadLock()};
+  auto it = GetStream(**locked_streams, stream_name);
+
+  return std::visit(
+      [&](auto &&stream_data) {
+        // This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after
+        // that
+        const auto locked_stream_source = stream_data.stream_source->ReadLock();
+        const auto transformation_name = stream_data.transformation_name;
+        locked_streams.reset();
+
+        auto *memory_resource = utils::NewDeleteResource();
+        mgp_result result{nullptr, memory_resource};
+        TransformationResult test_result;
+
+        auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name,
+                                  &transformation_name = transformation_name, &result,
+                                  &test_result]<typename T>(const std::vector<T> &messages) mutable {
+          auto accessor = interpreter_context->db->Access();
+          CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
+
+          for (auto &row : result.rows) {
+            auto [query, parameters] =
+                ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
+            std::vector<TypedValue> result_row;
+            result_row.reserve(kExpectedTransformationResultSize);
+            result_row.push_back(std::move(query));
+            result_row.push_back(std::move(parameters));
+
+            test_result.push_back(std::move(result_row));
+          }
+        };
+
+        locked_stream_source->Check(timeout, batch_limit, consumer_function);
+        return test_result;
+      },
+      it->second);
+}
+
+}  // namespace query

src/query/stream/streams.hpp

@@ -11,14 +11,22 @@
 
 #pragma once
 
+#include <concepts>
 #include <functional>
 #include <map>
 #include <optional>
+#include <type_traits>
 #include <unordered_map>
 
+#include <json/json.hpp>
+
 #include "integrations/kafka/consumer.hpp"
 #include "kvstore/kvstore.hpp"
+#include "query/stream/common.hpp"
+#include "query/stream/sources.hpp"
 #include "query/typed_value.hpp"
+#include "storage/v2/property_value.hpp"
+#include "utils/event_counter.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/rw_lock.hpp"
 #include "utils/synchronized.hpp"
@@ -30,32 +38,33 @@ class StreamsException : public utils::BasicException {
   using BasicException::BasicException;
 };
 
-using TransformationResult = std::vector<std::vector<TypedValue>>;
-using TransformFunction = std::function<TransformationResult(const std::vector<integrations::kafka::Message> &)>;
+template <typename T>
+struct StreamInfo;
 
-struct StreamInfo {
-  std::vector<std::string> topics;
-  std::string consumer_group;
-  std::optional<std::chrono::milliseconds> batch_interval;
-  std::optional<int64_t> batch_size;
-  std::string transformation_name;
-  std::optional<std::string> owner;
+template <>
+struct StreamInfo<void> {
+  using Type = CommonStreamInfo;
 };
 
+template <Stream TStream>
+struct StreamInfo<TStream> {
+  using Type = typename TStream::StreamInfo;
+};
+
+template <typename T>
+using StreamInfoType = typename StreamInfo<T>::Type;
+
+template <typename T = void>
 struct StreamStatus {
   std::string name;
-  StreamInfo info;
+  StreamSourceType type;
   bool is_running;
-};
-
-using SynchronizedConsumer = utils::Synchronized<integrations::kafka::Consumer, utils::WritePrioritizedRWLock>;
-
-struct StreamData {
-  std::string transformation_name;
+  StreamInfoType<T> info;
   std::optional<std::string> owner;
-  std::unique_ptr<SynchronizedConsumer> consumer;
 };
 
+using TransformationResult = std::vector<std::vector<TypedValue>>;
+
 struct InterpreterContext;
 
 /// Manages Kafka consumers.
@@ -66,9 +75,8 @@ class Streams final {
   /// Initializes the streams.
   ///
   /// @param interpreter_context context to use to run the result of transformations
-  /// @param bootstrap_servers initial list of brokers as a comma separated list of broker host or host:port
   /// @param directory a directory path to store the persisted streams metadata
-  Streams(InterpreterContext *interpreter_context, std::string bootstrap_servers, std::filesystem::path directory);
+  Streams(InterpreterContext *interpreter_context, std::filesystem::path directory);
 
   /// Restores the streams from the persisted metadata.
   /// The restoration is done in a best effort manner, therefore no exception is thrown on failure, but the error is
@@ -84,7 +92,8 @@ class Streams final {
   /// @param stream_info the necessary informations needed to create the Kafka consumer and transform the messages
   ///
   /// @throws StreamsException if the stream with the same name exists or if the creation of Kafka consumer fails
-  void Create(const std::string &stream_name, StreamInfo stream_info);
+  template <Stream TStream>
+  void Create(const std::string &stream_name, typename TStream::StreamInfo info, std::optional<std::string> owner);
 
   /// Deletes an existing stream and all the data that was persisted.
   ///
@@ -122,7 +131,7 @@ class Streams final {
   /// Return current status for all streams.
   /// It might happend that the is_running field is out of date if the one of the streams stops during the invocation of
   /// this function because of an error.
-  std::vector<StreamStatus> GetStreamInfo() const;
+  std::vector<StreamStatus<>> GetStreamInfo() const;
 
   /// Do a dry-run consume from a stream.
   ///
@@ -139,20 +148,37 @@ class Streams final {
                              std::optional<std::chrono::milliseconds> timeout = std::nullopt,
                              std::optional<int64_t> batch_limit = std::nullopt) const;
 
+  /// Return the configuration value passed to memgraph.
+  std::string_view BootstrapServers() const;
+
  private:
-  using StreamsMap = std::unordered_map<std::string, StreamData>;
+  template <Stream TStream>
+  using SynchronizedStreamSource = utils::Synchronized<TStream, utils::WritePrioritizedRWLock>;
+
+  template <Stream TStream>
+  struct StreamData {
+    std::string transformation_name;
+    std::optional<std::string> owner;
+    std::unique_ptr<SynchronizedStreamSource<TStream>> stream_source;
+  };
+
+  using StreamDataVariant = std::variant<StreamData<KafkaStream>, StreamData<PulsarStream>>;
+  using StreamsMap = std::unordered_map<std::string, StreamDataVariant>;
   using SynchronizedStreamsMap = utils::Synchronized<StreamsMap, utils::WritePrioritizedRWLock>;
 
-  static StreamStatus CreateStatus(const std::string &name, const std::string &transformation_name,
-                                   const std::optional<std::string> &owner,
-                                   const integrations::kafka::Consumer &consumer);
+  template <Stream TStream>
+  StreamsMap::iterator CreateConsumer(StreamsMap &map, const std::string &stream_name,
+                                      typename TStream::StreamInfo stream_info, std::optional<std::string> owner);
 
-  StreamsMap::iterator CreateConsumer(StreamsMap &map, const std::string &stream_name, StreamInfo stream_info);
-
-  void Persist(StreamStatus &&status);
+  template <Stream TStream>
+  void Persist(StreamStatus<TStream> &&status) {
+    const std::string stream_name = status.name;
+    if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
+      throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
+    }
+  }
 
   InterpreterContext *interpreter_context_;
-  std::string bootstrap_servers_;
   kvstore::KVStore storage_;
 
   SynchronizedStreamsMap streams_;

src/query/streams.cpp

@@ -1,428 +0,0 @@
-// Copyright 2021 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "query/streams.hpp"
-
-#include <shared_mutex>
-#include <string_view>
-#include <utility>
-
-#include <spdlog/spdlog.h>
-#include <json/json.hpp>
-#include "query/db_accessor.hpp"
-#include "query/discard_value_stream.hpp"
-#include "query/interpreter.hpp"
-#include "query/procedure/mg_procedure_impl.hpp"
-#include "query/procedure/module.hpp"
-#include "query/typed_value.hpp"
-#include "utils/event_counter.hpp"
-#include "utils/memory.hpp"
-#include "utils/on_scope_exit.hpp"
-#include "utils/pmr/string.hpp"
-
-namespace EventCounter {
-extern const Event MessagesConsumed;
-}  // namespace EventCounter
-
-namespace query {
-
-using Consumer = integrations::kafka::Consumer;
-using ConsumerInfo = integrations::kafka::ConsumerInfo;
-using Message = integrations::kafka::Message;
-namespace {
-constexpr auto kExpectedTransformationResultSize = 2;
-const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()};
-const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()};
-const std::map<std::string, storage::PropertyValue> empty_parameters{};
-
-auto GetStream(auto &map, const std::string &stream_name) {
-  if (auto it = map.find(stream_name); it != map.end()) {
-    return it;
-  }
-  throw StreamsException("Couldn't find stream '{}'", stream_name);
-}
-
-void CallCustomTransformation(const std::string &transformation_name, const std::vector<Message> &messages,
-                              mgp_result &result, storage::Storage::Accessor &storage_accessor,
-                              utils::MemoryResource &memory_resource, const std::string &stream_name) {
-  DbAccessor db_accessor{&storage_accessor};
-  {
-    auto maybe_transformation =
-        procedure::FindTransformation(procedure::gModuleRegistry, transformation_name, utils::NewDeleteResource());
-
-    if (!maybe_transformation) {
-      throw StreamsException("Couldn't find transformation {} for stream '{}'", transformation_name, stream_name);
-    };
-    const auto &trans = *maybe_transformation->second;
-    mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}};
-    std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages),
-                   [](const integrations::kafka::Message &message) { return mgp_message{&message}; });
-    mgp_graph graph{&db_accessor, storage::View::OLD, nullptr};
-    mgp_memory memory{&memory_resource};
-    result.rows.clear();
-    result.error_msg.reset();
-    result.signature = &trans.results;
-
-    MG_ASSERT(result.signature->size() == kExpectedTransformationResultSize);
-    MG_ASSERT(result.signature->contains(query_param_name));
-    MG_ASSERT(result.signature->contains(params_param_name));
-
-    spdlog::trace("Calling transformation in stream '{}'", stream_name);
-    trans.cb(&mgp_messages, &graph, &result, &memory);
-  }
-  if (result.error_msg.has_value()) {
-    throw StreamsException(result.error_msg->c_str());
-  }
-}
-
-std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformationResult(
-    utils::pmr::map<utils::pmr::string, TypedValue> &&values, const std::string_view transformation_name,
-    const std::string_view stream_name) {
-  if (values.size() != kExpectedTransformationResultSize) {
-    throw StreamsException(
-        "Transformation '{}' in stream '{}' did not yield all fields (query, parameters) as required.",
-        transformation_name, stream_name);
-  }
-
-  auto get_value = [&](const utils::pmr::string &field_name) mutable -> TypedValue & {
-    auto it = values.find(field_name);
-    if (it == values.end()) {
-      throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.",
-                             transformation_name, stream_name, field_name};
-    };
-    return it->second;
-  };
-
-  auto &query_value = get_value(query_param_name);
-  MG_ASSERT(query_value.IsString());
-  auto &params_value = get_value(params_param_name);
-  MG_ASSERT(params_value.IsNull() || params_value.IsMap());
-  return {std::move(query_value), std::move(params_value)};
-}
-}  // namespace
-
-// nlohmann::json doesn't support string_view access yet
-const std::string kStreamName{"name"};
-const std::string kTopicsKey{"topics"};
-const std::string kConsumerGroupKey{"consumer_group"};
-const std::string kBatchIntervalKey{"batch_interval"};
-const std::string kBatchSizeKey{"batch_size"};
-const std::string kIsRunningKey{"is_running"};
-const std::string kTransformationName{"transformation_name"};
-const std::string kOwner{"owner"};
-
-void to_json(nlohmann::json &data, StreamStatus &&status) {
-  auto &info = status.info;
-  data[kStreamName] = std::move(status.name);
-  data[kTopicsKey] = std::move(info.topics);
-  data[kConsumerGroupKey] = info.consumer_group;
-
-  if (info.batch_interval) {
-    data[kBatchIntervalKey] = info.batch_interval->count();
-  } else {
-    data[kBatchIntervalKey] = nullptr;
-  }
-
-  if (info.batch_size) {
-    data[kBatchSizeKey] = *info.batch_size;
-  } else {
-    data[kBatchSizeKey] = nullptr;
-  }
-
-  data[kIsRunningKey] = status.is_running;
-  data[kTransformationName] = status.info.transformation_name;
-
-  if (info.owner.has_value()) {
-    data[kOwner] = std::move(*info.owner);
-  } else {
-    data[kOwner] = nullptr;
-  }
-}
-
-void from_json(const nlohmann::json &data, StreamStatus &status) {
-  auto &info = status.info;
-  data.at(kStreamName).get_to(status.name);
-  data.at(kTopicsKey).get_to(info.topics);
-  data.at(kConsumerGroupKey).get_to(info.consumer_group);
-
-  if (const auto batch_interval = data.at(kBatchIntervalKey); !batch_interval.is_null()) {
-    using BatchInterval = decltype(info.batch_interval)::value_type;
-    info.batch_interval = BatchInterval{batch_interval.get<BatchInterval::rep>()};
-  } else {
-    info.batch_interval = {};
-  }
-
-  if (const auto batch_size = data.at(kBatchSizeKey); !batch_size.is_null()) {
-    info.batch_size = batch_size.get<decltype(info.batch_size)::value_type>();
-  } else {
-    info.batch_size = {};
-  }
-
-  data.at(kIsRunningKey).get_to(status.is_running);
-  data.at(kTransformationName).get_to(status.info.transformation_name);
-
-  if (const auto &owner = data.at(kOwner); !owner.is_null()) {
-    info.owner = owner.get<decltype(info.owner)::value_type>();
-  } else {
-    info.owner = {};
-  }
-}
-
-Streams::Streams(InterpreterContext *interpreter_context, std::string bootstrap_servers,
-                 std::filesystem::path directory)
-    : interpreter_context_(interpreter_context),
-      bootstrap_servers_(std::move(bootstrap_servers)),
-      storage_(std::move(directory)) {}
-
-void Streams::RestoreStreams() {
-  spdlog::info("Loading streams...");
-  auto locked_streams_map = streams_.Lock();
-  MG_ASSERT(locked_streams_map->empty(), "Cannot restore streams when some streams already exist!");
-
-  for (const auto &[stream_name, stream_data] : storage_) {
-    const auto get_failed_message = [&stream_name = stream_name](const std::string_view message,
-                                                                 const std::string_view nested_message) {
-      return fmt::format("Failed to load stream '{}', because: {} caused by {}", stream_name, message, nested_message);
-    };
-
-    StreamStatus status;
-    try {
-      nlohmann::json::parse(stream_data).get_to(status);
-    } catch (const nlohmann::json::type_error &exception) {
-      spdlog::warn(get_failed_message("invalid type conversion", exception.what()));
-      continue;
-    } catch (const nlohmann::json::out_of_range &exception) {
-      spdlog::warn(get_failed_message("non existing field", exception.what()));
-      continue;
-    }
-    MG_ASSERT(status.name == stream_name, "Expected stream name is '{}', but got '{}'", status.name, stream_name);
-
-    try {
-      auto it = CreateConsumer(*locked_streams_map, stream_name, std::move(status.info));
-      if (status.is_running) {
-        it->second.consumer->Lock()->Start();
-      }
-      spdlog::info("Stream '{}' is loaded", stream_name);
-    } catch (const utils::BasicException &exception) {
-      spdlog::warn(get_failed_message("unexpected error", exception.what()));
-    }
-  }
-}
-
-void Streams::Create(const std::string &stream_name, StreamInfo info) {
-  auto locked_streams = streams_.Lock();
-  auto it = CreateConsumer(*locked_streams, stream_name, std::move(info));
-
-  try {
-    Persist(
-        CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *it->second.consumer->ReadLock()));
-  } catch (...) {
-    locked_streams->erase(it);
-    throw;
-  }
-}
-
-void Streams::Drop(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-
-  auto it = GetStream(*locked_streams, stream_name);
-
-  // streams_ is write locked, which means there is no access to it outside of this function, thus only the Test
-  // function can be executing with the consumer, nothing else.
-  // By acquiring the write lock here for the consumer, we make sure there is
-  // no running Test function for this consumer, therefore it can be erased.
-  it->second.consumer->Lock();
-  locked_streams->erase(it);
-
-  if (!storage_.Delete(stream_name)) {
-    throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
-  }
-
-  // TODO(antaljanosbenjamin) Release the transformation
-}
-
-void Streams::Start(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-  auto it = GetStream(*locked_streams, stream_name);
-
-  auto locked_consumer = it->second.consumer->Lock();
-  locked_consumer->Start();
-
-  Persist(CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *locked_consumer));
-}
-
-void Streams::Stop(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-  auto it = GetStream(*locked_streams, stream_name);
-
-  auto locked_consumer = it->second.consumer->Lock();
-  locked_consumer->Stop();
-
-  Persist(CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *locked_consumer));
-}
-
-void Streams::StartAll() {
-  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
-    auto locked_consumer = stream_data.consumer->Lock();
-    if (!locked_consumer->IsRunning()) {
-      locked_consumer->Start();
-      Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_consumer));
-    }
-  }
-}
-
-void Streams::StopAll() {
-  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
-    auto locked_consumer = stream_data.consumer->Lock();
-    if (locked_consumer->IsRunning()) {
-      locked_consumer->Stop();
-      Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_consumer));
-    }
-  }
-}
-
-std::vector<StreamStatus> Streams::GetStreamInfo() const {
-  std::vector<StreamStatus> result;
-  {
-    for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) {
-      result.emplace_back(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner,
-                                       *stream_data.consumer->ReadLock()));
-    }
-  }
-  return result;
-}
-
-TransformationResult Streams::Check(const std::string &stream_name, std::optional<std::chrono::milliseconds> timeout,
-                                    std::optional<int64_t> batch_limit) const {
-  // This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after that
-  auto [locked_consumer,
-        transformation_name] = [this, &stream_name]() -> std::pair<SynchronizedConsumer::ReadLockedPtr, std::string> {
-    auto locked_streams = streams_.ReadLock();
-    auto it = GetStream(*locked_streams, stream_name);
-    return {it->second.consumer->ReadLock(), it->second.transformation_name};
-  }();
-
-  auto *memory_resource = utils::NewDeleteResource();
-  mgp_result result{nullptr, memory_resource};
-  TransformationResult test_result;
-
-  auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name,
-                            &transformation_name = transformation_name, &result,
-                            &test_result](const std::vector<Message> &messages) mutable {
-    auto accessor = interpreter_context->db->Access();
-    CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
-
-    for (auto &row : result.rows) {
-      auto [query, parameters] = ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
-      std::vector<TypedValue> result_row;
-      result_row.reserve(kExpectedTransformationResultSize);
-      result_row.push_back(std::move(query));
-      result_row.push_back(std::move(parameters));
-
-      test_result.push_back(std::move(result_row));
-    }
-  };
-
-  locked_consumer->Check(timeout, batch_limit, consumer_function);
-
-  return test_result;
-}
-
-StreamStatus Streams::CreateStatus(const std::string &name, const std::string &transformation_name,
-                                   const std::optional<std::string> &owner,
-                                   const integrations::kafka::Consumer &consumer) {
-  const auto &info = consumer.Info();
-  return StreamStatus{name,
-                      StreamInfo{
-                          info.topics,
-                          info.consumer_group,
-                          info.batch_interval,
-                          info.batch_size,
-                          transformation_name,
-                          owner,
-                      },
-                      consumer.IsRunning()};
-}
-
-Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std::string &stream_name,
-                                                      StreamInfo stream_info) {
-  if (map.contains(stream_name)) {
-    throw StreamsException{"Stream already exists with name '{}'", stream_name};
-  }
-
-  auto *memory_resource = utils::NewDeleteResource();
-
-  auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, stream_name,
-                            transformation_name = stream_info.transformation_name, owner = stream_info.owner,
-                            interpreter = std::make_shared<Interpreter>(interpreter_context_),
-                            result = mgp_result{nullptr, memory_resource}](
-                               const std::vector<integrations::kafka::Message> &messages) mutable {
-    auto accessor = interpreter_context->db->Access();
-    EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
-    CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
-
-    DiscardValueResultStream stream;
-
-    spdlog::trace("Start transaction in stream '{}'", stream_name);
-    utils::OnScopeExit cleanup{[&interpreter, &result]() {
-      result.rows.clear();
-      interpreter->Abort();
-    }};
-    interpreter->BeginTransaction();
-
-    for (auto &row : result.rows) {
-      spdlog::trace("Processing row in stream '{}'", stream_name);
-      auto [query_value, params_value] =
-          ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
-      storage::PropertyValue params_prop{params_value};
-
-      std::string query{query_value.ValueString()};
-      spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name);
-      auto prepare_result =
-          interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr);
-      if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges)) {
-        throw StreamsException{
-            "Couldn't execute query '{}' for stream '{}' becuase the owner is not authorized to execute the "
-            "query!",
-            query, stream_name};
-      }
-      interpreter->PullAll(&stream);
-    }
-
-    spdlog::trace("Commit transaction in stream '{}'", stream_name);
-    interpreter->CommitTransaction();
-    result.rows.clear();
-  };
-
-  ConsumerInfo consumer_info{
-      .consumer_name = stream_name,
-      .topics = std::move(stream_info.topics),
-      .consumer_group = std::move(stream_info.consumer_group),
-      .batch_interval = stream_info.batch_interval,
-      .batch_size = stream_info.batch_size,
-  };
-
-  auto insert_result = map.insert_or_assign(
-      stream_name, StreamData{std::move(stream_info.transformation_name), std::move(stream_info.owner),
-                              std::make_unique<SynchronizedConsumer>(bootstrap_servers_, std::move(consumer_info),
-                                                                     std::move(consumer_function))});
-  MG_ASSERT(insert_result.second, "Unexpected error during storing consumer '{}'", stream_name);
-  return insert_result.first;
-}
-
-void Streams::Persist(StreamStatus &&status) {
-  const std::string stream_name = status.name;
-  if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
-    throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
-  }
-}
-
-}  // namespace query

src/utils/license.cpp

@@ -175,13 +175,11 @@ void LicenseChecker::EnableTesting() {
 }
 
 void LicenseChecker::CheckEnvLicense() {
-  // NOLINTNEXTLINE(concurrency-mt-unsafe)
   const char *license_key = std::getenv("MEMGRAPH_ENTERPRISE_LICENSE");
   if (!license_key) {
     return;
   }
 
-  // NOLINTNEXTLINE(concurrency-mt-unsafe)
   const char *organization_name = std::getenv("MEMGRAPH_ORGANIZATION_NAME");
   if (!organization_name) {
     return;

src/utils/variant.hpp

@@ -0,0 +1,22 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+namespace utils {
+template <class... Ts>
+struct Overloaded : Ts... {
+  using Ts::operator()...;
+};
+
+template <class... Ts>
+Overloaded(Ts...) -> Overloaded<Ts...>;
+}  // namespace utils

tests/benchmark/data_structures/ring_buffer.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <benchmark/benchmark.h>
 #include <benchmark/benchmark_api.h>
 #include <iostream>

tests/benchmark/expansion.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <benchmark/benchmark.h>
 #include <benchmark/benchmark_api.h>
 
@@ -37,7 +48,7 @@ class ExpansionBenchFixture : public benchmark::Fixture {
 
     MG_ASSERT(db->CreateIndex(label));
 
-    interpreter_context.emplace(&*db, query::InterpreterConfig{}, data_directory, "non existing bootstrap servers");
+    interpreter_context.emplace(&*db, query::InterpreterConfig{}, data_directory);
     interpreter.emplace(&*interpreter_context);
   }
 

tests/benchmark/query/eval.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <benchmark/benchmark.h>
 
 #include "query/db_accessor.hpp"

tests/benchmark/query/execution.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <random>
 #include <string>
 

tests/benchmark/query/planner.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <benchmark/benchmark_api.h>
 #include <string>
 #include <variant>

tests/benchmark/query/profile.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <memory>
 #include <vector>
 

tests/benchmark/query/stripped.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #define LOG_NO_INFO 1
 
 #include <fstream>

tests/benchmark/rpc.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <optional>
 #include <thread>
 

tests/benchmark/skip_list_common.hpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #pragma once
 
 #include <atomic>

tests/benchmark/skip_list_random.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include "skip_list_common.hpp"
 
 #include "utils/skip_list.hpp"

tests/benchmark/skip_list_real_world.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include "skip_list_common.hpp"
 
 #include "utils/skip_list.hpp"

tests/benchmark/skip_list_same_item.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include "skip_list_common.hpp"
 
 #include "utils/skip_list.hpp"

tests/benchmark/skip_list_vs_stl.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <chrono>
 #include <iostream>
 #include <map>

tests/benchmark/storage_v2_gc.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <iostream>
 
 #include <gflags/gflags.h>

tests/benchmark/storage_v2_property_store.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <chrono>
 #include <iostream>
 #include <map>

tests/concurrent/common.hpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <chrono>
 #include <future>
 #include <iostream>

tests/concurrent/network_common.hpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #pragma once
 
 #include <array>

tests/concurrent/network_read_hang.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #ifndef NDEBUG
 #define NDEBUG
 #endif

tests/concurrent/network_server.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #ifndef NDEBUG
 #define NDEBUG
 #endif

tests/concurrent/network_session_leak.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #ifndef NDEBUG
 #define NDEBUG
 #endif

tests/concurrent/skip_list_insert.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <thread>
 #include <vector>
 

tests/concurrent/skip_list_insert_competitive.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <atomic>
 #include <thread>
 #include <vector>

tests/concurrent/skip_list_mixed.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <atomic>
 #include <chrono>
 #include <random>

tests/concurrent/skip_list_remove.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <thread>
 #include <vector>
 

tests/concurrent/skip_list_remove_competitive.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <atomic>
 #include <thread>
 #include <vector>

tests/concurrent/spin_lock.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <atomic>
 #include <chrono>
 #include <mutex>

tests/concurrent/stack.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <atomic>
 #include <chrono>
 #include <cstring>

tests/concurrent/storage_indices.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <thread>
 #include <unordered_map>
 

tests/concurrent/storage_unique_constraints.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <thread>
 
 #include <gtest/gtest.h>

tests/drivers/python/v4_1/docs_how_to_query.py

@@ -1,6 +1,17 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
 import sys
 from neo4j import GraphDatabase, basic_auth
 

tests/drivers/python/v4_1/max_query_length.py

@@ -1,6 +1,17 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
 from neo4j import GraphDatabase, basic_auth
 
 driver = GraphDatabase.driver("bolt://localhost:7687",

tests/drivers/python/v4_1/transactions.py

@@ -1,6 +1,17 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
 from neo4j import GraphDatabase, basic_auth
 from neo4j.exceptions import ClientError, TransientError
 

tests/e2e/isolation_levels/isolation_levels.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <gflags/gflags.h>
 #include <mgclient.hpp>
 

tests/e2e/memgraph.py

@@ -1,3 +1,14 @@
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
 import copy
 import os
 import subprocess

tests/e2e/memory/memory_control.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <gflags/gflags.h>
 #include <mgclient.hpp>
 

tests/e2e/memory/memory_limit_global_alloc.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <gflags/gflags.h>
 #include <mgclient.hpp>
 

tests/e2e/memory/memory_limit_global_alloc_proc.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <gflags/gflags.h>
 #include <mgclient.hpp>
 #include <algorithm>

tests/e2e/replication/common.hpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <chrono>
 #include <random>
 

tests/e2e/replication/constraints.cpp

@@ -1,3 +1,14 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
 #include <chrono>
 #include <random>
 #include <thread>