Fix tests for unordered configs

This reverts commit 521a085b79.

environment/os/centos-7.sh

@@ -49,6 +49,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/centos-8.sh

@@ -48,6 +48,7 @@ MEMGRAPH_BUILD_DEPS=(
     which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
     sbcl # for custom Lisp C++ preprocessing
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-10.sh

@@ -46,6 +46,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-11.sh

@@ -47,6 +47,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/debian-9.sh

@@ -44,6 +44,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/ubuntu-18.04.sh

@@ -45,6 +45,7 @@ MEMGRAPH_BUILD_DEPS=(
     doxygen graphviz # source documentation generators
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

environment/os/ubuntu-20.04.sh

@@ -46,6 +46,7 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
 )
 
 list() {

libs/.gitignore

@@ -5,3 +5,4 @@
 !CMakeLists.txt
 !__main.cpp
 !jemalloc.cmake
+!pulsar.patch

libs/CMakeLists.txt

@@ -48,7 +48,7 @@ endfunction(import_library)
 # INSTALL_COMMAND arguments.
 function(add_external_project name)
   set(options NO_C_COMPILER)
-  set(one_value_kwargs SOURCE_DIR)
+  set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE)
   set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
       CONFIGURE_COMMAND)
   cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
@@ -56,11 +56,16 @@ function(add_external_project name)
   if (KW_SOURCE_DIR)
     set(source_dir ${KW_SOURCE_DIR})
   endif()
+  set(build_in_source 0)
+  if (KW_BUILD_IN_SOURCE)
+    set(build_in_source ${KW_BUILD_IN_SOURCE})
+  endif()
   if (NOT KW_NO_C_COMPILER)
     set(KW_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} ${KW_CMAKE_ARGS})
   endif()
   ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
                       PREFIX ${source_dir} SOURCE_DIR ${source_dir}
+                      BUILD_IN_SOURCE ${build_in_source}
                       CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
                       CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
                                  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -169,9 +174,12 @@ import_external_library(bzip2 STATIC
     INSTALL_COMMAND true)
 
 # Setup zlib
+set(ZLIB_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/zlib)
+set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/libz.a)
+set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
 import_external_library(zlib STATIC
-    ${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a
+    ${ZLIB_LIBRARIES}
-    ${CMAKE_CURRENT_SOURCE_DIR}/zlib
+    ${ZLIB_INCLUDE_DIRS}
     CMAKE_ARGS -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
     BUILD_COMMAND $(MAKE) zlibstatic)
 
@@ -231,6 +239,7 @@ import_external_library(librdkafka STATIC
   CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
              -DRDKAFKA_BUILD_EXAMPLES=OFF
              -DRDKAFKA_BUILD_TESTS=OFF
+             -DENABLE_LZ4_EXT=OFF
              -DCMAKE_INSTALL_LIBDIR=lib
              -DWITH_SSL=ON
              # If we want SASL, we need to install it on build machines
@@ -242,3 +251,36 @@ import_library(librdkafka++ STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include
 )
 target_link_libraries(librdkafka++ INTERFACE librdkafka)
+
+set(PROTOBUF_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/protobuf/lib)
+import_external_library(protobuf STATIC
+  ${PROTOBUF_ROOT}/lib/libprotobuf.a
+  ${PROTOBUF_ROOT}/include
+  BUILD_IN_SOURCE 1
+  CONFIGURE_COMMAND true)
+
+set(BOOST_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/boost/lib)
+
+import_external_library(pulsar STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/pulsar/pulsar-client-cpp/lib/libpulsarwithdeps.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include
+  BUILD_IN_SOURCE 1
+  CONFIGURE_COMMAND cmake pulsar-client-cpp
+                            -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install
+                            -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                            -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+                            -DBUILD_DYNAMIC_LIB=OFF
+                            -DBUILD_STATIC_LIB=ON
+                            -DBUILD_TESTS=OFF
+                            -DLINK_STATIC=ON
+                            -DPROTOC_PATH=${PROTOBUF_ROOT}/bin/protoc
+                            -DBOOST_ROOT=${BOOST_ROOT}
+                            -DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT}
+                            -DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include
+                            -DZLIB_LIBRARIES=${ZLIB_LIBRARIES}
+                            -DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}
+                            -DBUILD_PYTHON_WRAPPER=OFF
+                            -DBUILD_PERF_TOOLS=OFF
+                            -DUSE_LOG4CXX=OFF
+ BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
+add_dependencies(pulsar-proj protobuf zlib)

libs/setup.sh

@@ -14,39 +14,49 @@ clone () {
   local git_repo=$1
   local dir_name=$2
   local checkout_id=$3
-  shift 3
+  local shallow=$4
+  shift 4
   # Clone if there's no repo.
   if [[ ! -d "$dir_name" ]]; then
     echo "Cloning from $git_repo"
     # If the clone fails, it doesn't make sense to continue with the function
     # execution but the whole script should continue executing because we might
     # clone the same repo from a different source.
-    git clone "$git_repo" "$dir_name" || return 1
+
+    if [ "$shallow" = true ]; then
+      git clone --depth 1 --branch "$checkout_id" "$git_repo" "$dir_name" || return 1
+    else
+      git clone "$git_repo" "$dir_name" || return 1
+    fi
   fi
   pushd "$dir_name"
-  # Just fetch new commits from remote repository. Don't merge/pull them in, so
-  # that we don't clobber local modifications.
-  git fetch
   # Check whether we have any local changes which need to be preserved.
   local local_changes=true
   if git diff --no-ext-diff --quiet && git diff --no-ext-diff --cached --quiet; then
     local_changes=false
   fi
-  # Stash regardless of local_changes, so that a user gets a message on stdout.
+
-  git stash
+  if [ "$shallow" = false ]; then
-  # Checkout the primary commit (there's no need to pull/merge).
+    # Stash regardless of local_changes, so that a user gets a message on stdout.
-  # The checkout fail should exit this script immediately because the target
+    git stash
-  # commit is not there and that will most likely create build-time errors.
+    # Just fetch new commits from remote repository. Don't merge/pull them in, so
-  git checkout "$checkout_id" || exit 1
+    # that we don't clobber local modifications.
-  # Apply any optional cherry pick fixes.
+    git fetch
-  while [[ $# -ne 0 ]]; do
+    # Checkout the primary commit (there's no need to pull/merge).
-    local cherry_pick_id=$1
+    # The checkout fail should exit this script immediately because the target
-    shift
+    # commit is not there and that will most likely create build-time errors.
-    # The cherry-pick fail should exit this script immediately because the
+    git checkout "$checkout_id" || exit 1
-    # target commit is not there and that will most likely create build-time
+    # Apply any optional cherry pick fixes.
-    # errors.
+    while [[ $# -ne 0 ]]; do
-    git cherry-pick -n "$cherry_pick_id" || exit 1
+      local cherry_pick_id=$1
-  done
+      shift
+      # The cherry-pick fail should exit this script immediately because the
+      # target commit is not there and that will most likely create build-time
+      # errors.
+      git cherry-pick -n "$cherry_pick_id" || exit 1
+    done
+  fi
+
   # Reapply any local changes.
   if [[ $local_changes == true ]]; then
     git stash pop
@@ -70,12 +80,13 @@ repo_clone_try_double () {
     secondary_url="$2"
     folder_name="$3"
     ref="$4"
+    shallow="${5:-false}"
     echo "Cloning primary from $primary_url secondary from $secondary_url"
     if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
     if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
     if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
     if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
-    clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
+    clone "$primary_url" "$folder_name" "$ref" "$shallow" || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
     echo ""
 }
 
@@ -113,6 +124,9 @@ declare -A primary_urls=(
   ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
   ["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
+  ["protobuf"]="http://$local_cache_host/git/protobuf.git"
+  ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+  ["pulsar"]="https://github.com/apache/pulsar.git"
 )
 
 # The goal of secondary urls is to have links to the "source of truth" of
@@ -140,13 +154,16 @@ declare -A secondary_urls=(
   ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
   ["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
   ["librdkafka"]="https://github.com/edenhill/librdkafka.git"
+  ["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
+  ["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
+  ["pulsar"]="https://github.com/apache/pulsar.git"
 )
 
 # antlr
 file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
 
-antlr4_tag="5e5b6d35b4183fd330102c40947b95c4b5c6abb5" # v4.9.2
+antlr4_tag="4.9.2" # v4.9.2
-repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
+repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
 # fix issue https://github.com/antlr/antlr4/issues/3194 - should update Antlr commit once the PR related to the issue gets merged
@@ -161,20 +178,20 @@ cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
 repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
 
 # fmt
-fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
+fmt_tag="7.1.3" # (2020-11-25)
-repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
+repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag" true
 
 # rapidcheck
 rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
 repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
 
 # google benchmark
-benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
+benchmark_tag="v1.1.0"
-repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
+repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
 
 # google test
-googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
+googletest_tag="release-1.8.0"
-repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
+repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
 
 # google flags
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
@@ -201,19 +218,19 @@ cd ..
 bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
 repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
 
-zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
+zlib_tag="v1.2.11" # v1.2.11.
-repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
+repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag" true
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
 
-rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
+rocksdb_tag="v6.14.6" # (2020-10-14)
-repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
+repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
 # remove shared library from install dependencies
 sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
 
 # mgclient
 mgclient_tag="v1.3.0" # (2021-09-23)
-repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
+repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag" true
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
 
 # pymgclient
@@ -222,10 +239,10 @@ repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient
 
 # mgconsole
 mgconsole_tag="v1.1.0" # (2021-10-07)
-repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag"
+repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
 
-spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
+spdlog_tag="v1.8.2" # (2020-12-01)
-repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
+repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
 
 jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
 repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
@@ -248,4 +265,27 @@ popd
 
 # librdkafka
 librdkafka_tag="v1.7.0" # (2021-05-06)
-repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag"
+repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag" true
+
+# protobuf
+protobuf_tag="v3.12.4"
+repo_clone_try_double "${primary_urls[protobuf]}" "${secondary_urls[protobuf]}" "protobuf" "$protobuf_tag" true
+pushd protobuf
+./autogen.sh && ./configure --prefix=$(pwd)/lib
+popd
+
+# boost
+file_get_try_double  "${primary_urls[boost]}" "${secondary_urls[boost]}"
+tar -xzf boost_1_77_0.tar.gz
+mv boost_1_77_0 boost
+pushd boost
+./bootstrap.sh --prefix=$(pwd)/lib --with-libraries="system,regex"
+./b2 -j$(nproc) install
+popd
+
+#pulsar
+pulsar_tag="v2.8.1"
+repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pulsar" "$pulsar_tag" true
+pushd pulsar
+git apply ../pulsar.patch
+popd

src/integrations/CMakeLists.txt

@@ -1 +1,2 @@
 add_subdirectory(kafka)
+add_subdirectory(pulsar)

src/integrations/constants.hpp

@@ -0,0 +1,22 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+#include <chrono>
+
+namespace integrations {
+constexpr std::chrono::milliseconds kDefaultBatchInterval{100};
+constexpr int64_t kDefaultBatchSize = 1000;
+constexpr int64_t kDefaultCheckBatchLimit = 1;
+constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
+constexpr std::chrono::milliseconds kMinimumInterval{1};
+constexpr int64_t kMinimumSize{1};
+}  // namespace integrations

src/integrations/kafka/consumer.cpp

@@ -19,6 +19,8 @@
 
 #include <librdkafka/rdkafkacpp.h>
 #include <spdlog/spdlog.h>
+
+#include "integrations/constants.hpp"
 #include "integrations/kafka/exceptions.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
@@ -27,13 +29,6 @@
 
 namespace integrations::kafka {
 
-constexpr std::chrono::milliseconds kDefaultBatchInterval{100};
-constexpr int64_t kDefaultBatchSize = 1000;
-constexpr int64_t kDefaultCheckBatchLimit = 1;
-constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
-constexpr std::chrono::milliseconds kMinimumInterval{1};
-constexpr int64_t kMinimumSize{1};
-
 namespace {
 utils::BasicResult<std::string, std::vector<Message>> GetBatch(RdKafka::KafkaConsumer &consumer,
                                                                const ConsumerInfo &info,
@@ -109,14 +104,14 @@ int64_t Message::Timestamp() const {
   return rd_kafka_message_timestamp(c_message, nullptr);
 }
 
-Consumer::Consumer(const std::string &bootstrap_servers, ConsumerInfo info, ConsumerFunction consumer_function)
+Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
     : info_{std::move(info)}, consumer_function_(std::move(consumer_function)) {
   MG_ASSERT(consumer_function_, "Empty consumer function for Kafka consumer");
   // NOLINTNEXTLINE (modernize-use-nullptr)
-  if (info.batch_interval.value_or(kMinimumInterval) < kMinimumInterval) {
+  if (info_.batch_interval.value_or(kMinimumInterval) < kMinimumInterval) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, "Batch interval has to be positive!");
   }
-  if (info.batch_size.value_or(kMinimumSize) < kMinimumSize) {
+  if (info_.batch_size.value_or(kMinimumSize) < kMinimumSize) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, "Batch size has to be positive!");
   }
 
@@ -139,7 +134,7 @@ Consumer::Consumer(const std::string &bootstrap_servers, ConsumerInfo info, Cons
     throw ConsumerFailedToInitializeException(info_.consumer_name, error);
   }
 
-  if (conf->set("bootstrap.servers", bootstrap_servers, error) != RdKafka::Conf::CONF_OK) {
+  if (conf->set("bootstrap.servers", info_.bootstrap_servers, error) != RdKafka::Conf::CONF_OK) {
     throw ConsumerFailedToInitializeException(info_.consumer_name, error);
   }
 

src/integrations/kafka/consumer.hpp

@@ -79,6 +79,7 @@ struct ConsumerInfo {
   std::string consumer_name;
   std::vector<std::string> topics;
   std::string consumer_group;
+  std::string bootstrap_servers;
   std::optional<std::chrono::milliseconds> batch_interval;
   std::optional<int64_t> batch_size;
 };
@@ -93,7 +94,7 @@ class Consumer final : public RdKafka::EventCb {
   ///
   /// @throws ConsumerFailedToInitializeException if the consumer can't connect
   ///         to the Kafka endpoint.
-  Consumer(const std::string &bootstrap_servers, ConsumerInfo info, ConsumerFunction consumer_function);
+  Consumer(ConsumerInfo info, ConsumerFunction consumer_function);
   ~Consumer() override;
 
   Consumer(const Consumer &other) = delete;

src/integrations/pulsar/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(integrations_pulsar_src_files
+    consumer.cpp
+)
+
+find_package(CURL REQUIRED)
+
+add_library(mg-integrations-pulsar STATIC ${integrations_pulsar_src_files})
+target_link_libraries(mg-integrations-pulsar mg-utils pulsar Threads::Threads ${CURL_LIBRARIES})

src/integrations/pulsar/consumer.cpp

@@ -0,0 +1,291 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "integrations/pulsar/consumer.hpp"
+
+#include <fmt/format.h>
+#include <pulsar/Client.h>
+#include <pulsar/InitialPosition.h>
+
+#include <chrono>
+#include <thread>
+
+#include "integrations/constants.hpp"
+#include "integrations/pulsar/exceptions.hpp"
+#include "utils/concepts.hpp"
+#include "utils/logging.hpp"
+#include "utils/on_scope_exit.hpp"
+#include "utils/result.hpp"
+#include "utils/thread.hpp"
+
+namespace integrations::pulsar {
+
+namespace {
+
+template <typename T>
+concept PulsarConsumer = utils::SameAsAnyOf<T, pulsar_client::Consumer, pulsar_client::Reader>;
+
+pulsar_client::Result ConsumeMessage(pulsar_client::Consumer &consumer, pulsar_client::Message &message,
+                                     int remaining_timeout_in_ms) {
+  return consumer.receive(message, remaining_timeout_in_ms);
+}
+
+pulsar_client::Result ConsumeMessage(pulsar_client::Reader &reader, pulsar_client::Message &message,
+                                     int remaining_timeout_in_ms) {
+  return reader.readNext(message, remaining_timeout_in_ms);
+}
+
+template <PulsarConsumer TConsumer>
+utils::BasicResult<std::string, std::vector<Message>> GetBatch(TConsumer &consumer, const ConsumerInfo &info,
+                                                               std::atomic<bool> &is_running) {
+  std::vector<Message> batch{};
+
+  const auto batch_size = info.batch_size.value_or(kDefaultBatchSize);
+  batch.reserve(batch_size);
+
+  auto remaining_timeout_in_ms = info.batch_interval.value_or(kDefaultBatchInterval).count();
+  auto start = std::chrono::steady_clock::now();
+
+  for (int64_t i = 0; remaining_timeout_in_ms > 0 && i < batch_size && is_running.load(); ++i) {
+    pulsar_client::Message message;
+    const auto result = ConsumeMessage(consumer, message, remaining_timeout_in_ms);
+    switch (result) {
+      case pulsar_client::Result::ResultTimeout:
+        return std::move(batch);
+      case pulsar_client::Result::ResultOk:
+        batch.emplace_back(Message{std::move(message)});
+        break;
+      default:
+        spdlog::warn(fmt::format("Unexpected error while consuming message from consumer {}, error: {}",
+                                 info.consumer_name, result));
+        return {pulsar_client::strResult(result)};
+    }
+
+    auto now = std::chrono::steady_clock::now();
+    auto took = std::chrono::duration_cast<std::chrono::milliseconds>(now - start);
+    remaining_timeout_in_ms = remaining_timeout_in_ms - took.count();
+    start = now;
+  }
+
+  return std::move(batch);
+}
+
+class SpdlogLogger : public pulsar_client::Logger {
+  bool isEnabled(Level /*level*/) override { return spdlog::should_log(spdlog::level::trace); }
+
+  void log(Level /*level*/, int /*line*/, const std::string &message) override {
+    spdlog::trace("[Pulsar] {}", message);
+  }
+};
+
+class SpdlogLoggerFactory : public pulsar_client::LoggerFactory {
+  pulsar_client::Logger *getLogger(const std::string & /*file_name*/) override {
+    if (!logger_) {
+      logger_ = std::make_unique<SpdlogLogger>();
+    }
+    return logger_.get();
+  }
+
+ private:
+  std::unique_ptr<SpdlogLogger> logger_;
+};
+
+pulsar_client::Client CreateClient(const std::string &service_url) {
+  pulsar_client::ClientConfiguration conf;
+  conf.setLogger(new SpdlogLoggerFactory);
+  return {service_url, conf};
+}
+}  // namespace
+
+Message::Message(pulsar_client::Message &&message) : message_{std::move(message)} {}
+
+std::span<const char> Message::Payload() const {
+  return {static_cast<const char *>(message_.getData()), message_.getLength()};
+}
+
+Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
+    : info_{std::move(info)},
+      client_{CreateClient(info_.service_url)},
+      consumer_function_{std::move(consumer_function)} {
+  pulsar_client::ConsumerConfiguration config;
+  config.setSubscriptionInitialPosition(pulsar_client::InitialPositionLatest);
+  config.setConsumerType(pulsar_client::ConsumerType::ConsumerExclusive);
+  if (pulsar_client::Result result = client_.subscribe(info_.topics, info_.consumer_name, config, consumer_);
+      result != pulsar_client::ResultOk) {
+    throw ConsumerFailedToInitializeException(info_.consumer_name, pulsar_client::strResult(result));
+  }
+}
+Consumer::~Consumer() {
+  StopIfRunning();
+  consumer_.close();
+  client_.close();
+}
+
+bool Consumer::IsRunning() const { return is_running_; }
+
+const ConsumerInfo &Consumer::Info() const { return info_; }
+
+void Consumer::Start() {
+  if (is_running_) {
+    throw ConsumerRunningException(info_.consumer_name);
+  }
+
+  StartConsuming();
+}
+
+void Consumer::Stop() {
+  if (!is_running_) {
+    throw ConsumerStoppedException(info_.consumer_name);
+  }
+  StopConsuming();
+}
+
+void Consumer::StopIfRunning() {
+  if (is_running_) {
+    StopConsuming();
+  }
+
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
+                     const ConsumerFunction &check_consumer_function) const {
+  // NOLINTNEXTLINE (modernize-use-nullptr)
+  if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Timeout has to be positive!");
+  }
+  if (limit_batches.value_or(kMinimumSize) < kMinimumSize) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Batch limit has to be positive!");
+  }
+  // The implementation of this function is questionable: it is const qualified, though it changes the inner state of
+  // PulsarConsumer. Though it changes the inner state, it saves the current assignment for future Check/Start calls to
+  // restore the current state, so the changes made by this function shouldn't be visible for the users of the class. It
+  // also passes a non const reference of PulsarConsumer to GetBatch function. That means the object is bitwise const
+  // (PulsarConsumer is stored in unique_ptr) and internally mostly synchronized. Mostly, because as Start/Stop requires
+  // exclusive access to consumer, so we don't have to deal with simultaneous calls to those functions. The only concern
+  // in this function is to prevent executing this function on multiple threads simultaneously.
+  if (is_running_.exchange(true)) {
+    throw ConsumerRunningException(info_.consumer_name);
+  }
+
+  utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
+
+  const auto num_of_batches = limit_batches.value_or(kDefaultCheckBatchLimit);
+  const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
+  const auto start = std::chrono::steady_clock::now();
+
+  if (info_.topics.size() != 1) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Check cannot be used for multiple topics");
+  }
+
+  std::vector<std::string> partitions;
+  const auto &topic = info_.topics.front();
+  client_.getPartitionsForTopic(topic, partitions);
+  if (partitions.size() > 1) {
+    throw ConsumerCheckFailedException(info_.consumer_name, "Check cannot be used for topics with multiple partitions");
+  }
+  pulsar_client::Reader reader;
+  client_.createReader(topic, last_message_id_, {}, reader);
+  for (int64_t i = 0; i < num_of_batches;) {
+    const auto now = std::chrono::steady_clock::now();
+    // NOLINTNEXTLINE (modernize-use-nullptr)
+    if (now - start >= timeout_to_use) {
+      throw ConsumerCheckFailedException(info_.consumer_name, "Timeout reached");
+    }
+
+    auto maybe_batch = GetBatch(reader, info_, is_running_);
+
+    if (maybe_batch.HasError()) {
+      throw ConsumerCheckFailedException(info_.consumer_name, maybe_batch.GetError());
+    }
+
+    const auto &batch = maybe_batch.GetValue();
+
+    if (batch.empty()) {
+      continue;
+    }
+    ++i;
+
+    try {
+      check_consumer_function(batch);
+    } catch (const std::exception &e) {
+      spdlog::warn("Pulsar consumer {} check failed with error {}", info_.consumer_name, e.what());
+      throw ConsumerCheckFailedException(info_.consumer_name, e.what());
+    }
+  }
+  reader.close();
+}
+
+void Consumer::StartConsuming() {
+  MG_ASSERT(!is_running_, "Cannot start already running consumer!");
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+
+  is_running_.store(true);
+
+  thread_ = std::thread([this] {
+    constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
+    const auto full_thread_name = "Cons#" + info_.consumer_name;
+
+    utils::ThreadSetName(full_thread_name.substr(0, kMaxThreadNameSize));
+
+    while (is_running_) {
+      auto maybe_batch = GetBatch(consumer_, info_, is_running_);
+
+      if (maybe_batch.HasError()) {
+        spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
+                     maybe_batch.GetError());
+        break;
+      }
+
+      const auto &batch = maybe_batch.GetValue();
+
+      if (batch.empty()) {
+        continue;
+      }
+
+      spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
+
+      try {
+        consumer_function_(batch);
+
+        if (std::any_of(batch.begin(), batch.end(), [&](const auto &message) {
+              if (const auto result = consumer_.acknowledge(message.message_); result != pulsar_client::ResultOk) {
+                spdlog::warn("Acknowledging a message of consumer {} failed: {}", info_.consumer_name, result);
+                return true;
+              }
+              last_message_id_ = message.message_.getMessageId();
+              return false;
+            })) {
+          break;
+        }
+      } catch (const std::exception &e) {
+        spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
+        break;
+      }
+
+      spdlog::info("Pulsar consumer {} finished processing", info_.consumer_name);
+    }
+    is_running_.store(false);
+  });
+}
+
+void Consumer::StopConsuming() {
+  is_running_.store(false);
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+}  // namespace integrations::pulsar

src/integrations/pulsar/consumer.hpp

@@ -0,0 +1,81 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+#include <atomic>
+#include <optional>
+#include <span>
+#include <thread>
+
+#include <pulsar/Client.h>
+
+namespace integrations::pulsar {
+
+namespace pulsar_client = ::pulsar;
+
+class Consumer;
+
+class Message final {
+ public:
+  explicit Message(pulsar_client::Message &&message);
+
+  std::span<const char> Payload() const;
+
+ private:
+  pulsar_client::Message message_;
+
+  friend Consumer;
+};
+
+using ConsumerFunction = std::function<void(const std::vector<Message> &)>;
+
+struct ConsumerInfo {
+  std::optional<int64_t> batch_size;
+  std::optional<std::chrono::milliseconds> batch_interval;
+  std::vector<std::string> topics;
+  std::string consumer_name;
+  std::string service_url;
+};
+
+class Consumer final {
+ public:
+  Consumer(ConsumerInfo info, ConsumerFunction consumer_function);
+  ~Consumer();
+
+  Consumer(const Consumer &) = delete;
+  Consumer(Consumer &&) noexcept = delete;
+  Consumer &operator=(const Consumer &) = delete;
+  Consumer &operator=(Consumer &&) = delete;
+
+  bool IsRunning() const;
+  void Start();
+  void Stop();
+  void StopIfRunning();
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
+             const ConsumerFunction &check_consumer_function) const;
+
+  const ConsumerInfo &Info() const;
+
+ private:
+  void StartConsuming();
+  void StopConsuming();
+
+  ConsumerInfo info_;
+  mutable pulsar_client::Client client_;
+  pulsar_client::Consumer consumer_;
+  ConsumerFunction consumer_function_;
+
+  mutable std::atomic<bool> is_running_{false};
+  pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};
+  std::thread thread_;
+};
+}  // namespace integrations::pulsar

src/integrations/pulsar/exceptions.hpp

@@ -0,0 +1,58 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <string>
+
+#include "utils/exceptions.hpp"
+
+namespace integrations::pulsar {
+class PulsarStreamException : public utils::BasicException {
+  using utils::BasicException::BasicException;
+};
+
+class ConsumerFailedToInitializeException : public PulsarStreamException {
+ public:
+  ConsumerFailedToInitializeException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Failed to initialize Pulsar consumer {} : {}", consumer_name, error) {}
+};
+
+class ConsumerRunningException : public PulsarStreamException {
+ public:
+  explicit ConsumerRunningException(const std::string &consumer_name)
+      : PulsarStreamException("Pulsar consumer {} is already running", consumer_name) {}
+};
+
+class ConsumerStoppedException : public PulsarStreamException {
+ public:
+  explicit ConsumerStoppedException(const std::string &consumer_name)
+      : PulsarStreamException("Pulsar consumer {} is already stopped", consumer_name) {}
+};
+
+class ConsumerCheckFailedException : public PulsarStreamException {
+ public:
+  explicit ConsumerCheckFailedException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Pulsar consumer {} check failed: {}", consumer_name, error) {}
+};
+
+class ConsumerStartFailedException : public PulsarStreamException {
+ public:
+  explicit ConsumerStartFailedException(const std::string &consumer_name, const std::string &error)
+      : PulsarStreamException("Starting Pulsar consumer {} failed: {}", consumer_name, error) {}
+};
+
+class TopicNotFoundException : public PulsarStreamException {
+ public:
+  TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
+      : PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
+};
+}  // namespace integrations::pulsar

src/memgraph.cpp

@@ -1119,11 +1119,11 @@ int main(int argc, char **argv) {
   }
   storage::Storage db(db_config);
 
-  query::InterpreterContext interpreter_context{
+  query::InterpreterContext interpreter_context{&db,
-      &db,
+                                                {.query = {.allow_load_csv = FLAGS_allow_load_csv},
-      {.query = {.allow_load_csv = FLAGS_allow_load_csv}, .execution_timeout_sec = FLAGS_query_execution_timeout_sec},
+                                                 .execution_timeout_sec = FLAGS_query_execution_timeout_sec,
-      FLAGS_data_directory,
+                                                 .default_kafka_bootstrap_servers = FLAGS_kafka_bootstrap_servers},
-      FLAGS_kafka_bootstrap_servers};
+                                                FLAGS_data_directory};
 #ifdef MG_ENTERPRISE
   SessionData session_data{&db, &interpreter_context, &auth, &audit_log};
 #else
@@ -1133,9 +1133,6 @@ int main(int argc, char **argv) {
   query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories);
   query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories();
 
-  // As the Stream transformations are using modules, they have to be restored after the query modules are loaded.
-  interpreter_context.streams.RestoreStreams();
-
   AuthQueryHandler auth_handler(&auth, FLAGS_auth_user_or_role_name_regex);
   AuthChecker auth_checker{&auth};
   interpreter_context.auth = &auth_handler;
@@ -1151,6 +1148,9 @@ int main(int argc, char **argv) {
                                                       interpreter_context.auth_checker);
   }
 
+  // As the Stream transformations are using modules, they have to be restored after the query modules are loaded.
+  interpreter_context.streams.RestoreStreams();
+
   ServerContext context;
   std::string service_name = "Bolt";
   if (!FLAGS_bolt_key_file.empty() && !FLAGS_bolt_cert_file.empty()) {

src/memory/new_delete.cpp

@@ -19,13 +19,12 @@
 #include <cstdlib>
 #endif
 
-#include "utils/likely.hpp"
 #include "utils/memory_tracker.hpp"
 
 namespace {
 void *newImpl(const std::size_t size) {
   auto *ptr = malloc(size);
-  if (LIKELY(ptr != nullptr)) {
+  if (ptr != nullptr) [[likely]] {
     return ptr;
   }
 
@@ -34,7 +33,7 @@ void *newImpl(const std::size_t size) {
 
 void *newImpl(const std::size_t size, const std::align_val_t align) {
   auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
-  if (LIKELY(ptr != nullptr)) {
+  if (ptr != nullptr) [[likely]] {
     return ptr;
   }
 
@@ -47,14 +46,22 @@ void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept
 }
 
 #if USE_JEMALLOC
-void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
+void deleteImpl(void *ptr) noexcept {
+  if (ptr == nullptr) [[unlikely]] {
+    return;
+  }
+  dallocx(ptr, 0);
+}
 
 void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
+  if (ptr == nullptr) [[unlikely]] {
+    return;
+  }
   dallocx(ptr, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
 }
 
 void deleteSized(void *ptr, const std::size_t size) noexcept {
-  if (UNLIKELY(ptr == nullptr)) {
+  if (ptr == nullptr) [[unlikely]] {
     return;
   }
 
@@ -62,7 +69,7 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
 }
 
 void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
-  if (UNLIKELY(ptr == nullptr)) {
+  if (ptr == nullptr) [[unlikely]] {
     return;
   }
 
@@ -81,7 +88,7 @@ void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t
 
 void TrackMemory(std::size_t size) {
 #if USE_JEMALLOC
-  if (LIKELY(size != 0)) {
+  if (size != 0) [[likely]] {
     size = nallocx(size, 0);
   }
 #endif
@@ -90,7 +97,7 @@ void TrackMemory(std::size_t size) {
 
 void TrackMemory(std::size_t size, const std::align_val_t align) {
 #if USE_JEMALLOC
-  if (LIKELY(size != 0)) {
+  if (size != 0) [[likely]] {
     size = nallocx(size, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
   }
 #endif
@@ -120,7 +127,7 @@ bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
 void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
   try {
 #if USE_JEMALLOC
-    if (LIKELY(ptr != nullptr)) {
+    if (ptr != nullptr) [[likely]] {
       utils::total_memory_tracker.Free(sallocx(ptr, 0));
     }
 #else
@@ -138,7 +145,7 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size
 void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
   try {
 #if USE_JEMALLOC
-    if (LIKELY(ptr != nullptr)) {
+    if (ptr != nullptr) [[likely]] {
       utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align)));  // NOLINT(hicpp-signed-bitwise)
     }
 #else
@@ -176,28 +183,28 @@ void *operator new[](const std::size_t size, const std::align_val_t align) {
 }
 
 void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size))) {
+  if (TrackMemoryNoExcept(size)) [[likely]] {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
 void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size))) {
+  if (TrackMemoryNoExcept(size)) [[likely]] {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
 void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+  if (TrackMemoryNoExcept(size, align)) [[likely]] {
     return newNoExcept(size, align);
   }
   return nullptr;
 }
 
 void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
-  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+  if (TrackMemoryNoExcept(size, align)) [[likely]] {
     return newNoExcept(size, align);
   }
   return nullptr;

src/query/CMakeLists.txt

@@ -32,7 +32,9 @@ set(mg_query_sources
     procedure/module.cpp
     procedure/py_module.cpp
     serialization/property_value.cpp
-    streams.cpp
+    stream/streams.cpp
+    stream/sources.cpp
+    stream/common.cpp
     trigger.cpp
     trigger_context.cpp
     typed_value.cpp)
@@ -41,7 +43,7 @@ add_library(mg-query STATIC ${mg_query_sources})
 add_dependencies(mg-query generate_lcp_query)
 target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(mg-query dl cppitertools)
-target_link_libraries(mg-query mg-integrations-kafka mg-storage-v2 mg-utils mg-kvstore mg-memory)
+target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-utils mg-kvstore mg-memory)
 if("${MG_PYTHON_VERSION}" STREQUAL "")
     find_package(Python3 3.5 REQUIRED COMPONENTS Development)
 else()

src/query/config.hpp

@@ -10,6 +10,7 @@
 // licenses/APL.txt.
 
 #pragma once
+#include <string>
 
 namespace query {
 struct InterpreterConfig {
@@ -19,5 +20,7 @@ struct InterpreterConfig {
 
   // The default execution timeout is 10 minutes.
   double execution_timeout_sec{600.0};
+
+  std::string default_kafka_bootstrap_servers;
 };
 }  // namespace query

src/query/frontend/ast/ast.lcp

@@ -2498,25 +2498,48 @@ cpp<#
   (:serialize (:slk))
   (:clone))
 
+(defun clone-variant-topic-names (source destination)
+  #>cpp
+    if (auto *topic_expression = std::get_if<Expression*>(&${source})) {
+      if (*topic_expression == nullptr) {
+        ${destination} = nullptr;
+      } else {
+        ${destination} = (*topic_expression)->Clone(storage);
+      }
+    } else {
+      ${destination} = std::get<std::vector<std::string>>(${source});
+    }
+  cpp<#)
+
 (lcp:define-class stream-query (query)
   ((action "Action" :scope :public)
+   (type "Type" :scope :public)
    (stream_name "std::string" :scope :public)
-   (topic_names "std::vector<std::string>" :scope :public)
+
-   (transform_name "std::string" :scope :public)
-   (consumer_group "std::string" :scope :public)
-   (batch_interval "Expression *" :initval "nullptr" :scope :public
-             :slk-save #'slk-save-ast-pointer
-             :slk-load (slk-load-ast-pointer "Expression"))
-   (batch_size "Expression *" :initval "nullptr" :scope :public
-             :slk-save #'slk-save-ast-pointer
-             :slk-load (slk-load-ast-pointer "Expression"))
    (batch_limit "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression"))
    (timeout "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression"))
+
+   (transform_name "std::string" :scope :public)
+   (batch_interval "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+   (batch_size "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+
+   (topic_names "std::variant<Expression*, std::vector<std::string>>" :initval "nullptr" 
+             :clone #'clone-variant-topic-names
+             :scope :public)
+   (consumer_group "std::string" :scope :public)
    (bootstrap_servers "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+
+   (service_url "Expression *" :initval "nullptr" :scope :public
              :slk-save #'slk-save-ast-pointer
              :slk-load (slk-load-ast-pointer "Expression")))
 
@@ -2524,6 +2547,9 @@ cpp<#
     (lcp:define-enum action
         (create-stream drop-stream start-stream stop-stream start-all-streams stop-all-streams show-streams check-stream)
       (:serialize))
+    (lcp:define-enum type
+        (kafka pulsar)
+      (:serialize))
     #>cpp
     StreamQuery() = default;
 

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -18,6 +18,7 @@
 // of the same name, EOF.
 // This hides the definition of the macro which causes
 // the compilation to fail.
+#include "query/frontend/ast/ast.hpp"
 #include "query/frontend/ast/ast_visitor.hpp"
 #include "query/procedure/module.hpp"
 //////////////////////////////////////////////////////
@@ -39,6 +40,7 @@
 #include "query/exceptions.hpp"
 #include "query/frontend/parsing.hpp"
 #include "query/interpret/awesome_memgraph_functions.hpp"
+#include "query/stream/common.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/logging.hpp"
 #include "utils/string.hpp"
@@ -492,45 +494,272 @@ antlrcpp::Any CypherMainVisitor::visitStreamQuery(MemgraphCypher::StreamQueryCon
 }
 
 antlrcpp::Any CypherMainVisitor::visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) {
+  MG_ASSERT(ctx->children.size() == 1, "CreateStreamQuery should have exactly one child!");
+  auto *stream_query = ctx->children[0]->accept(this).as<StreamQuery *>();
+  query_ = stream_query;
+  return stream_query;
+}
+
+std::vector<std::string> TopicNamesFromSymbols(
+    antlr4::tree::ParseTreeVisitor &visitor,
+    const std::vector<MemgraphCypher::SymbolicNameWithDotsAndMinusContext *> &topic_name_symbols) {
+  MG_ASSERT(!topic_name_symbols.empty());
+  std::vector<std::string> topic_names;
+  topic_names.reserve(topic_names.size());
+  std::transform(topic_name_symbols.begin(), topic_name_symbols.end(), std::back_inserter(topic_names),
+                 [&visitor](auto *topic_name) { return JoinSymbolicNamesWithDotsAndMinus(visitor, *topic_name); });
+  return topic_names;
+}
+
+template <bool required, typename... ValueTypes>
+void MapConfig(auto &memory, const auto &enum_key, auto &destination) {
+  const auto key = static_cast<uint8_t>(enum_key);
+  if (!memory.contains(key)) {
+    if constexpr (required) {
+      throw SemanticException("Config {} is required.", ToString(enum_key));
+    } else {
+      return;
+    }
+  }
+
+  std::visit(
+      [&]<typename T>(T &&value) {
+        using ValueType = std::decay_t<T>;
+        if constexpr (utils::SameAsAnyOf<ValueType, ValueTypes...>) {
+          destination = std::forward<T>(value);
+        } else {
+          LOG_FATAL("Invalid type mapped");
+        }
+      },
+      std::move(memory[key]));
+}
+
+enum class CommonStreamConfigKey : uint8_t { TRANSFORM, BATCH_INTERVAL, BATCH_SIZE, END };
+
+constexpr std::array all_common_stream_config_keys{
+    CommonStreamConfigKey::TRANSFORM, CommonStreamConfigKey::BATCH_INTERVAL, CommonStreamConfigKey::BATCH_SIZE};
+
+std::string_view ToString(const CommonStreamConfigKey key) {
+  switch (key) {
+    case CommonStreamConfigKey::TRANSFORM:
+      return "TRANSFORM";
+    case CommonStreamConfigKey::BATCH_INTERVAL:
+      return "BATCH_INTERVAL";
+    case CommonStreamConfigKey::BATCH_SIZE:
+      return "BATCH_SIZE";
+    case CommonStreamConfigKey::END:
+      LOG_FATAL("Invalid config key used");
+  }
+}
+
+#define CONCAT_HELPER(a, b) a##b
+#define CONCAT(a, b) CONCAT_HELPER(a, b)
+
+#define GENERATE_STREAM_CONFIG_KEY_ENUM(stream, first_config, ...)   \
+  enum class CONCAT(stream, ConfigKey) : uint8_t {                   \
+    first_config = static_cast<uint8_t>(CommonStreamConfigKey::END), \
+    __VA_ARGS__                                                      \
+  };
+
+GENERATE_STREAM_CONFIG_KEY_ENUM(Kafka, TOPICS, CONSUMER_GROUP, BOOTSTRAP_SERVERS);
+
+constexpr std::array all_kafka_config_keys{KafkaConfigKey::TOPICS, KafkaConfigKey::CONSUMER_GROUP,
+                                           KafkaConfigKey::BOOTSTRAP_SERVERS};
+
+std::string_view ToString(const KafkaConfigKey key) {
+  switch (key) {
+    case KafkaConfigKey::TOPICS:
+      return "TOPICS";
+    case KafkaConfigKey::CONSUMER_GROUP:
+      return "CONSUMER_GROUP";
+    case KafkaConfigKey::BOOTSTRAP_SERVERS:
+      return "BOOTSTRAP_SERVERS";
+  }
+}
+
+void MapCommonStreamConfigs(auto &memory, StreamQuery &stream_query) {
+  for (const auto key : all_common_stream_config_keys) {
+    switch (key) {
+      case CommonStreamConfigKey::TRANSFORM:
+        MapConfig<true, std::string>(memory, CommonStreamConfigKey::TRANSFORM, stream_query.transform_name_);
+        break;
+      case CommonStreamConfigKey::BATCH_INTERVAL:
+        MapConfig<false, Expression *>(memory, CommonStreamConfigKey::BATCH_INTERVAL, stream_query.batch_interval_);
+        break;
+      case CommonStreamConfigKey::BATCH_SIZE:
+        MapConfig<false, Expression *>(memory, CommonStreamConfigKey::BATCH_SIZE, stream_query.batch_size_);
+        break;
+      case CommonStreamConfigKey::END:
+        LOG_FATAL("Invalid config key used");
+    }
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) {
   auto *stream_query = storage_->Create<StreamQuery>();
   stream_query->action_ = StreamQuery::Action::CREATE_STREAM;
+  stream_query->type_ = StreamQuery::Type::KAFKA;
   stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as<std::string>();
 
-  auto *topic_names_ctx = ctx->topicNames();
+  for (auto *create_config_ctx : ctx->kafkaCreateStreamConfig()) {
-  MG_ASSERT(topic_names_ctx != nullptr);
+    create_config_ctx->accept(this);
-  auto topic_names = topic_names_ctx->symbolicNameWithDotsAndMinus();
+  }
-  MG_ASSERT(!topic_names.empty());
-  stream_query->topic_names_.reserve(topic_names.size());
-  std::transform(topic_names.begin(), topic_names.end(), std::back_inserter(stream_query->topic_names_),
-                 [this](auto *topic_name) { return JoinSymbolicNamesWithDotsAndMinus(*this, *topic_name); });
 
-  stream_query->transform_name_ = JoinSymbolicNames(this, ctx->transformationName->symbolicName());
+  for (const auto key : all_kafka_config_keys) {
+    switch (key) {
+      case KafkaConfigKey::TOPICS:
+        MapConfig<true, std::vector<std::string>>(memory_, KafkaConfigKey::TOPICS, stream_query->topic_names_);
+        break;
+      case KafkaConfigKey::CONSUMER_GROUP:
+        MapConfig<false, std::string>(memory_, KafkaConfigKey::CONSUMER_GROUP, stream_query->consumer_group_);
+        break;
+      case KafkaConfigKey::BOOTSTRAP_SERVERS:
+        MapConfig<false, Expression *>(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS, stream_query->bootstrap_servers_);
+        break;
+    }
+  }
+
+  MapCommonStreamConfigs(memory_, *stream_query);
+
+  return stream_query;
+}
+
+void ThrowIfExists(auto &map, const auto &enum_key) {
+  const auto key = static_cast<uint8_t>(enum_key);
+  if (map.contains(key)) {
+    throw SemanticException("{} defined multiple times in the query", ToString(enum_key));
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) {
+  if (ctx->commonCreateStreamConfig()) {
+    return ctx->commonCreateStreamConfig()->accept(this);
+  }
+
+  if (ctx->TOPICS()) {
+    ThrowIfExists(memory_, KafkaConfigKey::TOPICS);
+    auto *topic_names_ctx = ctx->topicNames();
+    MG_ASSERT(topic_names_ctx != nullptr);
+    const auto topic_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
+    memory_[topic_key] = TopicNamesFromSymbols(*this, topic_names_ctx->symbolicNameWithDotsAndMinus());
+    return {};
+  }
 
   if (ctx->CONSUMER_GROUP()) {
-    stream_query->consumer_group_ = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
+    ThrowIfExists(memory_, KafkaConfigKey::CONSUMER_GROUP);
+    const auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
+    memory_[consumer_group_key] = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
+    return {};
+  }
+
+  MG_ASSERT(ctx->BOOTSTRAP_SERVERS());
+  ThrowIfExists(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS);
+  if (!ctx->bootstrapServers->StringLiteral()) {
+    throw SemanticException("Bootstrap servers should be a string!");
+  }
+
+  const auto bootstrap_servers_key = static_cast<uint8_t>(KafkaConfigKey::BOOTSTRAP_SERVERS);
+  memory_[bootstrap_servers_key] = ctx->bootstrapServers->accept(this).as<Expression *>();
+  return {};
+}  // namespace query::frontend
+
+GENERATE_STREAM_CONFIG_KEY_ENUM(Pulsar, TOPICS, SERVICE_URL);
+
+constexpr std::array all_pulsar_config_keys{PulsarConfigKey::TOPICS, PulsarConfigKey::SERVICE_URL};
+
+std::string_view ToString(const PulsarConfigKey key) {
+  switch (key) {
+    case PulsarConfigKey::TOPICS:
+      return "TOPICS";
+    case PulsarConfigKey::SERVICE_URL:
+      return "SERVICE_URL";
+  }
+}
+
+antlrcpp::Any CypherMainVisitor::visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) {
+  auto *stream_query = storage_->Create<StreamQuery>();
+  stream_query->action_ = StreamQuery::Action::CREATE_STREAM;
+  stream_query->type_ = StreamQuery::Type::PULSAR;
+  stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as<std::string>();
+
+  for (auto *create_config_ctx : ctx->pulsarCreateStreamConfig()) {
+    create_config_ctx->accept(this);
+  }
+
+  for (const auto key : all_pulsar_config_keys) {
+    switch (key) {
+      case PulsarConfigKey::TOPICS:
+        MapConfig<true, std::vector<std::string>, Expression *>(memory_, PulsarConfigKey::TOPICS,
+                                                                stream_query->topic_names_);
+        break;
+      case PulsarConfigKey::SERVICE_URL:
+        MapConfig<true, Expression *>(memory_, PulsarConfigKey::SERVICE_URL, stream_query->service_url_);
+        break;
+    }
+  }
+
+  MapCommonStreamConfigs(memory_, *stream_query);
+
+  return stream_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) {
+  if (ctx->commonCreateStreamConfig()) {
+    return ctx->commonCreateStreamConfig()->accept(this);
+  }
+
+  if (ctx->TOPICS()) {
+    ThrowIfExists(memory_, PulsarConfigKey::TOPICS);
+    auto *pulsar_topic_names_ctx = ctx->pulsarTopicNames();
+    MG_ASSERT(pulsar_topic_names_ctx != nullptr);
+    const auto topics_key = static_cast<uint8_t>(PulsarConfigKey::TOPICS);
+    if (auto *topic_names_ctx = pulsar_topic_names_ctx->topicNames()) {
+      memory_[topics_key] = TopicNamesFromSymbols(*this, topic_names_ctx->symbolicNameWithDotsAndMinus());
+    } else {
+      if (!pulsar_topic_names_ctx->literal()->StringLiteral()) {
+        throw SemanticException("Topic names should be defined in a string");
+      }
+      memory_[topics_key] = pulsar_topic_names_ctx->accept(this).as<Expression *>();
+    }
+    return {};
+  }
+
+  MG_ASSERT(ctx->SERVICE_URL());
+  ThrowIfExists(memory_, PulsarConfigKey::SERVICE_URL);
+  if (!ctx->serviceUrl->StringLiteral()) {
+    throw SemanticException("Service url should be a string!");
+  }
+  const auto service_url_key = static_cast<uint8_t>(PulsarConfigKey::SERVICE_URL);
+  memory_[service_url_key] = ctx->serviceUrl->accept(this).as<Expression *>();
+  return {};
+}
+
+antlrcpp::Any CypherMainVisitor::visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) {
+  if (ctx->TRANSFORM()) {
+    ThrowIfExists(memory_, CommonStreamConfigKey::TRANSFORM);
+    const auto transform_key = static_cast<uint8_t>(CommonStreamConfigKey::TRANSFORM);
+    memory_[transform_key] = JoinSymbolicNames(this, ctx->transformationName->symbolicName());
+    return {};
   }
 
   if (ctx->BATCH_INTERVAL()) {
+    ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_INTERVAL);
     if (!ctx->batchInterval->numberLiteral() || !ctx->batchInterval->numberLiteral()->integerLiteral()) {
       throw SemanticException("Batch interval should be an integer literal!");
     }
-    stream_query->batch_interval_ = ctx->batchInterval->accept(this);
+    const auto batch_interval_key = static_cast<uint8_t>(CommonStreamConfigKey::BATCH_INTERVAL);
+    memory_[batch_interval_key] = ctx->batchInterval->accept(this).as<Expression *>();
+    return {};
   }
 
-  if (ctx->BATCH_SIZE()) {
+  MG_ASSERT(ctx->BATCH_SIZE());
-    if (!ctx->batchSize->numberLiteral() || !ctx->batchSize->numberLiteral()->integerLiteral()) {
+  ThrowIfExists(memory_, CommonStreamConfigKey::BATCH_SIZE);
-      throw SemanticException("Batch size should be an integer literal!");
+  if (!ctx->batchSize->numberLiteral() || !ctx->batchSize->numberLiteral()->integerLiteral()) {
-    }
+    throw SemanticException("Batch size should be an integer literal!");
-    stream_query->batch_size_ = ctx->batchSize->accept(this);
   }
-  if (ctx->BOOTSTRAP_SERVERS()) {
+  const auto batch_size_key = static_cast<uint8_t>(CommonStreamConfigKey::BATCH_SIZE);
-    if (!ctx->bootstrapServers->StringLiteral()) {
+  memory_[batch_size_key] = ctx->batchSize->accept(this).as<Expression *>();
-      throw SemanticException("Bootstrap servers should be a string!");
+  return {};
-    }
-    stream_query->bootstrap_servers_ = ctx->bootstrapServers->accept(this);
-  }
-
-  return stream_query;
 }
 
 antlrcpp::Any CypherMainVisitor::visitDropStream(MemgraphCypher::DropStreamContext *ctx) {

src/query/frontend/ast/cypher_main_visitor.hpp

@@ -269,6 +269,31 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
    */
   antlrcpp::Any visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) override;
 
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitKafkaCreateStreamConfig(MemgraphCypher::KafkaCreateStreamConfigContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitPulsarCreateStreamConfig(MemgraphCypher::PulsarCreateStreamConfigContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitPulsarCreateStream(MemgraphCypher::PulsarCreateStreamContext *ctx) override;
+
+  /**
+   * @return StreamQuery*
+   */
+  antlrcpp::Any visitCommonCreateStreamConfig(MemgraphCypher::CommonCreateStreamConfigContext *ctx) override;
+
   /**
    * @return StreamQuery*
    */
@@ -824,6 +849,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
   ParsingContext context_;
   AstStorage *storage_;
 
+  std::unordered_map<uint8_t, std::variant<Expression *, std::string, std::vector<std::string>>> memory_;
   // Set of identifiers from queries.
   std::unordered_set<std::string> users_identifiers;
   // Identifiers that user didn't name.

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -297,13 +297,29 @@ symbolicNameWithDotsAndMinus: symbolicNameWithMinus ( DOT symbolicNameWithMinus
 
 topicNames : symbolicNameWithDotsAndMinus ( COMMA symbolicNameWithDotsAndMinus )* ;
 
-createStream : CREATE STREAM streamName
+commonCreateStreamConfig : TRANSFORM transformationName=procedureName
-               TOPICS topicNames
+                         | BATCH_INTERVAL batchInterval=literal
-               TRANSFORM transformationName=procedureName
+                         | BATCH_SIZE batchSize=literal
-               ( CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus ) ?
+                         ;
-               ( BATCH_INTERVAL batchInterval=literal ) ?
+
-               ( BATCH_SIZE batchSize=literal ) ?
+createStream : kafkaCreateStream | pulsarCreateStream ;
-               ( BOOTSTRAP_SERVERS bootstrapServers=literal) ? ;
+
+kafkaCreateStreamConfig : TOPICS topicNames
+                        | CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus
+                        | BOOTSTRAP_SERVERS bootstrapServers=literal
+                        | commonCreateStreamConfig
+                        ;
+
+kafkaCreateStream : CREATE KAFKA STREAM streamName ( kafkaCreateStreamConfig ) * ;
+
+pulsarTopicNames : topicNames | literal ;
+
+pulsarCreateStreamConfig : TOPICS pulsarTopicNames
+                         | SERVICE_URL serviceUrl=literal
+                         | commonCreateStreamConfig
+                         ;
+
+pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig ) * ;
 
 dropStream : DROP STREAM streamName ;
 

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -62,6 +62,7 @@ HEADER              : H E A D E R ;
 IDENTIFIED          : I D E N T I F I E D ;
 IGNORE              : I G N O R E ;
 ISOLATION           : I S O L A T I O N ;
+KAFKA               : K A F K A ;
 LEVEL               : L E V E L ;
 LOAD                : L O A D ;
 LOCK                : L O C K ;
@@ -72,6 +73,7 @@ NO                  : N O ;
 PASSWORD            : P A S S W O R D ;
 PORT                : P O R T ;
 PRIVILEGES          : P R I V I L E G E S ;
+PULSAR              : P U L S A R ;
 READ                : R E A D ;
 READ_FILE           : R E A D UNDERSCORE F I L E ;
 REGISTER            : R E G I S T E R ;
@@ -82,6 +84,7 @@ REVOKE              : R E V O K E ;
 ROLE                : R O L E ;
 ROLES               : R O L E S ;
 QUOTE               : Q U O T E ;
+SERVICE_URL         : S E R V I C E UNDERSCORE U R L ;
 SESSION             : S E S S I O N ;
 SETTING             : S E T T I N G ;
 SETTINGS            : S E T T I N G S ;

src/query/frontend/stripped_lexer_constants.hpp

@@ -199,7 +199,10 @@ const trie::Trie kKeywords = {"union",
                               "check",
                               "setting",
                               "settings",
-                              "bootstrap_servers"};
+                              "bootstrap_servers",
+                              "kafka",
+                              "pulsar",
+                              "service_url"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

src/query/interpreter.cpp

@@ -34,7 +34,7 @@
 #include "query/plan/planner.hpp"
 #include "query/plan/profile.hpp"
 #include "query/plan/vertex_count_cache.hpp"
-#include "query/streams.hpp"
+#include "query/stream/common.hpp"
 #include "query/trigger.hpp"
 #include "query/typed_value.hpp"
 #include "storage/v2/property_value.hpp"
@@ -52,6 +52,7 @@
 #include "utils/settings.hpp"
 #include "utils/string.hpp"
 #include "utils/tsc.hpp"
+#include "utils/variant.hpp"
 
 namespace EventCounter {
 extern Event ReadQuery;
@@ -86,7 +87,8 @@ void UpdateTypeCount(const plan::ReadWriteTypeChecker::RWType type) {
 
 struct Callback {
   std::vector<std::string> header;
-  std::function<std::vector<std::vector<TypedValue>>()> fn;
+  using CallbackFunction = std::function<std::vector<std::vector<TypedValue>>()>;
+  CallbackFunction fn;
   bool should_abort_query{false};
 };
 
@@ -511,6 +513,76 @@ std::optional<std::string> StringPointerToOptional(const std::string *str) {
   return str == nullptr ? std::nullopt : std::make_optional(*str);
 }
 
+CommonStreamInfo GetCommonStreamInfo(StreamQuery *stream_query, ExpressionEvaluator &evaluator) {
+  return {.batch_interval = GetOptionalValue<std::chrono::milliseconds>(stream_query->batch_interval_, evaluator),
+          .batch_size = GetOptionalValue<int64_t>(stream_query->batch_size_, evaluator),
+          .transformation_name = stream_query->transform_name_};
+}
+
+std::vector<std::string> EvaluateTopicNames(ExpressionEvaluator &evaluator,
+                                            std::variant<Expression *, std::vector<std::string>> topic_variant) {
+  return std::visit(utils::Overloaded{[&](Expression *expression) {
+                                        auto topic_names = expression->Accept(evaluator);
+                                        MG_ASSERT(topic_names.IsString());
+                                        return utils::Split(topic_names.ValueString(), ",");
+                                      },
+                                      [&](std::vector<std::string> topic_names) { return topic_names; }},
+                    std::move(topic_variant));
+}
+
+Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator,
+                                                  InterpreterContext *interpreter_context,
+                                                  const std::string *username) {
+  constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
+  std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup
+                                                                   : stream_query->consumer_group_};
+
+  auto bootstrap = GetOptionalStringValue(stream_query->bootstrap_servers_, evaluator);
+  if (bootstrap && bootstrap->empty()) {
+    throw SemanticException("Bootstrap servers must not be an empty string!");
+  }
+  auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
+
+  return [interpreter_context, stream_name = stream_query->stream_name_,
+          topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
+          consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info),
+          bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username)]() mutable {
+    std::string bootstrap = bootstrap_servers
+                                ? std::move(*bootstrap_servers)
+                                : std::string{interpreter_context->config.default_kafka_bootstrap_servers};
+    interpreter_context->streams.Create<query::KafkaStream>(stream_name,
+                                                            {.common_info = std::move(common_stream_info),
+                                                             .topics = std::move(topic_names),
+                                                             .consumer_group = std::move(consumer_group),
+                                                             .bootstrap_servers = std::move(bootstrap)},
+                                                            std::move(owner));
+
+    return std::vector<std::vector<TypedValue>>{};
+  };
+}
+
+Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator,
+                                                   InterpreterContext *interpreter_context,
+                                                   const std::string *username) {
+  auto service_url = GetOptionalStringValue(stream_query->service_url_, evaluator);
+  if (service_url && service_url->empty()) {
+    throw SemanticException("Service URL must not be an empty string!");
+  }
+  auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
+  return [interpreter_context, stream_name = stream_query->stream_name_,
+          topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
+          common_stream_info = std::move(common_stream_info), service_url = std::move(service_url),
+          owner = StringPointerToOptional(username)]() mutable {
+    interpreter_context->streams.Create<query::PulsarStream>(stream_name,
+                                                             {.common_info = std::move(common_stream_info),
+                                                              .topics = std::move(topic_names),
+                                                              .service_url = std::move(*service_url)},
+                                                             std::move(owner));
+
+    return std::vector<std::vector<TypedValue>>{};
+  };
+}
+
 Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &parameters,
                            InterpreterContext *interpreter_context, DbAccessor *db_accessor,
                            const std::string *username) {
@@ -527,32 +599,15 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
   switch (stream_query->action_) {
     case StreamQuery::Action::CREATE_STREAM: {
       EventCounter::IncrementCounter(EventCounter::StreamsCreated);
-      constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
+      switch (stream_query->type_) {
-      std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup
+        case StreamQuery::Type::KAFKA:
-                                                                       : stream_query->consumer_group_};
+          callback.fn = GetKafkaCreateCallback(stream_query, evaluator, interpreter_context, username);
-
+          break;
-      auto bootstrap = GetOptionalStringValue(stream_query->bootstrap_servers_, evaluator);
+        case StreamQuery::Type::PULSAR:
-      if (bootstrap && bootstrap->empty()) {
+          callback.fn = GetPulsarCreateCallback(stream_query, evaluator, interpreter_context, username);
-        throw SemanticException("Bootstrap servers must not be an empty string!");
+          break;
       }
-      callback.fn = [interpreter_context, stream_name = stream_query->stream_name_,
+
-                     topic_names = stream_query->topic_names_, consumer_group = std::move(consumer_group),
-                     batch_interval =
-                         GetOptionalValue<std::chrono::milliseconds>(stream_query->batch_interval_, evaluator),
-                     batch_size = GetOptionalValue<int64_t>(stream_query->batch_size_, evaluator),
-                     transformation_name = stream_query->transform_name_, bootstrap_servers = std::move(bootstrap),
-                     owner = StringPointerToOptional(username)]() mutable {
-        std::string bootstrap = bootstrap_servers ? std::move(*bootstrap_servers) : "";
-        interpreter_context->streams.Create(stream_name,
-                                            query::StreamInfo{.topics = std::move(topic_names),
-                                                              .consumer_group = std::move(consumer_group),
-                                                              .batch_interval = batch_interval,
-                                                              .batch_size = batch_size,
-                                                              .transformation_name = std::move(transformation_name),
-                                                              .owner = std::move(owner),
-                                                              .bootstrap_servers = std::move(bootstrap)});
-        return std::vector<std::vector<TypedValue>>{};
-      };
       return callback;
     }
     case StreamQuery::Action::START_STREAM: {
@@ -591,28 +646,12 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
       return callback;
     }
     case StreamQuery::Action::SHOW_STREAMS: {
-      callback.header = {"name",           "topics",
+      callback.header = {"name", "batch_interval", "batch_size", "transformation_name", "owner", "is running"};
-                         "consumer_group", "batch_interval",
-                         "batch_size",     "transformation_name",
-                         "owner",          "bootstrap_servers",
-                         "is running"};
       callback.fn = [interpreter_context]() {
         auto streams_status = interpreter_context->streams.GetStreamInfo();
         std::vector<std::vector<TypedValue>> results;
         results.reserve(streams_status.size());
-        auto topics_as_typed_topics = [](const auto &topics) {
+        auto stream_info_as_typed_stream_info_emplace_in = [](auto &typed_status, const auto &stream_info) {
-          std::vector<TypedValue> typed_topics;
-          typed_topics.reserve(topics.size());
-          for (const auto &elem : topics) {
-            typed_topics.emplace_back(elem);
-          }
-          return typed_topics;
-        };
-
-        auto stream_info_as_typed_stream_info_emplace_in = [topics_as_typed_topics, interpreter_context](
-                                                               auto &typed_status, const auto &stream_info) {
-          typed_status.emplace_back(topics_as_typed_topics(stream_info.topics));
-          typed_status.emplace_back(stream_info.consumer_group);
           if (stream_info.batch_interval.has_value()) {
             typed_status.emplace_back(stream_info.batch_interval->count());
           } else {
@@ -624,16 +663,6 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
             typed_status.emplace_back();
           }
           typed_status.emplace_back(stream_info.transformation_name);
-          if (stream_info.owner.has_value()) {
-            typed_status.emplace_back(*stream_info.owner);
-          } else {
-            typed_status.emplace_back();
-          }
-          if (stream_info.bootstrap_servers.empty()) {
-            typed_status.emplace_back(interpreter_context->streams.BootstrapServers());
-          } else {
-            typed_status.emplace_back(stream_info.bootstrap_servers);
-          }
         };
 
         for (const auto &status : streams_status) {
@@ -641,6 +670,11 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
           typed_status.reserve(8);
           typed_status.emplace_back(status.name);
           stream_info_as_typed_stream_info_emplace_in(typed_status, status.info);
+          if (status.owner.has_value()) {
+            typed_status.emplace_back(*status.owner);
+          } else {
+            typed_status.emplace_back();
+          }
           typed_status.emplace_back(status.is_running);
           results.push_back(std::move(typed_status));
         }
@@ -897,11 +931,8 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
 }  // namespace
 
 InterpreterContext::InterpreterContext(storage::Storage *db, const InterpreterConfig config,
-                                       const std::filesystem::path &data_directory, std::string kafka_bootstrap_servers)
+                                       const std::filesystem::path &data_directory)
-    : db(db),
+    : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory / "streams"} {}
-      trigger_store(data_directory / "triggers"),
-      config(config),
-      streams{this, std::move(kafka_bootstrap_servers), data_directory / "streams"} {}
 
 Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
   MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");

src/query/interpreter.hpp

@@ -26,7 +26,7 @@
 #include "query/plan/operator.hpp"
 #include "query/plan/read_write_type_checker.hpp"
 #include "query/stream.hpp"
-#include "query/streams.hpp"
+#include "query/stream/streams.hpp"
 #include "query/trigger.hpp"
 #include "query/typed_value.hpp"
 #include "storage/v2/isolation_level.hpp"
@@ -164,7 +164,7 @@ struct PreparedQuery {
  */
 struct InterpreterContext {
   explicit InterpreterContext(storage::Storage *db, InterpreterConfig config,
-                              const std::filesystem::path &data_directory, std::string kafka_bootstrap_servers);
+                              const std::filesystem::path &data_directory);
 
   storage::Storage *db;
 

src/query/procedure/mg_procedure_impl.cpp

@@ -2493,27 +2493,109 @@ bool IsValidIdentifierName(const char *name) {
 }  // namespace query::procedure
 
 mgp_error mgp_message_payload(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->Payload().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Payload().data();
+              } else if constexpr (std::same_as<MessageType, mgp_message::PulsarMessage>) {
+                return msg.Payload().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_payload_size(mgp_message *message, size_t *result) {
-  return WrapExceptions([message] { return message->msg->Payload().size(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> size_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Payload().size();
+              } else if constexpr (std::same_as<MessageType, mgp_message::PulsarMessage>) {
+                return msg.Payload().size();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_topic_name(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->TopicName().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->TopicName().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_key(mgp_message *message, const char **result) {
-  return WrapExceptions([message] { return message->msg->Key().data(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> const char * {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Key().data();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_key_size(mgp_message *message, size_t *result) {
-  return WrapExceptions([message] { return message->msg->Key().size(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> size_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Key().size();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_message_timestamp(mgp_message *message, int64_t *result) {
-  return WrapExceptions([message] { return message->msg->Timestamp(); }, result);
+  return WrapExceptions(
+      [message] {
+        return std::visit(
+            []<typename T>(T &&msg) -> int64_t {
+              using MessageType = std::decay_t<T>;
+              if constexpr (std::same_as<MessageType, mgp_message::KafkaMessage>) {
+                return msg->Timestamp();
+              } else {
+                throw std::invalid_argument("Invalid source type");
+              }
+            },
+            message->msg);
+      },
+      result);
 }
 
 mgp_error mgp_messages_size(mgp_messages *messages, size_t *result) {

src/query/procedure/mg_procedure_impl.hpp

@@ -20,6 +20,7 @@
 #include <ostream>
 
 #include "integrations/kafka/consumer.hpp"
+#include "integrations/pulsar/consumer.hpp"
 #include "query/context.hpp"
 #include "query/db_accessor.hpp"
 #include "query/procedure/cypher_type_ptr.hpp"
@@ -790,7 +791,12 @@ bool IsValidIdentifierName(const char *name);
 }  // namespace query::procedure
 
 struct mgp_message {
-  const integrations::kafka::Message *msg;
+  explicit mgp_message(const integrations::kafka::Message &message) : msg{&message} {}
+  explicit mgp_message(const integrations::pulsar::Message &message) : msg{message} {}
+
+  using KafkaMessage = const integrations::kafka::Message *;
+  using PulsarMessage = integrations::pulsar::Message;
+  std::variant<KafkaMessage, PulsarMessage> msg;
 };
 
 struct mgp_messages {

src/query/stream/common.cpp

@@ -0,0 +1,55 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/common.hpp"
+
+#include <json/json.hpp>
+
+namespace query {
+namespace {
+const std::string kBatchIntervalKey{"batch_interval"};
+const std::string kBatchSizeKey{"batch_size"};
+const std::string kTransformationName{"transformation_name"};
+}  // namespace
+
+void to_json(nlohmann::json &data, CommonStreamInfo &&common_info) {
+  if (common_info.batch_interval) {
+    data[kBatchIntervalKey] = common_info.batch_interval->count();
+  } else {
+    data[kBatchIntervalKey] = nullptr;
+  }
+
+  if (common_info.batch_size) {
+    data[kBatchSizeKey] = *common_info.batch_size;
+  } else {
+    data[kBatchSizeKey] = nullptr;
+  }
+
+  data[kTransformationName] = common_info.transformation_name;
+}
+
+void from_json(const nlohmann::json &data, CommonStreamInfo &common_info) {
+  if (const auto batch_interval = data.at(kBatchIntervalKey); !batch_interval.is_null()) {
+    using BatchInterval = typename decltype(common_info.batch_interval)::value_type;
+    common_info.batch_interval = BatchInterval{batch_interval.get<typename BatchInterval::rep>()};
+  } else {
+    common_info.batch_interval = {};
+  }
+
+  if (const auto batch_size = data.at(kBatchSizeKey); !batch_size.is_null()) {
+    common_info.batch_size = batch_size.get<typename decltype(common_info.batch_size)::value_type>();
+  } else {
+    common_info.batch_size = {};
+  }
+
+  data.at(kTransformationName).get_to(common_info.transformation_name);
+}
+}  // namespace query

src/query/stream/common.hpp

@@ -0,0 +1,73 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <chrono>
+#include <cstdint>
+#include <functional>
+#include <optional>
+#include <string>
+
+#include <json/json.hpp>
+
+#include "query/procedure/mg_procedure_impl.hpp"
+
+namespace query {
+
+template <typename TMessage>
+using ConsumerFunction = std::function<void(const std::vector<TMessage> &)>;
+
+struct CommonStreamInfo {
+  std::optional<std::chrono::milliseconds> batch_interval;
+  std::optional<int64_t> batch_size;
+  std::string transformation_name;
+};
+
+template <typename T>
+concept ConvertableToJson = requires(T value, nlohmann::json data) {
+  { to_json(data, std::move(value)) } -> std::same_as<void>;
+  { from_json(data, value) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept ConvertableToMgpMessage = requires(T value) {
+  mgp_message{value};
+};
+
+template <typename TStream>
+concept Stream = requires(TStream stream) {
+  typename TStream::StreamInfo;
+  typename TStream::Message;
+  TStream{std::string{""}, typename TStream::StreamInfo{}, ConsumerFunction<typename TStream::Message>{}};
+  { stream.Start() } -> std::same_as<void>;
+  { stream.Stop() } -> std::same_as<void>;
+  { stream.IsRunning() } -> std::same_as<bool>;
+  {
+    stream.Check(std::optional<std::chrono::milliseconds>{}, std::optional<int64_t>{},
+                 ConsumerFunction<typename TStream::Message>{})
+    } -> std::same_as<void>;
+  { typename TStream::StreamInfo{}.common_info } -> std::same_as<CommonStreamInfo>;
+
+  requires ConvertableToMgpMessage<typename TStream::Message>;
+  requires ConvertableToJson<typename TStream::StreamInfo>;
+};
+
+enum class StreamSourceType : uint8_t { KAFKA, PULSAR };
+
+template <Stream T>
+StreamSourceType StreamType(const T & /*stream*/);
+
+const std::string kCommonInfoKey = "common_info";
+
+void to_json(nlohmann::json &data, CommonStreamInfo &&info);
+void from_json(const nlohmann::json &data, CommonStreamInfo &common_info);
+}  // namespace query

src/query/stream/sources.cpp

@@ -0,0 +1,113 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/sources.hpp"
+
+#include <json/json.hpp>
+
+namespace query {
+KafkaStream::KafkaStream(std::string stream_name, StreamInfo stream_info,
+                         ConsumerFunction<integrations::kafka::Message> consumer_function) {
+  integrations::kafka::ConsumerInfo consumer_info{
+      .consumer_name = std::move(stream_name),
+      .topics = std::move(stream_info.topics),
+      .consumer_group = std::move(stream_info.consumer_group),
+      .bootstrap_servers = std::move(stream_info.bootstrap_servers),
+      .batch_interval = stream_info.common_info.batch_interval,
+      .batch_size = stream_info.common_info.batch_size,
+  };
+  consumer_.emplace(std::move(consumer_info), std::move(consumer_function));
+};
+
+KafkaStream::StreamInfo KafkaStream::Info(std::string transformation_name) const {
+  const auto &info = consumer_->Info();
+  return {{.batch_interval = info.batch_interval,
+           .batch_size = info.batch_size,
+           .transformation_name = std::move(transformation_name)},
+          .topics = info.topics,
+          .consumer_group = info.consumer_group,
+          .bootstrap_servers = info.bootstrap_servers};
+}
+
+void KafkaStream::Start() { consumer_->Start(); }
+void KafkaStream::Stop() { consumer_->Stop(); }
+bool KafkaStream::IsRunning() const { return consumer_->IsRunning(); }
+
+void KafkaStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+                        const ConsumerFunction<integrations::kafka::Message> &consumer_function) const {
+  consumer_->Check(timeout, batch_limit, consumer_function);
+}
+
+namespace {
+const std::string kTopicsKey{"topics"};
+const std::string kConsumerGroupKey{"consumer_group"};
+const std::string kBoostrapServers{"bootstrap_servers"};
+}  // namespace
+
+void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info) {
+  data[kCommonInfoKey] = std::move(info.common_info);
+  data[kTopicsKey] = std::move(info.topics);
+  data[kConsumerGroupKey] = info.consumer_group;
+  data[kBoostrapServers] = std::move(info.bootstrap_servers);
+}
+
+void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info) {
+  data.at(kCommonInfoKey).get_to(info.common_info);
+  data.at(kTopicsKey).get_to(info.topics);
+  data.at(kConsumerGroupKey).get_to(info.consumer_group);
+  data.at(kBoostrapServers).get_to(info.bootstrap_servers);
+}
+
+PulsarStream::PulsarStream(std::string stream_name, StreamInfo stream_info,
+                           ConsumerFunction<integrations::pulsar::Message> consumer_function) {
+  integrations::pulsar::ConsumerInfo consumer_info{.batch_size = stream_info.common_info.batch_size,
+                                                   .batch_interval = stream_info.common_info.batch_interval,
+                                                   .topics = std::move(stream_info.topics),
+                                                   .consumer_name = std::move(stream_name),
+                                                   .service_url = std::move(stream_info.service_url)};
+
+  consumer_.emplace(std::move(consumer_info), std::move(consumer_function));
+};
+
+PulsarStream::StreamInfo PulsarStream::Info(std::string transformation_name) const {
+  const auto &info = consumer_->Info();
+  return {{.batch_interval = info.batch_interval,
+           .batch_size = info.batch_size,
+           .transformation_name = std::move(transformation_name)},
+          .topics = info.topics,
+          .service_url = info.service_url};
+}
+
+void PulsarStream::Start() { consumer_->Start(); }
+void PulsarStream::Stop() { consumer_->Stop(); }
+bool PulsarStream::IsRunning() const { return consumer_->IsRunning(); }
+
+void PulsarStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+                         const ConsumerFunction<Message> &consumer_function) const {
+  consumer_->Check(timeout, batch_limit, consumer_function);
+}
+
+namespace {
+const std::string kServiceUrl{"service_url"};
+}  // namespace
+
+void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info) {
+  data[kCommonInfoKey] = std::move(info.common_info);
+  data[kTopicsKey] = std::move(info.topics);
+  data[kServiceUrl] = std::move(info.service_url);
+}
+
+void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info) {
+  data.at(kCommonInfoKey).get_to(info.common_info);
+  data.at(kTopicsKey).get_to(info.topics);
+  data.at(kServiceUrl).get_to(info.service_url);
+}
+}  // namespace query

src/query/stream/sources.hpp

@@ -0,0 +1,89 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "query/stream/common.hpp"
+
+#include "integrations/kafka/consumer.hpp"
+#include "integrations/pulsar/consumer.hpp"
+
+namespace query {
+
+struct KafkaStream {
+  struct StreamInfo {
+    CommonStreamInfo common_info;
+    std::vector<std::string> topics;
+    std::string consumer_group;
+    std::string bootstrap_servers;
+  };
+
+  using Message = integrations::kafka::Message;
+
+  KafkaStream(std::string stream_name, StreamInfo stream_info,
+              ConsumerFunction<integrations::kafka::Message> consumer_function);
+
+  StreamInfo Info(std::string transformation_name) const;
+
+  void Start();
+  void Stop();
+  bool IsRunning() const;
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+             const ConsumerFunction<Message> &consumer_function) const;
+
+ private:
+  using Consumer = integrations::kafka::Consumer;
+  std::optional<Consumer> consumer_;
+};
+
+void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info);
+void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info);
+
+template <>
+inline StreamSourceType StreamType(const KafkaStream & /*stream*/) {
+  return StreamSourceType::KAFKA;
+}
+
+struct PulsarStream {
+  struct StreamInfo {
+    CommonStreamInfo common_info;
+    std::vector<std::string> topics;
+    std::string service_url;
+  };
+
+  using Message = integrations::pulsar::Message;
+
+  PulsarStream(std::string stream_name, StreamInfo stream_info, ConsumerFunction<Message> consumer_function);
+
+  StreamInfo Info(std::string transformation_name) const;
+
+  void Start();
+  void Stop();
+  bool IsRunning() const;
+
+  void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
+             const ConsumerFunction<Message> &consumer_function) const;
+
+ private:
+  using Consumer = integrations::pulsar::Consumer;
+  std::optional<Consumer> consumer_;
+};
+
+void to_json(nlohmann::json &data, PulsarStream::StreamInfo &&info);
+void from_json(const nlohmann::json &data, PulsarStream::StreamInfo &info);
+
+template <>
+inline StreamSourceType StreamType(const PulsarStream & /*stream*/) {
+  return StreamSourceType::PULSAR;
+}
+
+}  // namespace query

src/query/stream/streams.cpp

@@ -0,0 +1,434 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "query/stream/streams.hpp"
+
+#include <shared_mutex>
+#include <string_view>
+#include <utility>
+
+#include <spdlog/spdlog.h>
+#include <json/json.hpp>
+
+#include "query/db_accessor.hpp"
+#include "query/discard_value_stream.hpp"
+#include "query/interpreter.hpp"
+#include "query/procedure/mg_procedure_impl.hpp"
+#include "query/procedure/module.hpp"
+#include "query/typed_value.hpp"
+#include "utils/event_counter.hpp"
+#include "utils/memory.hpp"
+#include "utils/on_scope_exit.hpp"
+#include "utils/pmr/string.hpp"
+
+namespace EventCounter {
+extern const Event MessagesConsumed;
+}  // namespace EventCounter
+
+namespace query {
+namespace {
+constexpr auto kExpectedTransformationResultSize = 2;
+const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()};
+const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()};
+
+const std::map<std::string, storage::PropertyValue> empty_parameters{};
+
+auto GetStream(auto &map, const std::string &stream_name) {
+  if (auto it = map.find(stream_name); it != map.end()) {
+    return it;
+  }
+  throw StreamsException("Couldn't find stream '{}'", stream_name);
+}
+
+std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformationResult(
+    utils::pmr::map<utils::pmr::string, TypedValue> &&values, const std::string_view transformation_name,
+    const std::string_view stream_name) {
+  if (values.size() != kExpectedTransformationResultSize) {
+    throw StreamsException(
+        "Transformation '{}' in stream '{}' did not yield all fields (query, parameters) as required.",
+        transformation_name, stream_name);
+  }
+
+  auto get_value = [&](const utils::pmr::string &field_name) mutable -> TypedValue & {
+    auto it = values.find(field_name);
+    if (it == values.end()) {
+      throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.",
+                             transformation_name, stream_name, field_name};
+    };
+    return it->second;
+  };
+
+  auto &query_value = get_value(query_param_name);
+  MG_ASSERT(query_value.IsString());
+  auto &params_value = get_value(params_param_name);
+  MG_ASSERT(params_value.IsNull() || params_value.IsMap());
+  return {std::move(query_value), std::move(params_value)};
+}
+
+template <typename TMessage>
+void CallCustomTransformation(const std::string &transformation_name, const std::vector<TMessage> &messages,
+                              mgp_result &result, storage::Storage::Accessor &storage_accessor,
+                              utils::MemoryResource &memory_resource, const std::string &stream_name) {
+  DbAccessor db_accessor{&storage_accessor};
+  {
+    auto maybe_transformation =
+        procedure::FindTransformation(procedure::gModuleRegistry, transformation_name, utils::NewDeleteResource());
+
+    if (!maybe_transformation) {
+      throw StreamsException("Couldn't find transformation {} for stream '{}'", transformation_name, stream_name);
+    };
+    const auto &trans = *maybe_transformation->second;
+    mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}};
+    std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages),
+                   [](const TMessage &message) { return mgp_message{message}; });
+    mgp_graph graph{&db_accessor, storage::View::OLD, nullptr};
+    mgp_memory memory{&memory_resource};
+    result.rows.clear();
+    result.error_msg.reset();
+    result.signature = &trans.results;
+
+    MG_ASSERT(result.signature->size() == kExpectedTransformationResultSize);
+    MG_ASSERT(result.signature->contains(query_param_name));
+    MG_ASSERT(result.signature->contains(params_param_name));
+
+    spdlog::trace("Calling transformation in stream '{}'", stream_name);
+    trans.cb(&mgp_messages, &graph, &result, &memory);
+  }
+  if (result.error_msg.has_value()) {
+    throw StreamsException(result.error_msg->c_str());
+  }
+}
+
+template <Stream TStream>
+StreamStatus<TStream> CreateStatus(std::string stream_name, std::string transformation_name,
+                                   std::optional<std::string> owner, const TStream &stream) {
+  return {.name = std::move(stream_name),
+          .type = StreamType(stream),
+          .is_running = stream.IsRunning(),
+          .info = stream.Info(std::move(transformation_name)),
+          .owner = std::move(owner)};
+}
+
+// nlohmann::json doesn't support string_view access yet
+const std::string kStreamName{"name"};
+const std::string kIsRunningKey{"is_running"};
+const std::string kOwner{"owner"};
+const std::string kType{"type"};
+}  // namespace
+
+template <Stream TStream>
+void to_json(nlohmann::json &data, StreamStatus<TStream> &&status) {
+  data[kStreamName] = std::move(status.name);
+  data[kType] = status.type;
+  data[kIsRunningKey] = status.is_running;
+
+  if (status.owner.has_value()) {
+    data[kOwner] = std::move(*status.owner);
+  } else {
+    data[kOwner] = nullptr;
+  }
+
+  to_json(data, std::move(status.info));
+}
+
+template <Stream TStream>
+void from_json(const nlohmann::json &data, StreamStatus<TStream> &status) {
+  data.at(kStreamName).get_to(status.name);
+  data.at(kIsRunningKey).get_to(status.is_running);
+
+  if (const auto &owner = data.at(kOwner); !owner.is_null()) {
+    status.owner = owner.get<typename decltype(status.owner)::value_type>();
+  } else {
+    status.owner = {};
+  }
+
+  from_json(data, status.info);
+}
+
+Streams::Streams(InterpreterContext *interpreter_context, std::filesystem::path directory)
+    : interpreter_context_(interpreter_context), storage_(std::move(directory)) {}
+
+template <Stream TStream>
+void Streams::Create(const std::string &stream_name, typename TStream::StreamInfo info,
+                     std::optional<std::string> owner) {
+  auto locked_streams = streams_.Lock();
+  auto it = CreateConsumer<TStream>(*locked_streams, stream_name, std::move(info), std::move(owner));
+
+  try {
+    std::visit(
+        [&](auto &&stream_data) {
+          const auto stream_source_ptr = stream_data.stream_source->ReadLock();
+          Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+        },
+        it->second);
+  } catch (...) {
+    locked_streams->erase(it);
+    throw;
+  }
+}
+
+template void Streams::Create<KafkaStream>(const std::string &stream_name, KafkaStream::StreamInfo info,
+                                           std::optional<std::string> owner);
+template void Streams::Create<PulsarStream>(const std::string &stream_name, PulsarStream::StreamInfo info,
+                                            std::optional<std::string> owner);
+
+template <Stream TStream>
+Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std::string &stream_name,
+                                                      typename TStream::StreamInfo stream_info,
+                                                      std::optional<std::string> owner) {
+  if (map.contains(stream_name)) {
+    throw StreamsException{"Stream already exists with name '{}'", stream_name};
+  }
+
+  auto *memory_resource = utils::NewDeleteResource();
+
+  auto consumer_function =
+      [interpreter_context = interpreter_context_, memory_resource, stream_name,
+       transformation_name = stream_info.common_info.transformation_name, owner = owner,
+       interpreter = std::make_shared<Interpreter>(interpreter_context_),
+       result = mgp_result{nullptr, memory_resource}](const std::vector<typename TStream::Message> &messages) mutable {
+        auto accessor = interpreter_context->db->Access();
+        EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
+        CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
+
+        DiscardValueResultStream stream;
+
+        spdlog::trace("Start transaction in stream '{}'", stream_name);
+        utils::OnScopeExit cleanup{[&interpreter, &result]() {
+          result.rows.clear();
+          interpreter->Abort();
+        }};
+        interpreter->BeginTransaction();
+
+        const static std::map<std::string, storage::PropertyValue> empty_parameters{};
+
+        for (auto &row : result.rows) {
+          spdlog::trace("Processing row in stream '{}'", stream_name);
+          auto [query_value, params_value] =
+              ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
+          storage::PropertyValue params_prop{params_value};
+
+          std::string query{query_value.ValueString()};
+          spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name);
+          auto prepare_result =
+              interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr);
+          if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges)) {
+            throw StreamsException{
+                "Couldn't execute query '{}' for stream '{}' becuase the owner is not authorized to execute the "
+                "query!",
+                query, stream_name};
+          }
+          interpreter->PullAll(&stream);
+        }
+
+        spdlog::trace("Commit transaction in stream '{}'", stream_name);
+        interpreter->CommitTransaction();
+        result.rows.clear();
+      };
+
+  auto insert_result = map.try_emplace(
+      stream_name, StreamData<TStream>{std::move(stream_info.common_info.transformation_name), std::move(owner),
+                                       std::make_unique<SynchronizedStreamSource<TStream>>(
+                                           stream_name, std::move(stream_info), std::move(consumer_function))});
+  MG_ASSERT(insert_result.second, "Unexpected error during storing consumer '{}'", stream_name);
+  return insert_result.first;
+}
+
+void Streams::RestoreStreams() {
+  spdlog::info("Loading streams...");
+  auto locked_streams_map = streams_.Lock();
+  MG_ASSERT(locked_streams_map->empty(), "Cannot restore streams when some streams already exist!");
+
+  for (const auto &[stream_name, stream_data] : storage_) {
+    const auto get_failed_message = [&stream_name = stream_name](const std::string_view message,
+                                                                 const std::string_view nested_message) {
+      return fmt::format("Failed to load stream '{}', because: {} caused by {}", stream_name, message, nested_message);
+    };
+
+    const auto create_consumer = [&, &stream_name = stream_name, this]<typename T>(StreamStatus<T> status,
+                                                                                   auto &&stream_json_data) {
+      try {
+        stream_json_data.get_to(status);
+      } catch (const nlohmann::json::type_error &exception) {
+        spdlog::warn(get_failed_message("invalid type conversion", exception.what()));
+        return;
+      } catch (const nlohmann::json::out_of_range &exception) {
+        spdlog::warn(get_failed_message("non existing field", exception.what()));
+        return;
+      }
+      MG_ASSERT(status.name == stream_name, "Expected stream name is '{}', but got '{}'", status.name, stream_name);
+
+      try {
+        auto it = CreateConsumer<T>(*locked_streams_map, stream_name, std::move(status.info), std::move(status.owner));
+        if (status.is_running) {
+          std::visit(
+              [&](auto &&stream_data) {
+                auto stream_source_ptr = stream_data.stream_source->Lock();
+                stream_source_ptr->Start();
+              },
+              it->second);
+        }
+        spdlog::info("Stream '{}' is loaded", stream_name);
+      } catch (const utils::BasicException &exception) {
+        spdlog::warn(get_failed_message("unexpected error", exception.what()));
+      }
+    };
+
+    auto stream_json_data = nlohmann::json::parse(stream_data);
+    const auto stream_type = static_cast<StreamSourceType>(stream_json_data.at("type"));
+
+    switch (stream_type) {
+      case StreamSourceType::KAFKA:
+        create_consumer(StreamStatus<KafkaStream>{}, std::move(stream_json_data));
+        break;
+      case StreamSourceType::PULSAR:
+        create_consumer(StreamStatus<PulsarStream>{}, std::move(stream_json_data));
+        break;
+    }
+  }
+}
+
+void Streams::Drop(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+
+  auto it = GetStream(*locked_streams, stream_name);
+
+  // streams_ is write locked, which means there is no access to it outside of this function, thus only the Test
+  // function can be executing with the consumer, nothing else.
+  // By acquiring the write lock here for the consumer, we make sure there is
+  // no running Test function for this consumer, therefore it can be erased.
+  std::visit([&](auto &&stream_data) { stream_data.stream_source->Lock(); }, it->second);
+
+  locked_streams->erase(it);
+  if (!storage_.Delete(stream_name)) {
+    throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
+  }
+
+  // TODO(antaljanosbenjamin) Release the transformation
+}
+
+void Streams::Start(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+  auto it = GetStream(*locked_streams, stream_name);
+
+  std::visit(
+      [&, this](auto &&stream_data) {
+        auto stream_source_ptr = stream_data.stream_source->Lock();
+        stream_source_ptr->Start();
+        Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+      },
+      it->second);
+}
+
+void Streams::Stop(const std::string &stream_name) {
+  auto locked_streams = streams_.Lock();
+  auto it = GetStream(*locked_streams, stream_name);
+
+  std::visit(
+      [&, this](auto &&stream_data) {
+        auto stream_source_ptr = stream_data.stream_source->Lock();
+        stream_source_ptr->Stop();
+
+        Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
+      },
+      it->second);
+}
+
+void Streams::StartAll() {
+  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
+    std::visit(
+        [&stream_name = stream_name, this](auto &&stream_data) {
+          auto locked_stream_source = stream_data.stream_source->Lock();
+          if (!locked_stream_source->IsRunning()) {
+            locked_stream_source->Start();
+            Persist(
+                CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source));
+          }
+        },
+        stream_data);
+  }
+}
+
+void Streams::StopAll() {
+  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
+    std::visit(
+        [&stream_name = stream_name, this](auto &&stream_data) {
+          auto locked_stream_source = stream_data.stream_source->Lock();
+          if (locked_stream_source->IsRunning()) {
+            locked_stream_source->Stop();
+            Persist(
+                CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_stream_source));
+          }
+        },
+        stream_data);
+  }
+}
+
+std::vector<StreamStatus<>> Streams::GetStreamInfo() const {
+  std::vector<StreamStatus<>> result;
+  {
+    for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) {
+      std::visit(
+          [&, &stream_name = stream_name](auto &&stream_data) {
+            auto locked_stream_source = stream_data.stream_source->ReadLock();
+            auto info = locked_stream_source->Info(stream_data.transformation_name);
+            result.emplace_back(StreamStatus<>{stream_name, StreamType(*locked_stream_source),
+                                               locked_stream_source->IsRunning(), std::move(info.common_info),
+                                               stream_data.owner});
+          },
+          stream_data);
+    }
+  }
+  return result;
+}
+
+TransformationResult Streams::Check(const std::string &stream_name, std::optional<std::chrono::milliseconds> timeout,
+                                    std::optional<int64_t> batch_limit) const {
+  std::optional locked_streams{streams_.ReadLock()};
+  auto it = GetStream(**locked_streams, stream_name);
+
+  return std::visit(
+      [&](auto &&stream_data) {
+        // This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after
+        // that
+        const auto locked_stream_source = stream_data.stream_source->ReadLock();
+        const auto transformation_name = stream_data.transformation_name;
+        locked_streams.reset();
+
+        auto *memory_resource = utils::NewDeleteResource();
+        mgp_result result{nullptr, memory_resource};
+        TransformationResult test_result;
+
+        auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name,
+                                  &transformation_name = transformation_name, &result,
+                                  &test_result]<typename T>(const std::vector<T> &messages) mutable {
+          auto accessor = interpreter_context->db->Access();
+          CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
+
+          for (auto &row : result.rows) {
+            auto [query, parameters] =
+                ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
+            std::vector<TypedValue> result_row;
+            result_row.reserve(kExpectedTransformationResultSize);
+            result_row.push_back(std::move(query));
+            result_row.push_back(std::move(parameters));
+
+            test_result.push_back(std::move(result_row));
+          }
+        };
+
+        locked_stream_source->Check(timeout, batch_limit, consumer_function);
+        return test_result;
+      },
+      it->second);
+}
+
+}  // namespace query

src/query/stream/streams.hpp

@@ -11,14 +11,22 @@
 
 #pragma once
 
+#include <concepts>
 #include <functional>
 #include <map>
 #include <optional>
+#include <type_traits>
 #include <unordered_map>
 
+#include <json/json.hpp>
+
 #include "integrations/kafka/consumer.hpp"
 #include "kvstore/kvstore.hpp"
+#include "query/stream/common.hpp"
+#include "query/stream/sources.hpp"
 #include "query/typed_value.hpp"
+#include "storage/v2/property_value.hpp"
+#include "utils/event_counter.hpp"
 #include "utils/exceptions.hpp"
 #include "utils/rw_lock.hpp"
 #include "utils/synchronized.hpp"
@@ -30,33 +38,33 @@ class StreamsException : public utils::BasicException {
   using BasicException::BasicException;
 };
 
-using TransformationResult = std::vector<std::vector<TypedValue>>;
+template <typename T>
-using TransformFunction = std::function<TransformationResult(const std::vector<integrations::kafka::Message> &)>;
+struct StreamInfo;
 
-struct StreamInfo {
+template <>
-  std::vector<std::string> topics;
+struct StreamInfo<void> {
-  std::string consumer_group;
+  using Type = CommonStreamInfo;
-  std::optional<std::chrono::milliseconds> batch_interval;
-  std::optional<int64_t> batch_size;
-  std::string transformation_name;
-  std::optional<std::string> owner;
-  std::string bootstrap_servers;
 };
 
+template <Stream TStream>
+struct StreamInfo<TStream> {
+  using Type = typename TStream::StreamInfo;
+};
+
+template <typename T>
+using StreamInfoType = typename StreamInfo<T>::Type;
+
+template <typename T = void>
 struct StreamStatus {
   std::string name;
-  StreamInfo info;
+  StreamSourceType type;
   bool is_running;
-};
+  StreamInfoType<T> info;
-
-using SynchronizedConsumer = utils::Synchronized<integrations::kafka::Consumer, utils::WritePrioritizedRWLock>;
-
-struct StreamData {
-  std::string transformation_name;
   std::optional<std::string> owner;
-  std::unique_ptr<SynchronizedConsumer> consumer;
 };
 
+using TransformationResult = std::vector<std::vector<TypedValue>>;
+
 struct InterpreterContext;
 
 /// Manages Kafka consumers.
@@ -67,9 +75,8 @@ class Streams final {
   /// Initializes the streams.
   ///
   /// @param interpreter_context context to use to run the result of transformations
-  /// @param bootstrap_servers initial list of brokers as a comma separated list of broker host or host:port
   /// @param directory a directory path to store the persisted streams metadata
-  Streams(InterpreterContext *interpreter_context, std::string bootstrap_servers, std::filesystem::path directory);
+  Streams(InterpreterContext *interpreter_context, std::filesystem::path directory);
 
   /// Restores the streams from the persisted metadata.
   /// The restoration is done in a best effort manner, therefore no exception is thrown on failure, but the error is
@@ -85,7 +92,8 @@ class Streams final {
   /// @param stream_info the necessary informations needed to create the Kafka consumer and transform the messages
   ///
   /// @throws StreamsException if the stream with the same name exists or if the creation of Kafka consumer fails
-  void Create(const std::string &stream_name, StreamInfo stream_info);
+  template <Stream TStream>
+  void Create(const std::string &stream_name, typename TStream::StreamInfo info, std::optional<std::string> owner);
 
   /// Deletes an existing stream and all the data that was persisted.
   ///
@@ -123,7 +131,7 @@ class Streams final {
   /// Return current status for all streams.
   /// It might happend that the is_running field is out of date if the one of the streams stops during the invocation of
   /// this function because of an error.
-  std::vector<StreamStatus> GetStreamInfo() const;
+  std::vector<StreamStatus<>> GetStreamInfo() const;
 
   /// Do a dry-run consume from a stream.
   ///
@@ -144,19 +152,33 @@ class Streams final {
   std::string_view BootstrapServers() const;
 
  private:
-  using StreamsMap = std::unordered_map<std::string, StreamData>;
+  template <Stream TStream>
+  using SynchronizedStreamSource = utils::Synchronized<TStream, utils::WritePrioritizedRWLock>;
+
+  template <Stream TStream>
+  struct StreamData {
+    std::string transformation_name;
+    std::optional<std::string> owner;
+    std::unique_ptr<SynchronizedStreamSource<TStream>> stream_source;
+  };
+
+  using StreamDataVariant = std::variant<StreamData<KafkaStream>, StreamData<PulsarStream>>;
+  using StreamsMap = std::unordered_map<std::string, StreamDataVariant>;
   using SynchronizedStreamsMap = utils::Synchronized<StreamsMap, utils::WritePrioritizedRWLock>;
 
-  static StreamStatus CreateStatus(const std::string &name, const std::string &transformation_name,
+  template <Stream TStream>
-                                   const std::optional<std::string> &owner,
+  StreamsMap::iterator CreateConsumer(StreamsMap &map, const std::string &stream_name,
-                                   const integrations::kafka::Consumer &consumer);
+                                      typename TStream::StreamInfo stream_info, std::optional<std::string> owner);
 
-  StreamsMap::iterator CreateConsumer(StreamsMap &map, const std::string &stream_name, StreamInfo stream_info);
+  template <Stream TStream>
-
+  void Persist(StreamStatus<TStream> &&status) {
-  void Persist(StreamStatus &&status);
+    const std::string stream_name = status.name;
+    if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
+      throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
+    }
+  }
 
   InterpreterContext *interpreter_context_;
-  std::string bootstrap_servers_;
   kvstore::KVStore storage_;
 
   SynchronizedStreamsMap streams_;

src/query/streams.cpp

@@ -1,437 +0,0 @@
-// Copyright 2021 Memgraph Ltd.
-//
-// Use of this software is governed by the Business Source License
-// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
-// License, and you may not use this file except in compliance with the Business Source License.
-//
-// As of the Change Date specified in that file, in accordance with
-// the Business Source License, use of this software will be governed
-// by the Apache License, Version 2.0, included in the file
-// licenses/APL.txt.
-
-#include "query/streams.hpp"
-
-#include <shared_mutex>
-#include <string_view>
-#include <utility>
-
-#include <spdlog/spdlog.h>
-#include <json/json.hpp>
-#include "query/db_accessor.hpp"
-#include "query/discard_value_stream.hpp"
-#include "query/interpreter.hpp"
-#include "query/procedure/mg_procedure_impl.hpp"
-#include "query/procedure/module.hpp"
-#include "query/typed_value.hpp"
-#include "utils/event_counter.hpp"
-#include "utils/memory.hpp"
-#include "utils/on_scope_exit.hpp"
-#include "utils/pmr/string.hpp"
-
-namespace EventCounter {
-extern const Event MessagesConsumed;
-}  // namespace EventCounter
-
-namespace query {
-
-using Consumer = integrations::kafka::Consumer;
-using ConsumerInfo = integrations::kafka::ConsumerInfo;
-using Message = integrations::kafka::Message;
-namespace {
-constexpr auto kExpectedTransformationResultSize = 2;
-const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()};
-const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()};
-const std::map<std::string, storage::PropertyValue> empty_parameters{};
-
-auto GetStream(auto &map, const std::string &stream_name) {
-  if (auto it = map.find(stream_name); it != map.end()) {
-    return it;
-  }
-  throw StreamsException("Couldn't find stream '{}'", stream_name);
-}
-
-void CallCustomTransformation(const std::string &transformation_name, const std::vector<Message> &messages,
-                              mgp_result &result, storage::Storage::Accessor &storage_accessor,
-                              utils::MemoryResource &memory_resource, const std::string &stream_name) {
-  DbAccessor db_accessor{&storage_accessor};
-  {
-    auto maybe_transformation =
-        procedure::FindTransformation(procedure::gModuleRegistry, transformation_name, utils::NewDeleteResource());
-
-    if (!maybe_transformation) {
-      throw StreamsException("Couldn't find transformation {} for stream '{}'", transformation_name, stream_name);
-    };
-    const auto &trans = *maybe_transformation->second;
-    mgp_messages mgp_messages{mgp_messages::storage_type{&memory_resource}};
-    std::transform(messages.begin(), messages.end(), std::back_inserter(mgp_messages.messages),
-                   [](const integrations::kafka::Message &message) { return mgp_message{&message}; });
-    mgp_graph graph{&db_accessor, storage::View::OLD, nullptr};
-    mgp_memory memory{&memory_resource};
-    result.rows.clear();
-    result.error_msg.reset();
-    result.signature = &trans.results;
-
-    MG_ASSERT(result.signature->size() == kExpectedTransformationResultSize);
-    MG_ASSERT(result.signature->contains(query_param_name));
-    MG_ASSERT(result.signature->contains(params_param_name));
-
-    spdlog::trace("Calling transformation in stream '{}'", stream_name);
-    trans.cb(&mgp_messages, &graph, &result, &memory);
-  }
-  if (result.error_msg.has_value()) {
-    throw StreamsException(result.error_msg->c_str());
-  }
-}
-
-std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformationResult(
-    utils::pmr::map<utils::pmr::string, TypedValue> &&values, const std::string_view transformation_name,
-    const std::string_view stream_name) {
-  if (values.size() != kExpectedTransformationResultSize) {
-    throw StreamsException(
-        "Transformation '{}' in stream '{}' did not yield all fields (query, parameters) as required.",
-        transformation_name, stream_name);
-  }
-
-  auto get_value = [&](const utils::pmr::string &field_name) mutable -> TypedValue & {
-    auto it = values.find(field_name);
-    if (it == values.end()) {
-      throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.",
-                             transformation_name, stream_name, field_name};
-    };
-    return it->second;
-  };
-
-  auto &query_value = get_value(query_param_name);
-  MG_ASSERT(query_value.IsString());
-  auto &params_value = get_value(params_param_name);
-  MG_ASSERT(params_value.IsNull() || params_value.IsMap());
-  return {std::move(query_value), std::move(params_value)};
-}
-}  // namespace
-
-// nlohmann::json doesn't support string_view access yet
-const std::string kStreamName{"name"};
-const std::string kTopicsKey{"topics"};
-const std::string kConsumerGroupKey{"consumer_group"};
-const std::string kBatchIntervalKey{"batch_interval"};
-const std::string kBatchSizeKey{"batch_size"};
-const std::string kIsRunningKey{"is_running"};
-const std::string kTransformationName{"transformation_name"};
-const std::string kOwner{"owner"};
-const std::string kBoostrapServers{"bootstrap_servers"};
-
-void to_json(nlohmann::json &data, StreamStatus &&status) {
-  auto &info = status.info;
-  data[kStreamName] = std::move(status.name);
-  data[kTopicsKey] = std::move(info.topics);
-  data[kConsumerGroupKey] = info.consumer_group;
-
-  if (info.batch_interval) {
-    data[kBatchIntervalKey] = info.batch_interval->count();
-  } else {
-    data[kBatchIntervalKey] = nullptr;
-  }
-
-  if (info.batch_size) {
-    data[kBatchSizeKey] = *info.batch_size;
-  } else {
-    data[kBatchSizeKey] = nullptr;
-  }
-
-  data[kIsRunningKey] = status.is_running;
-  data[kTransformationName] = status.info.transformation_name;
-
-  if (info.owner.has_value()) {
-    data[kOwner] = std::move(*info.owner);
-  } else {
-    data[kOwner] = nullptr;
-  }
-
-  data[kBoostrapServers] = std::move(info.bootstrap_servers);
-}
-
-void from_json(const nlohmann::json &data, StreamStatus &status) {
-  auto &info = status.info;
-  data.at(kStreamName).get_to(status.name);
-  data.at(kTopicsKey).get_to(info.topics);
-  data.at(kConsumerGroupKey).get_to(info.consumer_group);
-
-  if (const auto batch_interval = data.at(kBatchIntervalKey); !batch_interval.is_null()) {
-    using BatchInterval = decltype(info.batch_interval)::value_type;
-    info.batch_interval = BatchInterval{batch_interval.get<BatchInterval::rep>()};
-  } else {
-    info.batch_interval = {};
-  }
-
-  if (const auto batch_size = data.at(kBatchSizeKey); !batch_size.is_null()) {
-    info.batch_size = batch_size.get<decltype(info.batch_size)::value_type>();
-  } else {
-    info.batch_size = {};
-  }
-
-  data.at(kIsRunningKey).get_to(status.is_running);
-  data.at(kTransformationName).get_to(status.info.transformation_name);
-
-  if (const auto &owner = data.at(kOwner); !owner.is_null()) {
-    info.owner = owner.get<decltype(info.owner)::value_type>();
-  } else {
-    info.owner = {};
-  }
-
-  info.owner = data.value(kBoostrapServers, "");
-}
-
-Streams::Streams(InterpreterContext *interpreter_context, std::string bootstrap_servers,
-                 std::filesystem::path directory)
-    : interpreter_context_(interpreter_context),
-      bootstrap_servers_(std::move(bootstrap_servers)),
-      storage_(std::move(directory)) {}
-
-void Streams::RestoreStreams() {
-  spdlog::info("Loading streams...");
-  auto locked_streams_map = streams_.Lock();
-  MG_ASSERT(locked_streams_map->empty(), "Cannot restore streams when some streams already exist!");
-
-  for (const auto &[stream_name, stream_data] : storage_) {
-    const auto get_failed_message = [&stream_name = stream_name](const std::string_view message,
-                                                                 const std::string_view nested_message) {
-      return fmt::format("Failed to load stream '{}', because: {} caused by {}", stream_name, message, nested_message);
-    };
-
-    StreamStatus status;
-    try {
-      nlohmann::json::parse(stream_data).get_to(status);
-    } catch (const nlohmann::json::type_error &exception) {
-      spdlog::warn(get_failed_message("invalid type conversion", exception.what()));
-      continue;
-    } catch (const nlohmann::json::out_of_range &exception) {
-      spdlog::warn(get_failed_message("non existing field", exception.what()));
-      continue;
-    }
-    MG_ASSERT(status.name == stream_name, "Expected stream name is '{}', but got '{}'", status.name, stream_name);
-
-    try {
-      auto it = CreateConsumer(*locked_streams_map, stream_name, std::move(status.info));
-      if (status.is_running) {
-        it->second.consumer->Lock()->Start();
-      }
-      spdlog::info("Stream '{}' is loaded", stream_name);
-    } catch (const utils::BasicException &exception) {
-      spdlog::warn(get_failed_message("unexpected error", exception.what()));
-    }
-  }
-}
-
-void Streams::Create(const std::string &stream_name, StreamInfo info) {
-  auto locked_streams = streams_.Lock();
-  auto it = CreateConsumer(*locked_streams, stream_name, std::move(info));
-
-  try {
-    Persist(
-        CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *it->second.consumer->ReadLock()));
-  } catch (...) {
-    locked_streams->erase(it);
-    throw;
-  }
-}
-
-void Streams::Drop(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-
-  auto it = GetStream(*locked_streams, stream_name);
-
-  // streams_ is write locked, which means there is no access to it outside of this function, thus only the Test
-  // function can be executing with the consumer, nothing else.
-  // By acquiring the write lock here for the consumer, we make sure there is
-  // no running Test function for this consumer, therefore it can be erased.
-  it->second.consumer->Lock();
-  locked_streams->erase(it);
-
-  if (!storage_.Delete(stream_name)) {
-    throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
-  }
-
-  // TODO(antaljanosbenjamin) Release the transformation
-}
-
-void Streams::Start(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-  auto it = GetStream(*locked_streams, stream_name);
-
-  auto locked_consumer = it->second.consumer->Lock();
-  locked_consumer->Start();
-
-  Persist(CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *locked_consumer));
-}
-
-void Streams::Stop(const std::string &stream_name) {
-  auto locked_streams = streams_.Lock();
-  auto it = GetStream(*locked_streams, stream_name);
-
-  auto locked_consumer = it->second.consumer->Lock();
-  locked_consumer->Stop();
-
-  Persist(CreateStatus(stream_name, it->second.transformation_name, it->second.owner, *locked_consumer));
-}
-
-void Streams::StartAll() {
-  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
-    auto locked_consumer = stream_data.consumer->Lock();
-    if (!locked_consumer->IsRunning()) {
-      locked_consumer->Start();
-      Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_consumer));
-    }
-  }
-}
-
-void Streams::StopAll() {
-  for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
-    auto locked_consumer = stream_data.consumer->Lock();
-    if (locked_consumer->IsRunning()) {
-      locked_consumer->Stop();
-      Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *locked_consumer));
-    }
-  }
-}
-
-std::vector<StreamStatus> Streams::GetStreamInfo() const {
-  std::vector<StreamStatus> result;
-  {
-    for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) {
-      result.emplace_back(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner,
-                                       *stream_data.consumer->ReadLock()));
-    }
-  }
-  return result;
-}
-
-TransformationResult Streams::Check(const std::string &stream_name, std::optional<std::chrono::milliseconds> timeout,
-                                    std::optional<int64_t> batch_limit) const {
-  // This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after that
-  auto [locked_consumer,
-        transformation_name] = [this, &stream_name]() -> std::pair<SynchronizedConsumer::ReadLockedPtr, std::string> {
-    auto locked_streams = streams_.ReadLock();
-    auto it = GetStream(*locked_streams, stream_name);
-    return {it->second.consumer->ReadLock(), it->second.transformation_name};
-  }();
-
-  auto *memory_resource = utils::NewDeleteResource();
-  mgp_result result{nullptr, memory_resource};
-  TransformationResult test_result;
-
-  auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, &stream_name,
-                            &transformation_name = transformation_name, &result,
-                            &test_result](const std::vector<Message> &messages) mutable {
-    auto accessor = interpreter_context->db->Access();
-    CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
-
-    for (auto &row : result.rows) {
-      auto [query, parameters] = ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
-      std::vector<TypedValue> result_row;
-      result_row.reserve(kExpectedTransformationResultSize);
-      result_row.push_back(std::move(query));
-      result_row.push_back(std::move(parameters));
-
-      test_result.push_back(std::move(result_row));
-    }
-  };
-
-  locked_consumer->Check(timeout, batch_limit, consumer_function);
-
-  return test_result;
-}
-
-StreamStatus Streams::CreateStatus(const std::string &name, const std::string &transformation_name,
-                                   const std::optional<std::string> &owner,
-                                   const integrations::kafka::Consumer &consumer) {
-  const auto &info = consumer.Info();
-  return StreamStatus{name,
-                      StreamInfo{
-                          info.topics,
-                          info.consumer_group,
-                          info.batch_interval,
-                          info.batch_size,
-                          transformation_name,
-                          owner,
-                      },
-                      consumer.IsRunning()};
-}
-
-Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std::string &stream_name,
-                                                      StreamInfo stream_info) {
-  if (map.contains(stream_name)) {
-    throw StreamsException{"Stream already exists with name '{}'", stream_name};
-  }
-
-  auto *memory_resource = utils::NewDeleteResource();
-
-  auto consumer_function = [interpreter_context = interpreter_context_, memory_resource, stream_name,
-                            transformation_name = stream_info.transformation_name, owner = stream_info.owner,
-                            interpreter = std::make_shared<Interpreter>(interpreter_context_),
-                            result = mgp_result{nullptr, memory_resource}](
-                               const std::vector<integrations::kafka::Message> &messages) mutable {
-    auto accessor = interpreter_context->db->Access();
-    EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
-    CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
-
-    DiscardValueResultStream stream;
-
-    spdlog::trace("Start transaction in stream '{}'", stream_name);
-    utils::OnScopeExit cleanup{[&interpreter, &result]() {
-      result.rows.clear();
-      interpreter->Abort();
-    }};
-    interpreter->BeginTransaction();
-
-    for (auto &row : result.rows) {
-      spdlog::trace("Processing row in stream '{}'", stream_name);
-      auto [query_value, params_value] =
-          ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
-      storage::PropertyValue params_prop{params_value};
-
-      std::string query{query_value.ValueString()};
-      spdlog::trace("Executing query '{}' in stream '{}'", query, stream_name);
-      auto prepare_result =
-          interpreter->Prepare(query, params_prop.IsNull() ? empty_parameters : params_prop.ValueMap(), nullptr);
-      if (!interpreter_context->auth_checker->IsUserAuthorized(owner, prepare_result.privileges)) {
-        throw StreamsException{
-            "Couldn't execute query '{}' for stream '{}' becuase the owner is not authorized to execute the "
-            "query!",
-            query, stream_name};
-      }
-      interpreter->PullAll(&stream);
-    }
-
-    spdlog::trace("Commit transaction in stream '{}'", stream_name);
-    interpreter->CommitTransaction();
-    result.rows.clear();
-  };
-
-  ConsumerInfo consumer_info{
-      .consumer_name = stream_name,
-      .topics = std::move(stream_info.topics),
-      .consumer_group = std::move(stream_info.consumer_group),
-      .batch_interval = stream_info.batch_interval,
-      .batch_size = stream_info.batch_size,
-  };
-
-  auto bootstrap_servers =
-      stream_info.bootstrap_servers.empty() ? bootstrap_servers_ : std::move(stream_info.bootstrap_servers);
-  auto insert_result = map.insert_or_assign(
-      stream_name,
-      StreamData{std::move(stream_info.transformation_name), std::move(stream_info.owner),
-                 std::make_unique<SynchronizedConsumer>(std::move(bootstrap_servers), std::move(consumer_info),
-                                                        std::move(consumer_function))});
-  MG_ASSERT(insert_result.second, "Unexpected error during storing consumer '{}'", stream_name);
-  return insert_result.first;
-}
-
-void Streams::Persist(StreamStatus &&status) {
-  const std::string stream_name = status.name;
-  if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
-    throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
-  }
-}
-
-std::string_view Streams::BootstrapServers() const { return bootstrap_servers_; }
-}  // namespace query

src/utils/variant.hpp

@@ -0,0 +1,22 @@
+// Copyright 2021 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+namespace utils {
+template <class... Ts>
+struct Overloaded : Ts... {
+  using Ts::operator()...;
+};
+
+template <class... Ts>
+Overloaded(Ts...) -> Overloaded<Ts...>;
+}  // namespace utils

tests/benchmark/expansion.cpp

@@ -48,7 +48,7 @@ class ExpansionBenchFixture : public benchmark::Fixture {
 
     MG_ASSERT(db->CreateIndex(label));
 
-    interpreter_context.emplace(&*db, query::InterpreterConfig{}, data_directory, "non existing bootstrap servers");
+    interpreter_context.emplace(&*db, query::InterpreterConfig{}, data_directory);
     interpreter.emplace(&*interpreter_context);
   }
 

tests/e2e/streams/common.py

@@ -15,14 +15,11 @@ import time
 # These are the indices of the different values in the result of SHOW STREAM
 # query
 NAME = 0
-TOPICS = 1
+BATCH_INTERVAL = 1
-CONSUMER_GROUP = 2
+BATCH_SIZE = 2
-BATCH_INTERVAL = 3
+TRANSFORM = 3
-BATCH_SIZE = 4
+OWNER = 4
-TRANSFORM = 5
+IS_RUNNING = 5
-OWNER = 6
-BOOTSTRAP_SERVERS = 7
-IS_RUNNING = 8
 
 
 def execute_and_fetch_all(cursor, query):
@@ -74,17 +71,19 @@ def check_one_result_row(cursor, query):
 
 
 def check_vertex_exists_with_topic_and_payload(cursor, topic, payload_bytes):
-    assert check_one_result_row(cursor,
+    assert check_one_result_row(
-                                "MATCH (n: MESSAGE {"
+        cursor,
-                                f"payload: '{payload_bytes.decode('utf-8')}',"
+        "MATCH (n: MESSAGE {"
-                                f"topic: '{topic}'"
+        f"payload: '{payload_bytes.decode('utf-8')}',"
-                                "}) RETURN n")
+        f"topic: '{topic}'"
+        "}) RETURN n",
+    )
 
 
 def get_stream_info(cursor, stream_name):
     stream_infos = execute_and_fetch_all(cursor, "SHOW STREAMS")
     for stream_info in stream_infos:
-        if (stream_info[NAME] == stream_name):
+        if stream_info[NAME] == stream_name:
             return stream_info
 
     return None

tests/e2e/streams/streams_owner_tests.py

@@ -77,9 +77,8 @@ def test_owner_is_shown(topics, connection):
                                  f"TOPICS {topics[0]} "
                                  f"TRANSFORM transform.simple")
 
-    common.check_stream_info(userless_cursor, "test", ("test", [
+    common.check_stream_info(userless_cursor, "test", ("test", None, None,
-        topics[0]], "mg_consumer", None, None,
+        "transform.simple", stream_user, False))
-        "transform.simple", stream_user, "localhost:9092", False))
 
 
 def test_insufficient_privileges(producer, topics, connection):

tests/e2e/streams/streams_tests.py

@@ -23,20 +23,21 @@ import common
 QUERY = 0
 PARAMS = 1
 
-TRANSFORMATIONS_TO_CHECK = [
+TRANSFORMATIONS_TO_CHECK = ["transform.simple", "transform.with_parameters"]
-    "transform.simple", "transform.with_parameters"]
 
-SIMPLE_MSG = b'message'
+SIMPLE_MSG = b"message"
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
 def test_simple(producer, topics, connection, transformation):
     assert len(topics) > 0
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test "
+        cursor,
-                                 f"TOPICS {','.join(topics)} "
+        "CREATE STREAM test "
-                                 f"TRANSFORM {transformation}")
+        f"TOPICS {','.join(topics)} "
+        f"TRANSFORM {transformation}",
+    )
     common.start_stream(cursor, "test")
     time.sleep(5)
 
@@ -44,8 +45,7 @@ def test_simple(producer, topics, connection, transformation):
         producer.send(topic, SIMPLE_MSG).get(timeout=60)
 
     for topic in topics:
-        common.check_vertex_exists_with_topic_and_payload(
+        common.check_vertex_exists_with_topic_and_payload(cursor, topic, SIMPLE_MSG)
-            cursor, topic, SIMPLE_MSG)
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@@ -57,10 +57,12 @@ def test_separate_consumers(producer, topics, connection, transformation):
     for topic in topics:
         stream_name = "stream_" + topic
         stream_names.append(stream_name)
-        common.execute_and_fetch_all(cursor,
+        common.execute_and_fetch_all(
-                                     f"CREATE STREAM {stream_name} "
+            cursor,
-                                     f"TOPICS {topic} "
+            f"CREATE STREAM {stream_name} "
-                                     f"TRANSFORM {transformation}")
+            f"TOPICS {topic} "
+            f"TRANSFORM {transformation}",
+        )
 
     for stream_name in stream_names:
         common.start_stream(cursor, stream_name)
@@ -71,8 +73,7 @@ def test_separate_consumers(producer, topics, connection, transformation):
         producer.send(topic, SIMPLE_MSG).get(timeout=60)
 
     for topic in topics:
-        common.check_vertex_exists_with_topic_and_payload(
+        common.check_vertex_exists_with_topic_and_payload(cursor, topic, SIMPLE_MSG)
-            cursor, topic, SIMPLE_MSG)
 
 
 def test_start_from_last_committed_offset(producer, topics, connection):
@@ -84,17 +85,16 @@ def test_start_from_last_committed_offset(producer, topics, connection):
     # restarting Memgraph during a single workload cannot be done currently.
     assert len(topics) > 0
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test " f"TOPICS {topics[0]} " "TRANSFORM transform.simple",
-                                 "TRANSFORM transform.simple")
+    )
     common.start_stream(cursor, "test")
     time.sleep(1)
 
     producer.send(topics[0], SIMPLE_MSG).get(timeout=60)
 
-    common.check_vertex_exists_with_topic_and_payload(
+    common.check_vertex_exists_with_topic_and_payload(cursor, topics[0], SIMPLE_MSG)
-        cursor, topics[0], SIMPLE_MSG)
 
     common.stop_stream(cursor, "test")
     common.drop_stream(cursor, "test")
@@ -106,32 +106,32 @@ def test_start_from_last_committed_offset(producer, topics, connection):
     for message in messages:
         vertices_with_msg = common.execute_and_fetch_all(
             cursor,
-            "MATCH (n: MESSAGE {"
+            "MATCH (n: MESSAGE {" f"payload: '{message.decode('utf-8')}'" "}) RETURN n",
-            f"payload: '{message.decode('utf-8')}'"
+        )
-            "}) RETURN n")
 
         assert len(vertices_with_msg) == 0
 
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test " f"TOPICS {topics[0]} " "TRANSFORM transform.simple",
-                                 "TRANSFORM transform.simple")
+    )
     common.start_stream(cursor, "test")
 
     for message in messages:
-        common.check_vertex_exists_with_topic_and_payload(
+        common.check_vertex_exists_with_topic_and_payload(cursor, topics[0], message)
-            cursor, topics[0], message)
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
 def test_check_stream(producer, topics, connection, transformation):
     assert len(topics) > 0
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test "
-                                 f"TRANSFORM {transformation} "
+        f"TOPICS {topics[0]} "
-                                 "BATCH_SIZE 1")
+        f"TRANSFORM {transformation} "
+        "BATCH_SIZE 1",
+    )
     common.start_stream(cursor, "test")
     time.sleep(1)
 
@@ -143,23 +143,27 @@ def test_check_stream(producer, topics, connection, transformation):
         producer.send(topics[0], message).get(timeout=60)
 
     def check_check_stream(batch_limit):
-        assert transformation == "transform.simple" \
+        assert (
+            transformation == "transform.simple"
             or transformation == "transform.with_parameters"
+        )
         test_results = common.execute_and_fetch_all(
-            cursor, f"CHECK STREAM test BATCH_LIMIT {batch_limit}")
+            cursor, f"CHECK STREAM test BATCH_LIMIT {batch_limit}"
+        )
         assert len(test_results) == batch_limit
 
         for i in range(batch_limit):
-            message_as_str = messages[i].decode('utf-8')
+            message_as_str = messages[i].decode("utf-8")
             if transformation == "transform.simple":
-                assert f"payload: '{message_as_str}'" in \
+                assert f"payload: '{message_as_str}'" in test_results[i][QUERY]
-                    test_results[i][QUERY]
                 assert test_results[i][PARAMS] is None
             else:
-                assert test_results[i][QUERY] == ("CREATE (n:MESSAGE "
+                assert test_results[i][QUERY] == (
-                                                  "{timestamp: $timestamp, "
+                    "CREATE (n:MESSAGE "
-                                                  "payload: $payload, "
+                    "{timestamp: $timestamp, "
-                                                  "topic: $topic})")
+                    "payload: $payload, "
+                    "topic: $topic})"
+                )
                 parameters = test_results[i][PARAMS]
                 # this is not a very sofisticated test, but checks if
                 # timestamp has some kind of value
@@ -173,46 +177,53 @@ def test_check_stream(producer, topics, connection, transformation):
     common.start_stream(cursor, "test")
 
     for message in messages:
-        common.check_vertex_exists_with_topic_and_payload(
+        common.check_vertex_exists_with_topic_and_payload(cursor, topics[0], message)
-            cursor, topics[0], message)
 
 
 def test_show_streams(producer, topics, connection):
     assert len(topics) > 1
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM default_values "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM default_values "
-                                 f"TRANSFORM transform.simple "
+        f"TOPICS {topics[0]} "
-                                 f"BOOTSTRAP_SERVERS \'localhost:9092\'")
+        f"TRANSFORM transform.simple "
+        f"BOOTSTRAP_SERVERS 'localhost:9092'",
+    )
 
     consumer_group = "my_special_consumer_group"
     batch_interval = 42
     batch_size = 3
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM complex_values "
+        cursor,
-                                 f"TOPICS {','.join(topics)} "
+        "CREATE STREAM complex_values "
-                                 f"TRANSFORM transform.with_parameters "
+        f"TOPICS {','.join(topics)} "
-                                 f"CONSUMER_GROUP {consumer_group} "
+        f"TRANSFORM transform.with_parameters "
-                                 f"BATCH_INTERVAL {batch_interval} "
+        f"CONSUMER_GROUP {consumer_group} "
-                                 f"BATCH_SIZE {batch_size} ")
+        f"BATCH_INTERVAL {batch_interval} "
+        f"BATCH_SIZE {batch_size} ",
+    )
 
     assert len(common.execute_and_fetch_all(cursor, "SHOW STREAMS")) == 2
 
-    common.check_stream_info(cursor, "default_values", ("default_values", [
+    common.check_stream_info(
-        topics[0]], "mg_consumer", None, None,
+        cursor,
-        "transform.simple", None, "localhost:9092", False))
+        "default_values",
+        ("default_values", None, None, "transform.simple", None, False),
+    )
 
-    common.check_stream_info(cursor, "complex_values", (
+    common.check_stream_info(
+        cursor,
         "complex_values",
-        topics,
+        (
-        consumer_group,
+            "complex_values",
-        batch_interval,
+            batch_interval,
-        batch_size,
+            batch_size,
-        "transform.with_parameters",
+            "transform.with_parameters",
-        None,
+            None,
-        "localhost:9092",
+            False,
-        False))
+        ),
+    )
 
 
 @pytest.mark.parametrize("operation", ["START", "STOP"])
@@ -229,14 +240,16 @@ def test_start_and_stop_during_check(producer, topics, connection, operation):
     assert len(topics) > 1
     assert operation == "START" or operation == "STOP"
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test_stream "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test_stream "
-                                 f"TRANSFORM transform.simple")
+        f"TOPICS {topics[0]} "
+        f"TRANSFORM transform.simple",
+    )
 
-    check_counter = Value('i', 0)
+    check_counter = Value("i", 0)
-    check_result_len = Value('i', 0)
+    check_result_len = Value("i", 0)
-    operation_counter = Value('i', 0)
+    operation_counter = Value("i", 0)
 
     CHECK_BEFORE_EXECUTE = 1
     CHECK_AFTER_FETCHALL = 2
@@ -249,8 +262,7 @@ def test_start_and_stop_during_check(producer, topics, connection, operation):
         connection = common.connect()
         cursor = connection.cursor()
         counter.value = CHECK_BEFORE_EXECUTE
-        result = common.execute_and_fetch_all(
+        result = common.execute_and_fetch_all(cursor, "CHECK STREAM test_stream")
-            cursor, "CHECK STREAM test_stream")
         result_len.value = len(result)
         counter.value = CHECK_AFTER_FETCHALL
         if len(result) > 0 and "payload: 'message'" in result[0][QUERY]:
@@ -271,8 +283,7 @@ def test_start_and_stop_during_check(producer, topics, connection, operation):
         cursor = connection.cursor()
         counter.value = OP_BEFORE_EXECUTE
         try:
-            common.execute_and_fetch_all(
+            common.execute_and_fetch_all(cursor, f"{operation} STREAM test_stream")
-                cursor, f"{operation} STREAM test_stream")
             counter.value = OP_AFTER_FETCHALL
         except mgclient.DatabaseError as e:
             if "Kafka consumer test_stream is already stopped" in str(e):
@@ -283,28 +294,27 @@ def test_start_and_stop_during_check(producer, topics, connection, operation):
             counter.value = OP_UNEXPECTED_EXCEPTION
 
     check_stream_proc = Process(
-        target=call_check, daemon=True, args=(check_counter, check_result_len))
+        target=call_check, daemon=True, args=(check_counter, check_result_len)
-    operation_proc = Process(target=call_operation,
+    )
-                             daemon=True, args=(operation_counter,))
+    operation_proc = Process(
+        target=call_operation, daemon=True, args=(operation_counter,)
+    )
 
     try:
         check_stream_proc.start()
 
         time.sleep(0.5)
 
-        assert common.timed_wait(
+        assert common.timed_wait(lambda: check_counter.value == CHECK_BEFORE_EXECUTE)
-            lambda: check_counter.value == CHECK_BEFORE_EXECUTE)
+        assert common.timed_wait(lambda: common.get_is_running(cursor, "test_stream"))
-        assert common.timed_wait(
+        assert check_counter.value == CHECK_BEFORE_EXECUTE, (
-            lambda: common.get_is_running(cursor, "test_stream"))
+            "SHOW STREAMS " "was blocked until the end of CHECK STREAM"
-        assert check_counter.value == CHECK_BEFORE_EXECUTE, "SHOW STREAMS " \
+        )
-            "was blocked until the end of CHECK STREAM"
         operation_proc.start()
-        assert common.timed_wait(
+        assert common.timed_wait(lambda: operation_counter.value == OP_BEFORE_EXECUTE)
-            lambda: operation_counter.value == OP_BEFORE_EXECUTE)
 
         producer.send(topics[0], SIMPLE_MSG).get(timeout=60)
-        assert common.timed_wait(
+        assert common.timed_wait(lambda: check_counter.value > CHECK_AFTER_FETCHALL)
-            lambda: check_counter.value > CHECK_AFTER_FETCHALL)
         assert check_counter.value == CHECK_CORRECT_RESULT
         assert check_result_len.value == 1
         check_stream_proc.join()
@@ -330,10 +340,12 @@ def test_check_already_started_stream(topics, connection):
     assert len(topics) > 0
     cursor = connection.cursor()
 
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM started_stream "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM started_stream "
-                                 f"TRANSFORM transform.simple")
+        f"TOPICS {topics[0]} "
+        f"TRANSFORM transform.simple",
+    )
     common.start_stream(cursor, "started_stream")
 
     with pytest.raises(mgclient.DatabaseError):
@@ -342,52 +354,52 @@ def test_check_already_started_stream(topics, connection):
 
 def test_start_checked_stream_after_timeout(topics, connection):
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test_stream "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test_stream "
-                                 f"TRANSFORM transform.simple")
+        f"TOPICS {topics[0]} "
+        f"TRANSFORM transform.simple",
+    )
 
     timeout_ms = 2000
 
     def call_check():
         common.execute_and_fetch_all(
-            common.connect().cursor(),
+            common.connect().cursor(), f"CHECK STREAM test_stream TIMEOUT {timeout_ms}"
-            f"CHECK STREAM test_stream TIMEOUT {timeout_ms}")
+        )
 
     check_stream_proc = Process(target=call_check, daemon=True)
 
     start = time.time()
     check_stream_proc.start()
-    assert common.timed_wait(
+    assert common.timed_wait(lambda: common.get_is_running(cursor, "test_stream"))
-        lambda: common.get_is_running(cursor, "test_stream"))
     common.start_stream(cursor, "test_stream")
     end = time.time()
 
-    assert (end - start) < 1.3 * \
+    assert (end - start) < 1.3 * timeout_ms, "The START STREAM was blocked too long"
-        timeout_ms, "The START STREAM was blocked too long"
     assert common.get_is_running(cursor, "test_stream")
     common.stop_stream(cursor, "test_stream")
 
 
 def test_restart_after_error(producer, topics, connection):
     cursor = connection.cursor()
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test_stream "
+        cursor,
-                                 f"TOPICS {topics[0]} "
+        "CREATE STREAM test_stream "
-                                 f"TRANSFORM transform.query")
+        f"TOPICS {topics[0]} "
+        f"TRANSFORM transform.query",
+    )
 
     common.start_stream(cursor, "test_stream")
     time.sleep(1)
 
     producer.send(topics[0], SIMPLE_MSG).get(timeout=60)
-    assert common.timed_wait(
+    assert common.timed_wait(lambda: not common.get_is_running(cursor, "test_stream"))
-        lambda: not common.get_is_running(cursor, "test_stream"))
 
     common.start_stream(cursor, "test_stream")
     time.sleep(1)
-    producer.send(topics[0], b'CREATE (n:VERTEX { id : 42 })')
+    producer.send(topics[0], b"CREATE (n:VERTEX { id : 42 })")
-    assert common.check_one_result_row(
+    assert common.check_one_result_row(cursor, "MATCH (n:VERTEX { id : 42 }) RETURN n")
-        cursor, "MATCH (n:VERTEX { id : 42 }) RETURN n")
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@@ -395,11 +407,13 @@ def test_bootstrap_server(producer, topics, connection, transformation):
     assert len(topics) > 0
     cursor = connection.cursor()
     local = "localhost:9092"
-    common.execute_and_fetch_all(cursor,
+    common.execute_and_fetch_all(
-                                 "CREATE STREAM test "
+        cursor,
-                                 f"TOPICS {','.join(topics)} "
+        "CREATE STREAM test "
-                                 f"TRANSFORM {transformation} "
+        f"TOPICS {','.join(topics)} "
-                                 f"BOOTSTRAP_SERVERS \'{local}\'")
+        f"TRANSFORM {transformation} "
+        f"BOOTSTRAP_SERVERS '{local}'",
+    )
     common.start_stream(cursor, "test")
     time.sleep(5)
 
@@ -407,8 +421,7 @@ def test_bootstrap_server(producer, topics, connection, transformation):
         producer.send(topic, SIMPLE_MSG).get(timeout=60)
 
     for topic in topics:
-        common.check_vertex_exists_with_topic_and_payload(
+        common.check_vertex_exists_with_topic_and_payload(cursor, topic, SIMPLE_MSG)
-            cursor, topic, SIMPLE_MSG)
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@@ -416,11 +429,13 @@ def test_bootstrap_server_empty(producer, topics, connection, transformation):
     assert len(topics) > 0
     cursor = connection.cursor()
     with pytest.raises(mgclient.DatabaseError):
-        common.execute_and_fetch_all(cursor,
+        common.execute_and_fetch_all(
-                                     "CREATE STREAM test "
+            cursor,
-                                     f"TOPICS {','.join(topics)} "
+            "CREATE STREAM test "
-                                     f"TRANSFORM {transformation} "
+            f"TOPICS {','.join(topics)} "
-                                     "BOOTSTRAP_SERVERS ''")
+            f"TRANSFORM {transformation} "
+            "BOOTSTRAP_SERVERS ''",
+        )
 
 
 if __name__ == "__main__":

tests/manual/single_query.cpp

@@ -31,8 +31,7 @@ int main(int argc, char *argv[]) {
   utils::OnScopeExit([&data_directory] { std::filesystem::remove_all(data_directory); });
 
   utils::license::global_license_checker.EnableTesting();
-  query::InterpreterContext interpreter_context{&db, query::InterpreterConfig{}, data_directory,
+  query::InterpreterContext interpreter_context{&db, query::InterpreterConfig{}, data_directory};
-                                                "non existing bootstrap servers"};
   query::Interpreter interpreter{&interpreter_context};
 
   ResultStreamFaker stream(&db);

tests/unit/CMakeLists.txt

@@ -53,7 +53,7 @@ target_link_libraries(storage_test_utils mg-storage-v2)
 # Test integrations-kafka
 
 add_library(kafka-mock STATIC kafka_mock.cpp)
-target_link_libraries(kafka-mock mg-utils librdkafka++ librdkafka Threads::Threads zlib gtest)
+target_link_libraries(kafka-mock mg-utils librdkafka++ librdkafka Threads::Threads gtest)
 # Include directories are intentionally not set, because kafka-mock isn't meant to be used apart from unit tests
 
 add_unit_test(integrations_kafka_consumer.cpp kafka_mock.cpp)

tests/unit/cypher_main_visitor.cpp

@@ -52,6 +52,7 @@
 #include "query/typed_value.hpp"
 
 #include "utils/string.hpp"
+#include "utils/variant.hpp"
 
 using namespace query;
 using namespace query::frontend;
@@ -2943,6 +2944,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithMemoryUnlimited) {
 namespace {
 template <typename TException = SyntaxException>
 void TestInvalidQuery(const auto &query, Base &ast_generator) {
+  SCOPED_TRACE(query);
   EXPECT_THROW(ast_generator.ParseQuery(query), TException) << query;
 }
 
@@ -3564,7 +3566,9 @@ void ValidateMostlyEmptyStreamQuery(Base &ast_generator, const std::string &quer
   ASSERT_NE(parsed_query, nullptr);
   EXPECT_EQ(parsed_query->action_, action);
   EXPECT_EQ(parsed_query->stream_name_, stream_name);
-  EXPECT_TRUE(parsed_query->topic_names_.empty());
+  auto topic_names = std::get_if<Expression *>(&parsed_query->topic_names_);
+  EXPECT_TRUE(topic_names);
+  EXPECT_FALSE(*topic_names);
   EXPECT_TRUE(parsed_query->transform_name_.empty());
   EXPECT_TRUE(parsed_query->consumer_group_.empty());
   EXPECT_EQ(parsed_query->batch_interval_, nullptr);
@@ -3642,12 +3646,17 @@ void ValidateCreateStreamQuery(Base &ast_generator, const std::string &query_str
                                const std::string_view consumer_group, const std::optional<TypedValue> &batch_interval,
                                const std::optional<TypedValue> &batch_size,
                                const std::string_view bootstrap_servers = "") {
+  SCOPED_TRACE(query_string);
   StreamQuery *parsed_query{nullptr};
   ASSERT_NO_THROW(parsed_query = dynamic_cast<StreamQuery *>(ast_generator.ParseQuery(query_string))) << query_string;
   ASSERT_NE(parsed_query, nullptr);
   EXPECT_EQ(parsed_query->stream_name_, stream_name);
 
-  EXPECT_EQ(parsed_query->topic_names_, topic_names);
+  std::visit(utils::Overloaded{
+                 [&](Expression *expression) { ast_generator.CheckLiteral(expression, utils::Join(topic_names, ",")); },
+                 [&](const std::vector<std::string> &topic_name_list) { EXPECT_EQ(topic_name_list, topic_names); }},
+             parsed_query->topic_names_);
+
   EXPECT_EQ(parsed_query->transform_name_, transform_name);
   EXPECT_EQ(parsed_query->consumer_group_, consumer_group);
   EXPECT_NO_FATAL_FAILURE(CheckOptionalExpression(ast_generator, parsed_query->batch_interval_, batch_interval));
@@ -3663,34 +3672,32 @@ void ValidateCreateStreamQuery(Base &ast_generator, const std::string &query_str
 TEST_P(CypherMainVisitorTest, CreateStream) {
   auto &ast_generator = *GetParam();
 
-  TestInvalidQuery("CREATE STREAM", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM", ast_generator);
-  TestInvalidQuery("CREATE STREAM invalid stream name TOPICS topic1 TRANSFORM transform", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM invalid stream name TOPICS topic1 TRANSFORM transform", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS invalid topic name TRANSFORM transform", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS invalid topic name TRANSFORM transform", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM invalid transform name", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM invalid transformation name", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TRANSFORM transform", ast_generator);
+  // required configs are missing
-  TestInvalidQuery("CREATE STREAM stream TOPICS TRANSFORM transform", ast_generator);
+  TestInvalidQuery<SemanticException>("CREATE KAFKA STREAM stream TRANSFORM transform", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS TRANSFORM transform", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM", ast_generator);
+  // required configs are missing
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform CONSUMER_GROUP", ast_generator);
+  TestInvalidQuery<SemanticException>("CREATE KAFKA STREAM stream TOPICS topic1", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform CONSUMER_GROUP invalid consumer group",
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform CONSUMER_GROUP", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform CONSUMER_GROUP invalid consumer group",
                    ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INTERVAL", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INTERVAL", ast_generator);
   TestInvalidQuery<SemanticException>(
-      "CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INTERVAL 'invalid interval'", ast_generator);
+      "CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INTERVAL 'invalid interval'", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE", ast_generator);
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE", ast_generator);
   TestInvalidQuery<SemanticException>(
-      "CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE 'invalid size'", ast_generator);
+      "CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE 'invalid size'", ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE 2 BATCH_INTERVAL 3",
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INVERVAL 2 CONSUMER_GROUP Gru",
                    ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_INVERVAL 2 CONSUMER_GROUP Gru",
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1, TRANSFORM transform BATCH_SIZE 2 CONSUMER_GROUP Gru",
                    ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BATCH_SIZE 2 CONSUMER_GROUP Gru",
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BOOTSTRAP_SERVERS localhost:9092",
                    ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1, TRANSFORM transform BATCH_SIZE 2 CONSUMER_GROUP Gru",
+  TestInvalidQuery("CREATE KAFKA STREAM stream TOPICS topic1 TRANSFORM transform BOOTSTRAP_SERVERS", ast_generator);
-                   ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BOOTSTRAP_SERVERS localhost:9092",
-                   ast_generator);
-  TestInvalidQuery("CREATE STREAM stream TOPICS topic1 TRANSFORM transform BOOTSTRAP_SERVERS", ast_generator);
 
   const std::vector<std::string> topic_names{"topic1_name.with_dot", "topic1_name.with_multiple.dots",
                                              "topic-name.with-multiple.dots-and-dashes"};
@@ -3709,41 +3716,41 @@ TEST_P(CypherMainVisitorTest, CreateStream) {
 
     ValidateCreateStreamQuery(
         ast_generator,
-        fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {}", kStreamName, topic_names_as_str, kTransformName),
+        fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {}", kStreamName, topic_names_as_str, kTransformName),
         kStreamName, topic_names, kTransformName, "", std::nullopt, std::nullopt);
 
     ValidateCreateStreamQuery(ast_generator,
-                              fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} ", kStreamName,
+                              fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} ",
-                                          topic_names_as_str, kTransformName, kConsumerGroup),
+                                          kStreamName, topic_names_as_str, kTransformName, kConsumerGroup),
                               kStreamName, topic_names, kTransformName, kConsumerGroup, std::nullopt, std::nullopt);
 
     ValidateCreateStreamQuery(ast_generator,
-                              fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} BATCH_INTERVAL {}", kStreamName,
+                              fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} BATCH_INTERVAL {}",
-                                          topic_names_as_str, kTransformName, kBatchInterval),
+                                          kStreamName, topic_names_as_str, kTransformName, kBatchInterval),
                               kStreamName, topic_names, kTransformName, "", batch_interval_value, std::nullopt);
 
     ValidateCreateStreamQuery(ast_generator,
-                              fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} BATCH_SIZE {}", kStreamName,
+                              fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} BATCH_SIZE {}", kStreamName,
                                           topic_names_as_str, kTransformName, kBatchSize),
                               kStreamName, topic_names, kTransformName, "", std::nullopt, batch_size_value);
 
     ValidateCreateStreamQuery(
         ast_generator,
-        fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {}",
+        fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {}",
                     kStreamName, topic_names_as_str, kTransformName, kConsumerGroup, kBatchInterval, kBatchSize),
         kStreamName, topic_names, kTransformName, kConsumerGroup, batch_interval_value, batch_size_value);
     using namespace std::string_literals;
     const auto host1 = "localhost:9094"s;
     ValidateCreateStreamQuery(
         ast_generator,
-        fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {} "
+        fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {} "
                     "BOOTSTRAP_SERVERS '{}'",
                     kStreamName, topic_names_as_str, kTransformName, kConsumerGroup, kBatchInterval, kBatchSize, host1),
         kStreamName, topic_names, kTransformName, kConsumerGroup, batch_interval_value, batch_size_value, host1);
     const auto host2 = "localhost:9094,localhost:1994,168.1.1.256:345"s;
     ValidateCreateStreamQuery(
         ast_generator,
-        fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {} "
+        fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {} BATCH_INTERVAL {} BATCH_SIZE {} "
                     "BOOTSTRAP_SERVERS '{}'",
                     kStreamName, topic_names_as_str, kTransformName, kConsumerGroup, kBatchInterval, kBatchSize, host2),
         kStreamName, topic_names, kTransformName, kConsumerGroup, batch_interval_value, batch_size_value, host2);
@@ -3758,8 +3765,8 @@ TEST_P(CypherMainVisitorTest, CreateStream) {
   auto check_consumer_group = [&](const std::string_view consumer_group) {
     const std::string kTopicName{"topic1"};
     ValidateCreateStreamQuery(ast_generator,
-                              fmt::format("CREATE STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {}", kStreamName,
+                              fmt::format("CREATE KAFKA STREAM {} TOPICS {} TRANSFORM {} CONSUMER_GROUP {}",
-                                          kTopicName, kTransformName, consumer_group),
+                                          kStreamName, kTopicName, kTransformName, consumer_group),
                               kStreamName, {kTopicName}, kTransformName, consumer_group, std::nullopt, std::nullopt);
   };
 

tests/unit/integrations_kafka_consumer.cpp

@@ -51,6 +51,7 @@ struct ConsumerTest : public ::testing::Test {
         .consumer_name = "Consumer" + test_name,
         .topics = {kTopicName},
         .consumer_group = "ConsumerGroup " + test_name,
+        .bootstrap_servers = cluster.Bootstraps(),
         .batch_interval = std::nullopt,
         .batch_size = std::nullopt,
     };
@@ -76,8 +77,7 @@ struct ConsumerTest : public ::testing::Test {
       }
     };
 
-    auto consumer =
+    auto consumer = std::make_unique<Consumer>(std::move(info), std::move(consumer_function_wrapper));
-        std::make_unique<Consumer>(cluster.Bootstraps(), std::move(info), std::move(consumer_function_wrapper));
     int sent_messages{1};
     SeedTopicWithInt(kTopicName, sent_messages);
 
@@ -169,7 +169,7 @@ TEST_F(ConsumerTest, BatchInterval) {
 }
 
 TEST_F(ConsumerTest, StartStop) {
-  Consumer consumer{cluster.Bootstraps(), CreateDefaultConsumerInfo(), kDummyConsumerFunction};
+  Consumer consumer{CreateDefaultConsumerInfo(), kDummyConsumerFunction};
 
   auto start = [&consumer](const bool use_conditional) {
     if (use_conditional) {
@@ -279,41 +279,41 @@ TEST_F(ConsumerTest, BatchSize) {
 
 TEST_F(ConsumerTest, InvalidBootstrapServers) {
   auto info = CreateDefaultConsumerInfo();
+  info.bootstrap_servers = "non.existing.host:9092";
 
-  EXPECT_THROW(Consumer("non.existing.host:9092", std::move(info), kDummyConsumerFunction),
+  EXPECT_THROW(Consumer(std::move(info), kDummyConsumerFunction), ConsumerFailedToInitializeException);
-               ConsumerFailedToInitializeException);
 }
 
 TEST_F(ConsumerTest, InvalidTopic) {
   auto info = CreateDefaultConsumerInfo();
   info.topics = {"Non existing topic"};
-  EXPECT_THROW(Consumer(cluster.Bootstraps(), std::move(info), kDummyConsumerFunction), TopicNotFoundException);
+  EXPECT_THROW(Consumer(std::move(info), kDummyConsumerFunction), TopicNotFoundException);
 }
 
 TEST_F(ConsumerTest, InvalidBatchInterval) {
   auto info = CreateDefaultConsumerInfo();
 
   info.batch_interval = std::chrono::milliseconds{0};
-  EXPECT_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
+  EXPECT_THROW(Consumer(info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
 
   info.batch_interval = std::chrono::milliseconds{-1};
-  EXPECT_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
+  EXPECT_THROW(Consumer(info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
 
   info.batch_interval = std::chrono::milliseconds{1};
-  EXPECT_NO_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction));
+  EXPECT_NO_THROW(Consumer(info, kDummyConsumerFunction));
 }
 
 TEST_F(ConsumerTest, InvalidBatchSize) {
   auto info = CreateDefaultConsumerInfo();
 
   info.batch_size = 0;
-  EXPECT_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
+  EXPECT_THROW(Consumer(info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
 
   info.batch_size = -1;
-  EXPECT_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
+  EXPECT_THROW(Consumer(info, kDummyConsumerFunction), ConsumerFailedToInitializeException);
 
   info.batch_size = 1;
-  EXPECT_NO_THROW(Consumer(cluster.Bootstraps(), info, kDummyConsumerFunction));
+  EXPECT_NO_THROW(Consumer(info, kDummyConsumerFunction));
 }
 
 TEST_F(ConsumerTest, DISABLED_StartsFromPreviousOffset) {
@@ -348,7 +348,7 @@ TEST_F(ConsumerTest, DISABLED_StartsFromPreviousOffset) {
                         std::string_view{kMessagePrefix + std::to_string(received_message_count + sent_messages)});
     }
     auto expected_total_messages = received_message_count + batch_count;
-    auto consumer = std::make_unique<Consumer>(cluster.Bootstraps(), ConsumerInfo{info}, consumer_function);
+    auto consumer = std::make_unique<Consumer>(ConsumerInfo{info}, consumer_function);
     ASSERT_FALSE(consumer->IsRunning());
     consumer->Start();
     const auto start = std::chrono::steady_clock::now();
@@ -419,7 +419,7 @@ TEST_F(ConsumerTest, CheckMethodWorks) {
 }
 
 TEST_F(ConsumerTest, CheckMethodTimeout) {
-  Consumer consumer{cluster.Bootstraps(), CreateDefaultConsumerInfo(), kDummyConsumerFunction};
+  Consumer consumer{CreateDefaultConsumerInfo(), kDummyConsumerFunction};
 
   std::chrono::milliseconds timeout{3000};
 
@@ -433,7 +433,7 @@ TEST_F(ConsumerTest, CheckMethodTimeout) {
 }
 
 TEST_F(ConsumerTest, CheckWithInvalidTimeout) {
-  Consumer consumer{cluster.Bootstraps(), CreateDefaultConsumerInfo(), kDummyConsumerFunction};
+  Consumer consumer{CreateDefaultConsumerInfo(), kDummyConsumerFunction};
 
   const auto start = std::chrono::steady_clock::now();
   EXPECT_THROW(consumer.Check(std::chrono::milliseconds{0}, std::nullopt, kDummyConsumerFunction),
@@ -448,7 +448,7 @@ TEST_F(ConsumerTest, CheckWithInvalidTimeout) {
 }
 
 TEST_F(ConsumerTest, CheckWithInvalidBatchSize) {
-  Consumer consumer{cluster.Bootstraps(), CreateDefaultConsumerInfo(), kDummyConsumerFunction};
+  Consumer consumer{CreateDefaultConsumerInfo(), kDummyConsumerFunction};
 
   const auto start = std::chrono::steady_clock::now();
   EXPECT_THROW(consumer.Check(std::nullopt, 0, kDummyConsumerFunction), ConsumerCheckFailedException);
@@ -482,9 +482,9 @@ TEST_F(ConsumerTest, ConsumerStatus) {
     EXPECT_EQ(topics[1], info.topics[1]);
   };
 
-  Consumer consumer{cluster.Bootstraps(),
+  Consumer consumer{
-                    ConsumerInfo{kConsumerName, topics, kConsumerGroupName, kBatchInterval, kBatchSize},
+      ConsumerInfo{kConsumerName, topics, kConsumerGroupName, cluster.Bootstraps(), kBatchInterval, kBatchSize},
-                    kDummyConsumerFunction};
+      kDummyConsumerFunction};
 
   check_info(consumer.Info());
   consumer.Start();

tests/unit/interpreter.cpp

@@ -53,8 +53,7 @@ auto ToEdgeList(const communication::bolt::Value &v) {
 struct InterpreterFaker {
   InterpreterFaker(storage::Storage *db, const query::InterpreterConfig config,
                    const std::filesystem::path &data_directory)
-      : interpreter_context(db, config, data_directory, "not used bootstrap servers"),
+      : interpreter_context(db, config, data_directory), interpreter(&interpreter_context) {
-        interpreter(&interpreter_context) {
     interpreter_context.auth_checker = &auth_checker;
   }
 

tests/unit/mgp_kafka_c_api.cpp

@@ -135,7 +135,7 @@ class MgpApiTest : public ::testing::Test {
     auto v = utils::pmr::vector<mgp_message>(utils::NewDeleteResource());
     v.reserve(expected.size());
     std::transform(msgs_storage_.begin(), msgs_storage_.end(), std::back_inserter(v),
-                   [](auto &msgs) { return mgp_message{&msgs}; });
+                   [](auto &msgs) { return mgp_message{msgs}; });
     return v;
   }
 

tests/unit/query_dump.cpp

@@ -203,7 +203,7 @@ DatabaseState GetState(storage::Storage *db) {
 
 auto Execute(storage::Storage *db, const std::string &query) {
   auto data_directory = std::filesystem::temp_directory_path() / "MG_tests_unit_query_dump";
-  query::InterpreterContext context(db, query::InterpreterConfig{}, data_directory, "non existing bootstrap servers");
+  query::InterpreterContext context(db, query::InterpreterConfig{}, data_directory);
   query::Interpreter interpreter(&context);
   ResultStreamFaker stream(db);
 
@@ -746,9 +746,7 @@ TEST(DumpTest, ExecuteDumpDatabase) {
 class StatefulInterpreter {
  public:
   explicit StatefulInterpreter(storage::Storage *db)
-      : db_(db),
+      : db_(db), context_(db_, query::InterpreterConfig{}, data_directory_), interpreter_(&context_) {}
-        context_(db_, query::InterpreterConfig{}, data_directory_, "non existing bootstrap servers"),
-        interpreter_(&context_) {}
 
   auto Execute(const std::string &query) {
     ResultStreamFaker stream(db_);

tests/unit/query_plan_edge_cases.cpp

@@ -35,7 +35,7 @@ class QueryExecution : public testing::Test {
 
   void SetUp() {
     db_.emplace();
-    interpreter_context_.emplace(&*db_, query::InterpreterConfig{}, data_directory, "non existing bootstrap servers");
+    interpreter_context_.emplace(&*db_, query::InterpreterConfig{}, data_directory);
     interpreter_.emplace(&*interpreter_context_);
   }
 

tests/unit/query_streams.cpp

@@ -19,12 +19,12 @@
 #include "kafka_mock.hpp"
 #include "query/config.hpp"
 #include "query/interpreter.hpp"
-#include "query/streams.hpp"
+#include "query/stream/streams.hpp"
 #include "storage/v2/storage.hpp"
 
 using Streams = query::Streams;
-using StreamInfo = query::StreamInfo;
+using StreamInfo = query::KafkaStream::StreamInfo;
-using StreamStatus = query::StreamStatus;
+using StreamStatus = query::StreamStatus<query::KafkaStream>;
 namespace {
 const static std::string kTopicName{"TrialTopic"};
 
@@ -32,25 +32,13 @@ struct StreamCheckData {
   std::string name;
   StreamInfo info;
   bool is_running;
+  std::optional<std::string> owner;
 };
 
 std::string GetDefaultStreamName() {
   return std::string{::testing::UnitTest::GetInstance()->current_test_info()->name()};
 }
 
-StreamInfo CreateDefaultStreamInfo() {
-  return StreamInfo{
-      .topics = {kTopicName},
-      .consumer_group = "ConsumerGroup " + GetDefaultStreamName(),
-      .batch_interval = std::nullopt,
-      .batch_size = std::nullopt,
-      .transformation_name = "not used in the tests",
-      .owner = std::nullopt,
-  };
-}
-
-StreamCheckData CreateDefaultStreamCheckData() { return {GetDefaultStreamName(), CreateDefaultStreamInfo(), false}; }
-
 std::filesystem::path GetCleanDataDirectory() {
   const auto path = std::filesystem::temp_directory_path() / "query-streams";
   std::filesystem::remove_all(path);
@@ -71,14 +59,11 @@ class StreamsTest : public ::testing::Test {
   // Streams constructor.
   // InterpreterContext::auth_checker_ is used in the Streams object, but only in the message processing part. Because
   // these tests don't send any messages, the auth_checker_ pointer can be left as nullptr.
-  query::InterpreterContext interpreter_context_{&db_, query::InterpreterConfig{}, data_directory_,
+  query::InterpreterContext interpreter_context_{&db_, query::InterpreterConfig{}, data_directory_};
-                                                 "dont care bootstrap servers"};
   std::filesystem::path streams_data_directory_{data_directory_ / "separate-dir-for-test"};
   std::optional<Streams> streams_;
 
-  void ResetStreamsObject() {
+  void ResetStreamsObject() { streams_.emplace(&interpreter_context_, streams_data_directory_); }
-    streams_.emplace(&interpreter_context_, mock_cluster_.Bootstraps(), streams_data_directory_);
-  }
 
   void CheckStreamStatus(const StreamCheckData &check_data) {
     SCOPED_TRACE(fmt::format("Checking status of '{}'", check_data.name));
@@ -87,13 +72,9 @@ class StreamsTest : public ::testing::Test {
                            [&check_data](const auto &stream_status) { return stream_status.name == check_data.name; });
     ASSERT_NE(it, stream_statuses.end());
     const auto &status = *it;
-    // the order don't have to be strictly the same, but based on the implementation it shouldn't change
+    EXPECT_EQ(check_data.info.common_info.batch_interval, status.info.batch_interval);
-    EXPECT_TRUE(std::equal(check_data.info.topics.begin(), check_data.info.topics.end(), status.info.topics.begin(),
+    EXPECT_EQ(check_data.info.common_info.batch_size, status.info.batch_size);
-                           status.info.topics.end()));
+    EXPECT_EQ(check_data.info.common_info.transformation_name, status.info.transformation_name);
-    EXPECT_EQ(check_data.info.consumer_group, status.info.consumer_group);
-    EXPECT_EQ(check_data.info.batch_interval, status.info.batch_interval);
-    EXPECT_EQ(check_data.info.batch_size, status.info.batch_size);
-    EXPECT_EQ(check_data.info.transformation_name, status.info.transformation_name);
     EXPECT_EQ(check_data.is_running, status.is_running);
   }
 
@@ -107,6 +88,21 @@ class StreamsTest : public ::testing::Test {
     check_data.is_running = false;
   }
 
+  StreamInfo CreateDefaultStreamInfo() {
+    return StreamInfo{.common_info{
+                          .batch_interval = std::nullopt,
+                          .batch_size = std::nullopt,
+                          .transformation_name = "not used in the tests",
+                      },
+                      .topics = {kTopicName},
+                      .consumer_group = "ConsumerGroup " + GetDefaultStreamName(),
+                      .bootstrap_servers = mock_cluster_.Bootstraps()};
+  }
+
+  StreamCheckData CreateDefaultStreamCheckData() {
+    return {GetDefaultStreamName(), CreateDefaultStreamInfo(), false, std::nullopt};
+  }
+
   void Clear() {
     if (!std::filesystem::exists(data_directory_)) return;
     std::filesystem::remove_all(data_directory_);
@@ -115,7 +111,7 @@ class StreamsTest : public ::testing::Test {
 
 TEST_F(StreamsTest, SimpleStreamManagement) {
   auto check_data = CreateDefaultStreamCheckData();
-  streams_->Create(check_data.name, check_data.info);
+  streams_->Create<query::KafkaStream>(check_data.name, check_data.info, check_data.owner);
   EXPECT_NO_FATAL_FAILURE(CheckStreamStatus(check_data));
 
   EXPECT_NO_THROW(streams_->Start(check_data.name));
@@ -141,10 +137,10 @@ TEST_F(StreamsTest, SimpleStreamManagement) {
 TEST_F(StreamsTest, CreateAlreadyExisting) {
   auto stream_info = CreateDefaultStreamInfo();
   auto stream_name = GetDefaultStreamName();
-  streams_->Create(stream_name, stream_info);
+  streams_->Create<query::KafkaStream>(stream_name, stream_info, std::nullopt);
 
   try {
-    streams_->Create(stream_name, stream_info);
+    streams_->Create<query::KafkaStream>(stream_name, stream_info, std::nullopt);
     FAIL() << "Creating already existing stream should throw\n";
   } catch (query::StreamsException &exception) {
     EXPECT_EQ(exception.what(), fmt::format("Stream already exists with name '{}'", stream_name));
@@ -155,7 +151,7 @@ TEST_F(StreamsTest, DropNotExistingStream) {
   const auto stream_info = CreateDefaultStreamInfo();
   const auto stream_name = GetDefaultStreamName();
   const std::string not_existing_stream_name{"ThisDoesn'tExists"};
-  streams_->Create(stream_name, stream_info);
+  streams_->Create<query::KafkaStream>(stream_name, stream_info, std::nullopt);
 
   try {
     streams_->Drop(not_existing_stream_name);
@@ -182,18 +178,18 @@ TEST_F(StreamsTest, RestoreStreams) {
     stream_check_data.name += iteration_postfix;
     stream_info.topics[0] += iteration_postfix;
     stream_info.consumer_group += iteration_postfix;
-    stream_info.transformation_name += iteration_postfix;
+    stream_info.common_info.transformation_name += iteration_postfix;
     if (i > 0) {
-      stream_info.batch_interval = std::chrono::milliseconds((i + 1) * 10);
+      stream_info.common_info.batch_interval = std::chrono::milliseconds((i + 1) * 10);
-      stream_info.batch_size = 1000 + i;
+      stream_info.common_info.batch_size = 1000 + i;
-      stream_info.owner = std::string{"owner"} + iteration_postfix;
+      stream_check_data.owner = std::string{"owner"} + iteration_postfix;
     }
 
     mock_cluster_.CreateTopic(stream_info.topics[0]);
   }
-  stream_check_datas[1].info.batch_interval = {};
+  stream_check_datas[1].info.common_info.batch_interval = {};
-  stream_check_datas[2].info.batch_size = {};
+  stream_check_datas[2].info.common_info.batch_size = {};
-  stream_check_datas[3].info.owner = {};
+  stream_check_datas[3].owner = {};
 
   const auto check_restore_logic = [&stream_check_datas, this]() {
     // Reset the Streams object to trigger reloading
@@ -210,7 +206,7 @@ TEST_F(StreamsTest, RestoreStreams) {
   EXPECT_TRUE(streams_->GetStreamInfo().empty());
 
   for (auto &check_data : stream_check_datas) {
-    streams_->Create(check_data.name, check_data.info);
+    streams_->Create<query::KafkaStream>(check_data.name, check_data.info, check_data.owner);
   }
   {
     SCOPED_TRACE("After streams are created");
@@ -246,7 +242,7 @@ TEST_F(StreamsTest, RestoreStreams) {
 TEST_F(StreamsTest, CheckWithTimeout) {
   const auto stream_info = CreateDefaultStreamInfo();
   const auto stream_name = GetDefaultStreamName();
-  streams_->Create(stream_name, stream_info);
+  streams_->Create<query::KafkaStream>(stream_name, stream_info, std::nullopt);
 
   std::chrono::milliseconds timeout{3000};