Improve e2e Kafka testing

In and only in RelWithDebInfo mode the access of the maybe_unused
variable results in segfaults, this change is making sure that does no
happen ever if the maybe_unused variable is nullopt without changing the
overall logic.

.github/workflows/release_debian10.yaml

@@ -178,7 +178,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
+    runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}

libs/setup.sh

@@ -259,7 +259,7 @@ repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "
 
 # jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756
 JEMALLOC_COMMIT_VERSION="5.2.1"
-repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
+repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
 
 # this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
 pushd jemalloc

licenses/BSL.txt

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT:   You may use the Licensed Work in accordance with the
                         3. using the Licensed Work to create a work or solution
                         which competes (or might reasonably be expected to
                         compete) with the Licensed Work.
-CHANGE DATE:            2027-30-10
+CHANGE DATE:            2027-08-12
 CHANGE LICENSE:         Apache License, Version 2.0
 
 For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

query_modules/schema.cpp

@@ -108,31 +108,83 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
   record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
 }
 
+struct Property {
+  std::string name;
+  mgp::Value value;
+
+  Property(const std::string &name, mgp::Value &&value) : name(name), value(std::move(value)) {}
+};
+
+struct LabelsHash {
+  std::size_t operator()(const std::set<std::string> &set) const {
+    std::size_t seed = set.size();
+    for (const auto &i : set) {
+      seed ^= std::hash<std::string>{}(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+    }
+    return seed;
+  }
+};
+
+struct LabelsComparator {
+  bool operator()(const std::set<std::string> &lhs, const std::set<std::string> &rhs) const { return lhs == rhs; }
+};
+
+struct PropertyComparator {
+  bool operator()(const Property &lhs, const Property &rhs) const { return lhs.name < rhs.name; }
+};
+
+struct PropertyInfo {
+  std::set<Property, PropertyComparator> properties;
+  bool mandatory;
+};
+
 void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
                                 mgp_memory *memory) {
   mgp::MemoryDispatcherGuard guard{memory};
   const auto record_factory = mgp::RecordFactory(result);
   try {
-    const mgp::Graph graph = mgp::Graph(memgraph_graph);
-    for (auto node : graph.Nodes()) {
-      std::string type;
-      mgp::List labels = mgp::List();
+    std::unordered_map<std::set<std::string>, PropertyInfo, LabelsHash, LabelsComparator> node_types_properties;
+
+    for (auto node : mgp::Graph(memgraph_graph).Nodes()) {
+      std::set<std::string> labels_set = {};
       for (auto label : node.Labels()) {
-        labels.AppendExtend(mgp::Value(label));
-        type += ":`" + std::string(label) + "`";
+        labels_set.emplace(label);
+      }
+
+      if (node_types_properties.find(labels_set) == node_types_properties.end()) {
+        node_types_properties[labels_set] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
       }
 
       if (node.Properties().empty()) {
-        auto record = record_factory.NewRecord();
-        ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
+        node_types_properties[labels_set].mandatory = false;  // if there is node with no property, it is not mandatory
         continue;
       }
 
+      auto &property_info = node_types_properties.at(labels_set);
       for (auto &[key, prop] : node.Properties()) {
-        auto property_type = mgp::List();
+        property_info.properties.emplace(key, std::move(prop));
+        if (property_info.mandatory) {
+          property_info.mandatory =
+              property_info.properties.size() == 1;  // if there is only one property, it is mandatory
+        }
+      }
+    }
+
+    for (auto &[labels, property_info] : node_types_properties) {
+      std::string label_type;
+      mgp::List labels_list = mgp::List();
+      for (auto const &label : labels) {
+        label_type += ":`" + std::string(label) + "`";
+        labels_list.AppendExtend(mgp::Value(label));
+      }
+      for (auto const &prop : property_info.properties) {
         auto record = record_factory.NewRecord();
-        property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
-        ProcessPropertiesNode<mgp::List>(record, type, labels, key, property_type, true);
+        ProcessPropertiesNode(record, label_type, labels_list, prop.name, TypeOf(prop.value.Type()),
+                              property_info.mandatory);
+      }
+      if (property_info.properties.empty()) {
+        auto record = record_factory.NewRecord();
+        ProcessPropertiesNode<std::string>(record, label_type, labels_list, "", "", false);
       }
     }
 
@@ -144,23 +196,41 @@ void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph,
 
 void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
   mgp::MemoryDispatcherGuard guard{memory};
+
+  std::unordered_map<std::string, PropertyInfo> rel_types_properties;
   const auto record_factory = mgp::RecordFactory(result);
   try {
     const mgp::Graph graph = mgp::Graph(memgraph_graph);
-
     for (auto rel : graph.Relationships()) {
-      std::string type = ":`" + std::string(rel.Type()) + "`";
+      std::string rel_type = std::string(rel.Type());
+      if (rel_types_properties.find(rel_type) == rel_types_properties.end()) {
+        rel_types_properties[rel_type] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
+      }
+
       if (rel.Properties().empty()) {
-        auto record = record_factory.NewRecord();
-        ProcessPropertiesRel<std::string>(record, type, "", "", false);
+        rel_types_properties[rel_type].mandatory = false;  // if there is rel with no property, it is not mandatory
         continue;
       }
 
+      auto &property_info = rel_types_properties.at(rel_type);
       for (auto &[key, prop] : rel.Properties()) {
-        auto property_type = mgp::List();
+        property_info.properties.emplace(key, std::move(prop));
+        if (property_info.mandatory) {
+          property_info.mandatory =
+              property_info.properties.size() == 1;  // if there is only one property, it is mandatory
+        }
+      }
+    }
+
+    for (auto &[type, property_info] : rel_types_properties) {
+      std::string type_str = ":`" + std::string(type) + "`";
+      for (auto const &prop : property_info.properties) {
         auto record = record_factory.NewRecord();
-        property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
-        ProcessPropertiesRel<mgp::List>(record, type, key, property_type, true);
+        ProcessPropertiesRel(record, type_str, prop.name, TypeOf(prop.value.Type()), property_info.mandatory);
+      }
+      if (property_info.properties.empty()) {
+        auto record = record_factory.NewRecord();
+        ProcessPropertiesRel<std::string>(record, type_str, "", "", false);
       }
     }
 

src/CMakeLists.txt

@@ -53,7 +53,10 @@ set_target_properties(memgraph PROPERTIES
         OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
 
         # Output the executable in main binary dir.
-        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+        RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
+
+        POSITION_INDEPENDENT_CODE ON
+        )
 
 # Create symlink to the built executable.
 add_custom_command(TARGET memgraph POST_BUILD

src/memgraph.cpp

@@ -163,7 +163,7 @@ int main(int argc, char **argv) {
         // libstd.
         auto gil = memgraph::py::EnsureGIL();
         // NOLINTNEXTLINE(hicpp-signed-bitwise)
-        auto *flag = PyLong_FromLong(RTLD_NOW | RTLD_DEEPBIND);
+        auto *flag = PyLong_FromLong(RTLD_NOW);
         auto *setdl = PySys_GetObject("setdlopenflags");
         MG_ASSERT(setdl);
         auto *arg = PyTuple_New(1);

src/memory/global_memory_control.cpp

@@ -119,7 +119,7 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz
 
   memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
   if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
-    GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
+    GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
   }
 
   return false;

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -1276,28 +1276,59 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
       call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(result_alias));
     }
   } else {
-    const auto &maybe_found =
-        procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource());
-    if (!maybe_found) {
-      throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
+    call_proc->is_write_ = maybe_found->second->info.is_write;
+
+    auto *yield_ctx = ctx->yieldProcedureResults();
+    if (!yield_ctx) {
+      if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) {
+        throw SemanticException(
+            "CALL without YIELD may only be used on procedures which do not "
+            "return any result fields.");
+      }
+      // When we return, we will release the lock on modules. This means that
+      // someone may reload the procedure and change the result signature. But to
+      // keep the implementation simple, we ignore the case as the rest of the
+      // code doesn't really care whether we yield or not, so it should not break.
+      return call_proc;
     }
-    const auto &[module, proc] = *maybe_found;
-    call_proc->result_fields_.reserve(proc->results.size());
-    call_proc->result_identifiers_.reserve(proc->results.size());
-    for (const auto &[result_name, desc] : proc->results) {
-      bool is_deprecated = desc.second;
-      if (is_deprecated) continue;
-      call_proc->result_fields_.emplace_back(result_name);
-      call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(std::string(result_name)));
+    if (yield_ctx->getTokens(MemgraphCypher::ASTERISK).empty()) {
+      call_proc->result_fields_.reserve(yield_ctx->procedureResult().size());
+      call_proc->result_identifiers_.reserve(yield_ctx->procedureResult().size());
+      for (auto *result : yield_ctx->procedureResult()) {
+        MG_ASSERT(result->variable().size() == 1 || result->variable().size() == 2);
+        call_proc->result_fields_.push_back(std::any_cast<std::string>(result->variable()[0]->accept(this)));
+        std::string result_alias;
+        if (result->variable().size() == 2) {
+          result_alias = std::any_cast<std::string>(result->variable()[1]->accept(this));
+        } else {
+          result_alias = std::any_cast<std::string>(result->variable()[0]->accept(this));
+        }
+        call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(result_alias));
+      }
+    } else {
+      const auto &maybe_found =
+          procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource());
+      if (!maybe_found) {
+        throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
+      }
+      const auto &[module, proc] = *maybe_found;
+      call_proc->result_fields_.reserve(proc->results.size());
+      call_proc->result_identifiers_.reserve(proc->results.size());
+      for (const auto &[result_name, desc] : proc->results) {
+        bool is_deprecated = desc.second;
+        if (is_deprecated) continue;
+        call_proc->result_fields_.emplace_back(result_name);
+        call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(std::string(result_name)));
+      }
+      // When we leave the scope, we will release the lock on modules. This means
+      // that someone may reload the procedure and change its result signature. We
+      // are fine with this, because if new result fields were added then we yield
+      // the subset of those and that will appear to a user as if they used the
+      // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the
+      // new fields as well. In case the result signature has had some result
+      // fields removed, then the query execution will report an error that we are
+      // yielding missing fields. The user can then just retry the query.
     }
-    // When we leave the scope, we will release the lock on modules. This means
-    // that someone may reload the procedure and change its result signature. We
-    // are fine with this, because if new result fields were added then we yield
-    // the subset of those and that will appear to a user as if they used the
-    // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the
-    // new fields as well. In case the result signature has had some result
-    // fields removed, then the query execution will report an error that we are
-    // yielding missing fields. The user can then just retry the query.
   }
 
   return call_proc;

src/query/procedure/module.hpp

@@ -169,7 +169,7 @@ class ModuleRegistry final {
   // mentioned library will be first performed in the already existing binded
   // libraries and then the global namespace.
   // RTLD_DEEPBIND => https://linux.die.net/man/3/dlopen
-  SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND, kLibstdcppWarning};
+  SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL, kLibstdcppWarning};
 #endif
   std::vector<std::filesystem::path> modules_dirs_;
   std::filesystem::path internal_module_dir_;

src/storage/v2/edge_accessor.cpp

@@ -21,6 +21,7 @@
 #include "storage/v2/result.hpp"
 #include "storage/v2/storage.hpp"
 #include "storage/v2/vertex_accessor.hpp"
+#include "utils/atomic_memory_block.hpp"
 #include "utils/memory_tracker.hpp"
 
 namespace memgraph::storage {
@@ -126,24 +127,28 @@ Result<storage::PropertyValue> EdgeAccessor::SetProperty(PropertyId property, co
   if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
 
   if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
-
-  auto current_value = edge_.ptr->properties.GetProperty(property);
-  // We could skip setting the value if the previous one is the same to the new
-  // one. This would save some memory as a delta would not be created as well as
-  // avoid copying the value. The reason we are not doing that is because the
-  // current code always follows the logical pattern of "create a delta" and
-  // "modify in-place". Additionally, the created delta will make other
-  // transactions get a SERIALIZATION_ERROR.
-
-  CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, current_value);
-  edge_.ptr->properties.SetProperty(property, value);
+  using ReturnType = decltype(edge_.ptr->properties.GetProperty(property));
+  std::optional<ReturnType> current_value;
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [&current_value, &property, &value, transaction = transaction_, edge = edge_]() {
+        current_value.emplace(edge.ptr->properties.GetProperty(property));
+        // We could skip setting the value if the previous one is the same to the new
+        // one. This would save some memory as a delta would not be created as well as
+        // avoid copying the value. The reason we are not doing that is because the
+        // current code always follows the logical pattern of "create a delta" and
+        // "modify in-place". Additionally, the created delta will make other
+        // transactions get a SERIALIZATION_ERROR.
+        CreateAndLinkDelta(transaction, edge.ptr, Delta::SetPropertyTag(), property, *current_value);
+        edge.ptr->properties.SetProperty(property, value);
+      }};
+  std::invoke(atomic_memory_block);
 
   if (transaction_->IsDiskStorage()) {
     ModifiedEdgeInfo modified_edge(Delta::Action::SET_PROPERTY, from_vertex_->gid, to_vertex_->gid, edge_type_, edge_);
     transaction_->AddModifiedEdge(Gid(), modified_edge);
   }
 
-  return std::move(current_value);
+  return std::move(*current_value);
 }
 
 Result<bool> EdgeAccessor::InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
@@ -157,9 +162,12 @@ Result<bool> EdgeAccessor::InitProperties(const std::map<storage::PropertyId, st
   if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
 
   if (!edge_.ptr->properties.InitProperties(properties)) return false;
-  for (const auto &[property, _] : properties) {
-    CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, PropertyValue());
-  }
+  utils::AtomicMemoryBlock atomic_memory_block{[&properties, transaction_ = transaction_, edge_ = edge_]() {
+    for (const auto &[property, _] : properties) {
+      CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, PropertyValue());
+    }
+  }};
+  std::invoke(atomic_memory_block);
 
   return true;
 }
@@ -175,13 +183,18 @@ Result<std::vector<std::tuple<PropertyId, PropertyValue, PropertyValue>>> EdgeAc
 
   if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
 
-  auto id_old_new_change = edge_.ptr->properties.UpdateProperties(properties);
+  using ReturnType = decltype(edge_.ptr->properties.UpdateProperties(properties));
+  std::optional<ReturnType> id_old_new_change;
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [transaction_ = transaction_, edge_ = edge_, &properties, &id_old_new_change]() {
+        id_old_new_change.emplace(edge_.ptr->properties.UpdateProperties(properties));
+        for (auto &[property, old_value, new_value] : *id_old_new_change) {
+          CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, std::move(old_value));
+        }
+      }};
+  std::invoke(atomic_memory_block);
 
-  for (auto &[property, old_value, new_value] : id_old_new_change) {
-    CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, std::move(old_value));
-  }
-
-  return id_old_new_change;
+  return id_old_new_change.has_value() ? std::move(id_old_new_change.value()) : ReturnType{};
 }
 
 Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
@@ -193,14 +206,19 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
 
   if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
 
-  auto properties = edge_.ptr->properties.Properties();
-  for (const auto &property : properties) {
-    CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property.first, property.second);
-  }
+  using ReturnType = decltype(edge_.ptr->properties.Properties());
+  std::optional<ReturnType> properties;
+  utils::AtomicMemoryBlock atomic_memory_block{[&properties, transaction_ = transaction_, edge_ = edge_]() {
+    properties.emplace(edge_.ptr->properties.Properties());
+    for (const auto &property : *properties) {
+      CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property.first, property.second);
+    }
 
-  edge_.ptr->properties.ClearProperties();
+    edge_.ptr->properties.ClearProperties();
+  }};
+  std::invoke(atomic_memory_block);
 
-  return std::move(properties);
+  return properties.has_value() ? std::move(properties.value()) : ReturnType{};
 }
 
 Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {

src/storage/v2/inmemory/storage.cpp

@@ -26,6 +26,7 @@
 #include "storage/v2/inmemory/replication/recovery.hpp"
 #include "storage/v2/inmemory/unique_constraints.hpp"
 #include "storage/v2/property_value.hpp"
+#include "utils/atomic_memory_block.hpp"
 #include "utils/resource_lock.hpp"
 #include "utils/stat.hpp"
 
@@ -338,18 +339,22 @@ Result<EdgeAccessor> InMemoryStorage::InMemoryAccessor::CreateEdge(VertexAccesso
       delta->prev.Set(&*it);
     }
   }
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [this, edge, from_vertex = from_vertex, edge_type = edge_type, to_vertex = to_vertex]() {
+        CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
+        from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge);
 
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
-  from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge);
+        CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge);
+        to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge);
 
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge);
-  to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge);
+        transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
+        transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
 
-  transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
+        // Increment edge count.
+        storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
+      }};
 
-  // Increment edge count.
-  storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
+  std::invoke(atomic_memory_block);
 
   return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_);
 }
@@ -433,18 +438,22 @@ Result<EdgeAccessor> InMemoryStorage::InMemoryAccessor::CreateEdgeEx(VertexAcces
       delta->prev.Set(&*it);
     }
   }
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [this, edge, from_vertex = from_vertex, edge_type = edge_type, to_vertex = to_vertex]() {
+        CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
+        from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge);
 
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
-  from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge);
+        CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge);
+        to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge);
 
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge);
-  to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge);
+        transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
+        transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
 
-  transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
+        // Increment edge count.
+        storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
+      }};
 
-  // Increment edge count.
-  storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
+  std::invoke(atomic_memory_block);
 
   return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, storage_, &transaction_);
 }
@@ -534,18 +543,22 @@ Result<EdgeAccessor> InMemoryStorage::InMemoryAccessor::EdgeSetFrom(EdgeAccessor
       return Error::DELETED_OBJECT;
     }
   }
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [this, edge_ref, old_from_vertex, new_from_vertex, edge_type, to_vertex]() {
+        CreateAndLinkDelta(&transaction_, old_from_vertex, Delta::AddOutEdgeTag(), edge_type, to_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, to_vertex, Delta::AddInEdgeTag(), edge_type, old_from_vertex, edge_ref);
 
-  CreateAndLinkDelta(&transaction_, old_from_vertex, Delta::AddOutEdgeTag(), edge_type, to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::AddInEdgeTag(), edge_type, old_from_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, new_from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge_ref);
+        new_from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, new_from_vertex, edge_ref);
+        to_vertex->in_edges.emplace_back(edge_type, new_from_vertex, edge_ref);
 
-  CreateAndLinkDelta(&transaction_, new_from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge_ref);
-  new_from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, new_from_vertex, edge_ref);
-  to_vertex->in_edges.emplace_back(edge_type, new_from_vertex, edge_ref);
+        transaction_.manyDeltasCache.Invalidate(new_from_vertex, edge_type, EdgeDirection::OUT);
+        transaction_.manyDeltasCache.Invalidate(old_from_vertex, edge_type, EdgeDirection::OUT);
+        transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
+      }};
 
-  transaction_.manyDeltasCache.Invalidate(new_from_vertex, edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(old_from_vertex, edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(to_vertex, edge_type, EdgeDirection::IN);
+  std::invoke(atomic_memory_block);
 
   return EdgeAccessor(edge_ref, edge_type, new_from_vertex, to_vertex, storage_, &transaction_);
 }
@@ -636,17 +649,22 @@ Result<EdgeAccessor> InMemoryStorage::InMemoryAccessor::EdgeSetTo(EdgeAccessor *
     }
   }
 
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::AddOutEdgeTag(), edge_type, old_to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, old_to_vertex, Delta::AddInEdgeTag(), edge_type, from_vertex, edge_ref);
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [this, edge_ref, old_to_vertex, from_vertex, edge_type, new_to_vertex]() {
+        CreateAndLinkDelta(&transaction_, from_vertex, Delta::AddOutEdgeTag(), edge_type, old_to_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, old_to_vertex, Delta::AddInEdgeTag(), edge_type, from_vertex, edge_ref);
 
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, new_to_vertex, edge_ref);
-  from_vertex->out_edges.emplace_back(edge_type, new_to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, new_to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge_ref);
-  new_to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, new_to_vertex, edge_ref);
+        from_vertex->out_edges.emplace_back(edge_type, new_to_vertex, edge_ref);
+        CreateAndLinkDelta(&transaction_, new_to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex, edge_ref);
+        new_to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge_ref);
 
-  transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(old_to_vertex, edge_type, EdgeDirection::IN);
-  transaction_.manyDeltasCache.Invalidate(new_to_vertex, edge_type, EdgeDirection::IN);
+        transaction_.manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT);
+        transaction_.manyDeltasCache.Invalidate(old_to_vertex, edge_type, EdgeDirection::IN);
+        transaction_.manyDeltasCache.Invalidate(new_to_vertex, edge_type, EdgeDirection::IN);
+      }};
+
+  std::invoke(atomic_memory_block);
 
   return EdgeAccessor(edge_ref, edge_type, from_vertex, new_to_vertex, storage_, &transaction_);
 }
@@ -709,17 +727,21 @@ Result<EdgeAccessor> InMemoryStorage::InMemoryAccessor::EdgeChangeType(EdgeAcces
 
   MG_ASSERT((op1 && op2), "Invalid database state!");
 
-  // "deleting" old edge
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::AddOutEdgeTag(), edge_type, to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::AddInEdgeTag(), edge_type, from_vertex, edge_ref);
+  utils::AtomicMemoryBlock atomic_memory_block{[this, to_vertex, new_edge_type, edge_ref, from_vertex, edge_type]() {
+    // "deleting" old edge
+    CreateAndLinkDelta(&transaction_, from_vertex, Delta::AddOutEdgeTag(), edge_type, to_vertex, edge_ref);
+    CreateAndLinkDelta(&transaction_, to_vertex, Delta::AddInEdgeTag(), edge_type, from_vertex, edge_ref);
 
-  // "adding" new edge
-  CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), new_edge_type, to_vertex, edge_ref);
-  CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), new_edge_type, from_vertex, edge_ref);
+    // "adding" new edge
+    CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), new_edge_type, to_vertex, edge_ref);
+    CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(), new_edge_type, from_vertex, edge_ref);
 
-  // edge type is not used while invalidating cache so we can only call it once
-  transaction_.manyDeltasCache.Invalidate(from_vertex, new_edge_type, EdgeDirection::OUT);
-  transaction_.manyDeltasCache.Invalidate(to_vertex, new_edge_type, EdgeDirection::IN);
+    // edge type is not used while invalidating cache so we can only call it once
+    transaction_.manyDeltasCache.Invalidate(from_vertex, new_edge_type, EdgeDirection::OUT);
+    transaction_.manyDeltasCache.Invalidate(to_vertex, new_edge_type, EdgeDirection::IN);
+  }};
+
+  std::invoke(atomic_memory_block);
 
   return EdgeAccessor(edge_ref, new_edge_type, from_vertex, to_vertex, storage_, &transaction_);
 }

src/storage/v2/storage.cpp

@@ -17,6 +17,7 @@
 #include "storage/v2/storage.hpp"
 #include "storage/v2/transaction.hpp"
 #include "storage/v2/vertex_accessor.hpp"
+#include "utils/atomic_memory_block.hpp"
 #include "utils/event_counter.hpp"
 #include "utils/event_histogram.hpp"
 #include "utils/exceptions.hpp"
@@ -390,22 +391,29 @@ Result<std::optional<std::vector<EdgeAccessor>>> Storage::Accessor::ClearEdgesOn
       if (!PrepareForWrite(&transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
       MG_ASSERT(!vertex_ptr->deleted, "Invalid database state!");
 
-      attached_edges_to_vertex->pop_back();
-      if (storage_->config_.items.properties_on_edges) {
-        auto *edge_ptr = edge_ref.ptr;
-        MarkEdgeAsDeleted(edge_ptr);
-      }
+      // MarkEdgeAsDeleted allocates additional memory
+      // and CreateAndLinkDelta needs memory
+      utils::AtomicMemoryBlock atomic_memory_block{[&attached_edges_to_vertex, &deleted_edge_ids, &reverse_vertex_order,
+                                                    &vertex_ptr, &deleted_edges, deletion_delta = deletion_delta,
+                                                    edge_type = edge_type, opposing_vertex = opposing_vertex,
+                                                    edge_ref = edge_ref, this]() {
+        attached_edges_to_vertex->pop_back();
+        if (this->storage_->config_.items.properties_on_edges) {
+          auto *edge_ptr = edge_ref.ptr;
+          MarkEdgeAsDeleted(edge_ptr);
+        }
 
-      auto const edge_gid = storage_->config_.items.properties_on_edges ? edge_ref.ptr->gid : edge_ref.gid;
-      auto const [_, was_inserted] = deleted_edge_ids.insert(edge_gid);
-      bool const edge_cleared_from_both_directions = !was_inserted;
-      if (edge_cleared_from_both_directions) {
-        auto *from_vertex = reverse_vertex_order ? vertex_ptr : opposing_vertex;
-        auto *to_vertex = reverse_vertex_order ? opposing_vertex : vertex_ptr;
-        deleted_edges.emplace_back(edge_ref, edge_type, from_vertex, to_vertex, storage_, &transaction_, true);
-      }
-
-      CreateAndLinkDelta(&transaction_, vertex_ptr, deletion_delta, edge_type, opposing_vertex, edge_ref);
+        auto const edge_gid = storage_->config_.items.properties_on_edges ? edge_ref.ptr->gid : edge_ref.gid;
+        auto const [_, was_inserted] = deleted_edge_ids.insert(edge_gid);
+        bool const edge_cleared_from_both_directions = !was_inserted;
+        if (edge_cleared_from_both_directions) {
+          auto *from_vertex = reverse_vertex_order ? vertex_ptr : opposing_vertex;
+          auto *to_vertex = reverse_vertex_order ? opposing_vertex : vertex_ptr;
+          deleted_edges.emplace_back(edge_ref, edge_type, from_vertex, to_vertex, storage_, &transaction_, true);
+        }
+        CreateAndLinkDelta(&transaction_, vertex_ptr, deletion_delta, edge_type, opposing_vertex, edge_ref);
+      }};
+      std::invoke(atomic_memory_block);
     }
 
     return std::make_optional<ReturnType>();
@@ -449,30 +457,37 @@ Result<std::optional<std::vector<EdgeAccessor>>> Storage::Accessor::DetachRemain
           return !set_for_erasure.contains(edge_gid);
         });
 
-    for (auto it = mid; it != edges_attached_to_vertex->end(); it++) {
-      auto const &[edge_type, opposing_vertex, edge_ref] = *it;
-      std::unique_lock<utils::RWSpinLock> guard;
-      if (storage_->config_.items.properties_on_edges) {
-        auto edge_ptr = edge_ref.ptr;
-        guard = std::unique_lock{edge_ptr->lock};
-        // this can happen only if we marked edges for deletion with no nodes,
-        // so the method detaching nodes will not do anything
-        MarkEdgeAsDeleted(edge_ptr);
+    // Creating deltas and erasing edge only at the end -> we might have incomplete state as
+    // delta might cause OOM, so we don't remove edges from edges_attached_to_vertex
+    utils::AtomicMemoryBlock atomic_memory_block{[&mid, &edges_attached_to_vertex, &deleted_edges,
+                                                  &partially_detached_edge_ids, this, vertex_ptr, deletion_delta,
+                                                  reverse_vertex_order]() {
+      for (auto it = mid; it != edges_attached_to_vertex->end(); it++) {
+        auto const &[edge_type, opposing_vertex, edge_ref] = *it;
+        std::unique_lock<utils::RWSpinLock> guard;
+        if (storage_->config_.items.properties_on_edges) {
+          auto edge_ptr = edge_ref.ptr;
+          guard = std::unique_lock{edge_ptr->lock};
+          // this can happen only if we marked edges for deletion with no nodes,
+          // so the method detaching nodes will not do anything
+          MarkEdgeAsDeleted(edge_ptr);
+        }
+
+        CreateAndLinkDelta(&transaction_, vertex_ptr, deletion_delta, edge_type, opposing_vertex, edge_ref);
+
+        auto const edge_gid = storage_->config_.items.properties_on_edges ? edge_ref.ptr->gid : edge_ref.gid;
+        auto const [_, was_inserted] = partially_detached_edge_ids.insert(edge_gid);
+        bool const edge_cleared_from_both_directions = !was_inserted;
+        if (edge_cleared_from_both_directions) {
+          auto *from_vertex = reverse_vertex_order ? opposing_vertex : vertex_ptr;
+          auto *to_vertex = reverse_vertex_order ? vertex_ptr : opposing_vertex;
+          deleted_edges.emplace_back(edge_ref, edge_type, from_vertex, to_vertex, storage_, &transaction_, true);
+        }
       }
+      edges_attached_to_vertex->erase(mid, edges_attached_to_vertex->end());
+    }};
 
-      CreateAndLinkDelta(&transaction_, vertex_ptr, deletion_delta, edge_type, opposing_vertex, edge_ref);
-
-      auto const edge_gid = storage_->config_.items.properties_on_edges ? edge_ref.ptr->gid : edge_ref.gid;
-      auto const [_, was_inserted] = partially_detached_edge_ids.insert(edge_gid);
-      bool const edge_cleared_from_both_directions = !was_inserted;
-      if (edge_cleared_from_both_directions) {
-        auto *from_vertex = reverse_vertex_order ? opposing_vertex : vertex_ptr;
-        auto *to_vertex = reverse_vertex_order ? vertex_ptr : opposing_vertex;
-        deleted_edges.emplace_back(edge_ref, edge_type, from_vertex, to_vertex, storage_, &transaction_, true);
-      }
-    }
-
-    edges_attached_to_vertex->erase(mid, edges_attached_to_vertex->end());
+    std::invoke(atomic_memory_block);
 
     return std::make_optional<ReturnType>();
   };

src/storage/v2/vertex_accessor.cpp

@@ -26,6 +26,7 @@
 #include "storage/v2/storage.hpp"
 #include "storage/v2/vertex_info_cache.hpp"
 #include "storage/v2/vertex_info_helpers.hpp"
+#include "utils/atomic_memory_block.hpp"
 #include "utils/logging.hpp"
 #include "utils/memory_tracker.hpp"
 #include "utils/variant_helpers.hpp"
@@ -107,8 +108,11 @@ Result<bool> VertexAccessor::AddLabel(LabelId label) {
   if (vertex_->deleted) return Error::DELETED_OBJECT;
   if (std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end()) return false;
 
-  CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label);
-  vertex_->labels.push_back(label);
+  utils::AtomicMemoryBlock atomic_memory_block{[transaction = transaction_, vertex = vertex_, &label]() {
+    CreateAndLinkDelta(transaction, vertex, Delta::RemoveLabelTag(), label);
+    vertex->labels.push_back(label);
+  }};
+  std::invoke(atomic_memory_block);
 
   if (storage_->config_.items.enable_schema_metadata) {
     storage_->stored_node_labels_.try_insert(label);
@@ -136,9 +140,12 @@ Result<bool> VertexAccessor::RemoveLabel(LabelId label) {
   auto it = std::find(vertex_->labels.begin(), vertex_->labels.end(), label);
   if (it == vertex_->labels.end()) return false;
 
-  CreateAndLinkDelta(transaction_, vertex_, Delta::AddLabelTag(), label);
-  *it = vertex_->labels.back();
-  vertex_->labels.pop_back();
+  utils::AtomicMemoryBlock atomic_memory_block{[transaction = transaction_, vertex = vertex_, &label, &it]() {
+    CreateAndLinkDelta(transaction, vertex, Delta::AddLabelTag(), label);
+    *it = vertex->labels.back();
+    vertex->labels.pop_back();
+  }};
+  std::invoke(atomic_memory_block);
 
   /// TODO: some by pointers, some by reference => not good, make it better
   storage_->constraints_.unique_constraints_->UpdateOnRemoveLabel(label, *vertex_, transaction_->start_timestamp);
@@ -262,8 +269,12 @@ Result<PropertyValue> VertexAccessor::SetProperty(PropertyId property, const Pro
   // "modify in-place". Additionally, the created delta will make other
   // transactions get a SERIALIZATION_ERROR.
 
-  CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, current_value);
-  vertex_->properties.SetProperty(property, value);
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [transaction = transaction_, vertex = vertex_, &value, &property, &current_value]() {
+        CreateAndLinkDelta(transaction, vertex, Delta::SetPropertyTag(), property, current_value);
+        vertex->properties.SetProperty(property, value);
+      }};
+  std::invoke(atomic_memory_block);
 
   if (!value.IsNull()) {
     transaction_->constraint_verification_info.AddedProperty(vertex_);
@@ -287,20 +298,28 @@ Result<bool> VertexAccessor::InitProperties(const std::map<storage::PropertyId,
   if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
 
   if (vertex_->deleted) return Error::DELETED_OBJECT;
+  bool result{false};
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [&result, &properties, storage = storage_, transaction = transaction_, vertex = vertex_]() {
+        if (!vertex->properties.InitProperties(properties)) {
+          result = false;
+          return;
+        }
+        for (const auto &[property, value] : properties) {
+          CreateAndLinkDelta(transaction, vertex, Delta::SetPropertyTag(), property, PropertyValue());
+          storage->indices_.UpdateOnSetProperty(property, value, vertex, *transaction);
+          transaction->manyDeltasCache.Invalidate(vertex, property);
+          if (!value.IsNull()) {
+            transaction->constraint_verification_info.AddedProperty(vertex);
+          } else {
+            transaction->constraint_verification_info.RemovedProperty(vertex);
+          }
+        }
+        result = true;
+      }};
+  std::invoke(atomic_memory_block);
 
-  if (!vertex_->properties.InitProperties(properties)) return false;
-  for (const auto &[property, value] : properties) {
-    CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, PropertyValue());
-    storage_->indices_.UpdateOnSetProperty(property, value, vertex_, *transaction_);
-    transaction_->manyDeltasCache.Invalidate(vertex_, property);
-    if (!value.IsNull()) {
-      transaction_->constraint_verification_info.AddedProperty(vertex_);
-    } else {
-      transaction_->constraint_verification_info.RemovedProperty(vertex_);
-    }
-  }
-
-  return true;
+  return result;
 }
 
 Result<std::vector<std::tuple<PropertyId, PropertyValue, PropertyValue>>> VertexAccessor::UpdateProperties(
@@ -316,20 +335,28 @@ Result<std::vector<std::tuple<PropertyId, PropertyValue, PropertyValue>>> Vertex
 
   if (vertex_->deleted) return Error::DELETED_OBJECT;
 
-  auto id_old_new_change = vertex_->properties.UpdateProperties(properties);
+  using ReturnType = decltype(vertex_->properties.UpdateProperties(properties));
+  std::optional<ReturnType> id_old_new_change;
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [storage = storage_, transaction = transaction_, vertex = vertex_, &properties, &id_old_new_change]() {
+        id_old_new_change.emplace(vertex->properties.UpdateProperties(properties));
+        if (!id_old_new_change.has_value()) {
+          return;
+        }
+        for (auto &[id, old_value, new_value] : *id_old_new_change) {
+          storage->indices_.UpdateOnSetProperty(id, new_value, vertex, *transaction);
+          CreateAndLinkDelta(transaction, vertex, Delta::SetPropertyTag(), id, std::move(old_value));
+          transaction->manyDeltasCache.Invalidate(vertex, id);
+          if (!new_value.IsNull()) {
+            transaction->constraint_verification_info.AddedProperty(vertex);
+          } else {
+            transaction->constraint_verification_info.RemovedProperty(vertex);
+          }
+        }
+      }};
+  std::invoke(atomic_memory_block);
 
-  for (auto &[id, old_value, new_value] : id_old_new_change) {
-    storage_->indices_.UpdateOnSetProperty(id, new_value, vertex_, *transaction_);
-    CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), id, std::move(old_value));
-    transaction_->manyDeltasCache.Invalidate(vertex_, id);
-    if (!new_value.IsNull()) {
-      transaction_->constraint_verification_info.AddedProperty(vertex_);
-    } else {
-      transaction_->constraint_verification_info.RemovedProperty(vertex_);
-    }
-  }
-
-  return id_old_new_change;
+  return id_old_new_change.has_value() ? std::move(id_old_new_change.value()) : ReturnType{};
 }
 
 Result<std::map<PropertyId, PropertyValue>> VertexAccessor::ClearProperties() {
@@ -342,17 +369,25 @@ Result<std::map<PropertyId, PropertyValue>> VertexAccessor::ClearProperties() {
 
   if (vertex_->deleted) return Error::DELETED_OBJECT;
 
-  auto properties = vertex_->properties.Properties();
-  for (const auto &[property, value] : properties) {
-    CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, value);
-    storage_->indices_.UpdateOnSetProperty(property, PropertyValue(), vertex_, *transaction_);
-    transaction_->constraint_verification_info.RemovedProperty(vertex_);
-    transaction_->manyDeltasCache.Invalidate(vertex_, property);
-  }
+  using ReturnType = decltype(vertex_->properties.Properties());
+  std::optional<ReturnType> properties;
+  utils::AtomicMemoryBlock atomic_memory_block{
+      [storage = storage_, transaction = transaction_, vertex = vertex_, &properties]() {
+        properties.emplace(vertex->properties.Properties());
+        if (!properties.has_value()) {
+          return;
+        }
+        for (const auto &[property, value] : *properties) {
+          CreateAndLinkDelta(transaction, vertex, Delta::SetPropertyTag(), property, value);
+          storage->indices_.UpdateOnSetProperty(property, PropertyValue(), vertex, *transaction);
+          transaction->constraint_verification_info.RemovedProperty(vertex);
+          transaction->manyDeltasCache.Invalidate(vertex, property);
+        }
+        vertex->properties.ClearProperties();
+      }};
+  std::invoke(atomic_memory_block);
 
-  vertex_->properties.ClearProperties();
-
-  return std::move(properties);
+  return properties.has_value() ? std::move(properties.value()) : ReturnType{};
 }
 
 Result<PropertyValue> VertexAccessor::GetProperty(PropertyId property, View view) const {

src/telemetry/telemetry.cpp

@@ -110,7 +110,13 @@ void Telemetry::CollectData(const std::string &event) {
   {
     std::lock_guard<std::mutex> guard(lock_);
     for (auto &collector : collectors_) {
-      data[collector.first] = collector.second();
+      try {
+        data[collector.first] = collector.second();
+      } catch (std::exception &e) {
+        spdlog::warn(fmt::format(
+            "Unknwon exception occured on in telemetry server {}, please contact support on https://memgr.ph/unknown ",
+            e.what()));
+      }
     }
   }
   if (event == "") {

src/utils/atomic_memory_block.hpp

@@ -0,0 +1,44 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include <functional>
+#include "utils/memory_tracker.hpp"
+
+namespace memgraph::utils {
+
+// Calls a function with out of memory exception blocker, checks memory allocation after block execution.
+// Use it in case you need block which will be executed atomically considering memory execution
+// but will check after block is executed if OOM exceptions needs to be thrown
+template <typename Callable>
+class [[nodiscard]] AtomicMemoryBlock {
+ public:
+  explicit AtomicMemoryBlock(Callable &&function) : function_{std::forward<Callable>(function)} {}
+  AtomicMemoryBlock(AtomicMemoryBlock const &) = delete;
+  AtomicMemoryBlock(AtomicMemoryBlock &&) = delete;
+  AtomicMemoryBlock &operator=(AtomicMemoryBlock const &) = delete;
+  AtomicMemoryBlock &operator=(AtomicMemoryBlock &&) = delete;
+  ~AtomicMemoryBlock() = default;
+
+  void operator()() {
+    {
+      utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_blocker;
+      function_();
+    }
+    total_memory_tracker.DoCheck();
+  }
+
+ private:
+  Callable function_;
+};
+
+}  // namespace memgraph::utils

src/utils/memory_tracker.cpp

@@ -124,6 +124,19 @@ void MemoryTracker::Alloc(const int64_t size) {
   UpdatePeak(will_be);
 }
 
+void MemoryTracker::DoCheck() {
+  const auto current_hard_limit = hard_limit_.load(std::memory_order_relaxed);
+  const auto current_amount = amount_.load(std::memory_order_relaxed);
+  if (current_hard_limit && current_amount > current_hard_limit && MemoryTrackerCanThrow()) [[unlikely]] {
+    MemoryTracker::OutOfMemoryExceptionBlocker exception_blocker;
+    throw OutOfMemoryException(
+        fmt::format("Memory limit exceeded! Current "
+                    "use is {}, while the maximum allowed size for allocation is set to {}.",
+                    GetReadableSize(static_cast<double>(current_amount)),
+                    GetReadableSize(static_cast<double>(current_hard_limit))));
+  }
+}
+
 void MemoryTracker::Free(const int64_t size) { amount_.fetch_sub(size, std::memory_order_relaxed); }
 
 }  // namespace memgraph::utils

src/utils/memory_tracker.hpp

@@ -49,6 +49,7 @@ class MemoryTracker final {
 
   void Alloc(int64_t size);
   void Free(int64_t size);
+  void DoCheck();
 
   auto Amount() const { return amount_.load(std::memory_order_relaxed); }
 

src/utils/stat.hpp

@@ -25,17 +25,32 @@ namespace memgraph::utils {
 static constexpr int64_t VM_MAX_MAP_COUNT_DEFAULT{-1};
 
 /// Returns the number of bytes a directory is using on disk. If the given path
-/// isn't a directory, zero will be returned.
+/// isn't a directory, zero will be returned. If there are some files with
+/// wrong permission, it will be skipped
 template <bool IgnoreSymlink = true>
 inline uint64_t GetDirDiskUsage(const std::filesystem::path &path) {
   if (!std::filesystem::is_directory(path)) return 0;
 
+  if (!utils::HasReadAccess(path)) {
+    spdlog::warn(
+        "Skipping directory path on collecting directory disk usage '{}' because it is not readable, check file "
+        "ownership and read permissions!",
+        path);
+    return 0;
+  }
   uint64_t size = 0;
-  for (auto &p : std::filesystem::directory_iterator(path)) {
+  for (const auto &p : std::filesystem::directory_iterator(path)) {
     if (IgnoreSymlink && std::filesystem::is_symlink(p)) continue;
     if (std::filesystem::is_directory(p)) {
       size += GetDirDiskUsage(p);
     } else if (std::filesystem::is_regular_file(p)) {
+      if (!utils::HasReadAccess(p)) {
+        spdlog::warn(
+            "Skipping file path on collecting directory disk usage '{}' because it is not readable, check file "
+            "ownership and read permissions!",
+            p);
+        continue;
+      }
       size += std::filesystem::file_size(p);
     }
   }

tests/drivers/java/v5_8/run.sh

@@ -9,8 +9,8 @@ fi
 if [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then
   export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
 fi
-if [ -d "/opt/apache-maven-3.9.2" ]; then
-  export M2_HOME="/opt/apache-maven-3.9.2"
+if [ -d "/opt/apache-maven-3.9.3" ]; then
+  export M2_HOME="/opt/apache-maven-3.9.3"
 fi
 export PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH"
 

tests/e2e/interactive_mg_runner.py

@@ -80,6 +80,8 @@ ACTIONS = {
     "quit": lambda _: sys.exit(1),
 }
 
+CLEANUP_DIRECTORIES_ON_EXIT = False
+
 log = logging.getLogger("memgraph.tests.e2e")
 
 
@@ -109,10 +111,11 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_
     assert not is_port_in_use(
         extract_bolt_port(args)
     ), "If this raises, you are trying to start an instance on a port already used by one already running instance."
-    mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl)
-    MEMGRAPH_INSTANCES[name] = mg_instance
+
     log_file_path = os.path.join(BUILD_DIR, "logs", log_file)
     data_directory_path = os.path.join(BUILD_DIR, data_directory)
+    mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl, {data_directory_path})
+    MEMGRAPH_INSTANCES[name] = mg_instance
     binary_args = args + ["--log-file", log_file_path] + ["--data-directory", data_directory_path]
 
     if len(procdir) != 0:
@@ -122,39 +125,43 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_
     assert mg_instance.is_running(), "An error occured after starting Memgraph instance: application stopped running."
 
 
-def stop_all():
+def stop_all(keep_directories=True):
     for mg_instance in MEMGRAPH_INSTANCES.values():
-        mg_instance.stop()
+        mg_instance.stop(keep_directories)
     MEMGRAPH_INSTANCES.clear()
 
 
-def stop_instance(context, name):
+def stop_instance(context, name, keep_directories=True):
     for key, _ in context.items():
         if key != name:
             continue
-        MEMGRAPH_INSTANCES[name].stop()
+        MEMGRAPH_INSTANCES[name].stop(keep_directories)
         MEMGRAPH_INSTANCES.pop(name)
 
 
-def stop(context, name):
+def stop(context, name, keep_directories=True):
     if name != "all":
-        stop_instance(context, name)
+        stop_instance(context, name, keep_directories)
         return
 
     stop_all()
 
 
-def kill(context, name):
+def kill(context, name, keep_directories=True):
     for key in context.keys():
         if key != name:
             continue
-        MEMGRAPH_INSTANCES[name].kill()
+        MEMGRAPH_INSTANCES[name].kill(keep_directories)
         MEMGRAPH_INSTANCES.pop(name)
 
 
+def cleanup_directories_on_exit(value=True):
+    CLEANUP_DIRECTORIES_ON_EXIT = value
+
+
 @atexit.register
 def cleanup():
-    stop_all()
+    stop_all(CLEANUP_DIRECTORIES_ON_EXIT)
 
 
 def start_instance(context, name, procdir):
@@ -184,8 +191,8 @@ def start_instance(context, name, procdir):
     assert len(mg_instances) == 1
 
 
-def start_all(context, procdir=""):
-    stop_all()
+def start_all(context, procdir="", keep_directories=True):
+    stop_all(keep_directories)
     for key, _ in context.items():
         start_instance(context, key, procdir)
 

tests/e2e/memgraph.py

@@ -11,6 +11,7 @@
 
 import copy
 import os
+import shutil
 import subprocess
 import sys
 import time
@@ -56,13 +57,14 @@ def replace_paths(path):
 
 
 class MemgraphInstanceRunner:
-    def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False):
+    def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False, delete_on_stop=None):
         self.host = "127.0.0.1"
         self.bolt_port = None
         self.binary_path = binary_path
         self.args = None
         self.proc_mg = None
         self.ssl = use_ssl
+        self.delete_on_stop = delete_on_stop
 
     def execute_setup_queries(self, setup_queries):
         if setup_queries is None:
@@ -128,7 +130,7 @@ class MemgraphInstanceRunner:
             return False
         return True
 
-    def stop(self):
+    def stop(self, keep_directories=False):
         if not self.is_running():
             return
 
@@ -140,9 +142,16 @@ class MemgraphInstanceRunner:
 
         time.sleep(1)
 
-    def kill(self):
+        if not keep_directories:
+            for folder in self.delete_on_stop or {}:
+                shutil.rmtree(folder)
+
+    def kill(self, keep_directories=False):
         if not self.is_running():
             return
         self.proc_mg.kill()
         code = self.proc_mg.wait()
+        if not keep_directories:
+            for folder in self.delete_on_stop or {}:
+                shutil.rmtree(folder)
         assert code == -9, "The killed Memgraph process exited with non-nine!"

tests/e2e/mg_utils.py

@@ -1,12 +1,11 @@
 import time
 
 
-def mg_sleep_and_assert(expected_value, function_to_retrieve_data, max_duration=20, time_between_attempt=0.05):
+def mg_sleep_and_assert(expected_value, function_to_retrieve_data, max_duration=20, time_between_attempt=0.2):
     result = function_to_retrieve_data()
     start_time = time.time()
     while result != expected_value:
-        current_time = time.time()
-        duration = current_time - start_time
+        duration = time.time() - start_time
         if duration > max_duration:
             assert (
                 False

tests/e2e/query_modules/schema_test.py

@@ -431,7 +431,7 @@ def test_node_type_properties1():
             f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
         )[0]
     )
-    assert (result) == [":`Activity`", ["Activity"], "location", ["String"], True]
+    assert (result) == [":`Activity`", ["Activity"], "location", "String", False]
 
     result = list(
         execute_and_fetch_all(
@@ -439,7 +439,7 @@ def test_node_type_properties1():
             f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
         )[1]
     )
-    assert (result) == [":`Activity`", ["Activity"], "name", ["String"], True]
+    assert (result) == [":`Activity`", ["Activity"], "name", "String", False]
 
     result = list(
         execute_and_fetch_all(
@@ -447,7 +447,7 @@ def test_node_type_properties1():
             f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
         )[2]
     )
-    assert (result) == [":`Dog`", ["Dog"], "name", ["String"], True]
+    assert (result) == [":`Dog`", ["Dog"], "name", "String", False]
 
     result = list(
         execute_and_fetch_all(
@@ -455,7 +455,81 @@ def test_node_type_properties1():
             f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
         )[3]
     )
-    assert (result) == [":`Dog`", ["Dog"], "owner", ["String"], True]
+    assert (result) == [":`Dog`", ["Dog"], "owner", "String", False]
+
+
+def test_node_type_properties2():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (d:MyNode)
+        CREATE (n:MyNode)
+        """,
+    )
+    result = execute_and_fetch_all(
+        cursor,
+        f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+    )
+    assert (list(result[0])) == [":`MyNode`", ["MyNode"], "", "", False]
+    assert (result.__len__()) == 1
+
+
+def test_node_type_properties3():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (d:Dog {name: 'Rex', owner: 'Carl'})
+        CREATE (n:Dog)
+        """,
+    )
+    result = execute_and_fetch_all(
+        cursor,
+        f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+    )
+
+    assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", False]
+    assert (list(result[1])) == [":`Dog`", ["Dog"], "owner", "String", False]
+    assert (result.__len__()) == 2
+
+
+def test_node_type_properties4():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (n:Label1:Label2 {property1: 'value1', property2: 'value2'})
+        CREATE (m:Label2:Label1 {property3: 'value3'})
+        """,
+    )
+    result = list(
+        execute_and_fetch_all(
+            cursor,
+            f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+        )
+    )
+    assert (list(result[0])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property1", "String", False]
+    assert (list(result[1])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property2", "String", False]
+    assert (list(result[2])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property3", "String", False]
+    assert (result.__len__()) == 3
+
+
+def test_node_type_properties5():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (d:Dog {name: 'Rex'})
+        """,
+    )
+    result = execute_and_fetch_all(
+        cursor,
+        f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
+    )
+
+    assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", True]
+    assert (result.__len__()) == 1
 
 
 def test_rel_type_properties1():
@@ -473,5 +547,38 @@ def test_rel_type_properties1():
     assert (result) == [":`LOVES`", "", "", False]
 
 
+def test_rel_type_properties2():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (d:Dog {name: 'Rex', owner: 'Carl'})-[l:LOVES]->(a:Activity {name: 'Running', location: 'Zadar'})
+        CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'})
+        """,
+    )
+    result = execute_and_fetch_all(
+        cursor,
+        f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;",
+    )
+    assert (list(result[0])) == [":`LOVES`", "duration", "Int", False]
+    assert (result.__len__()) == 1
+
+
+def test_rel_type_properties3():
+    cursor = connect().cursor()
+    execute_and_fetch_all(
+        cursor,
+        """
+        CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'})
+        """,
+    )
+    result = execute_and_fetch_all(
+        cursor,
+        f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;",
+    )
+    assert (list(result[0])) == [":`LOVES`", "duration", "Int", True]
+    assert (result.__len__()) == 1
+
+
 if __name__ == "__main__":
     sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/runner.py

@@ -53,8 +53,8 @@ def run(args):
 
         # Setup.
         @atexit.register
-        def cleanup():
-            interactive_mg_runner.stop_all()
+        def cleanup(keep_directories=True):
+            interactive_mg_runner.stop_all(keep_directories)
 
         if "pre_set_workload" in workload:
             binary = os.path.join(BUILD_DIR, workload["pre_set_workload"])
@@ -92,7 +92,7 @@ def run(args):
                     data = mg_instance.query(validation["query"], conn)[0][0]
                     assert data == validation["expected"]
                 conn.close()
-        cleanup()
+        cleanup(keep_directories=False)
         log.info("%s PASSED.", workload_name)
 
 

tests/e2e/streams/common.py

@@ -9,12 +9,12 @@
 # by the Apache License, Version 2.0, included in the file
 # licenses/APL.txt.
 
+import time
+from multiprocessing import Manager, Process, Value
+
 import mgclient
 import pytest
-import time
-
 from mg_utils import mg_sleep_and_assert
-from multiprocessing import Manager, Process, Value
 
 # These are the indices of the different values in the result of SHOW STREAM
 # query
@@ -103,15 +103,53 @@ def get_stream_info(cursor, stream_name):
 
 def get_is_running(cursor, stream_name):
     stream_info = get_stream_info(cursor, stream_name)
-
-    assert stream_info
+    assert stream_info is not None
     return stream_info[IS_RUNNING]
 
 
-def start_stream(cursor, stream_name):
-    execute_and_fetch_all(cursor, f"START STREAM {stream_name}")
+def create_stream(
+    cursor,
+    stream_name,
+    topics,
+    transformation,
+    consumer_group=None,
+    batch_interval=None,
+    batch_size=None,
+    bootstrap_servers=None,
+    configs=None,
+    credentials=None,
+):
+    query_str = f"CREATE KAFKA STREAM {stream_name} TOPICS {topics} TRANSFORM {transformation}"
+    if consumer_group is not None:
+        query_str += f" CONSUMER_GROUP {consumer_group}"
+    if batch_interval is not None:
+        query_str += f" BATCH_INTERVAL {batch_interval}"
+    if batch_size is not None:
+        query_str += f" BATCH_SIZE {batch_size}"
+    if bootstrap_servers is not None:
+        query_str += f" BOOTSTRAP_SERVERS {bootstrap_servers}"
+    if configs is not None:
+        query_str += f" CONFIGS {configs}"
+    if credentials is not None:
+        query_str += f" CREDENTIALS {credentials}"
+    execute_and_fetch_all(cursor, query_str)
 
+
+def start_stream(cursor, stream_name, sleep=True):
+    # Sleep is needed because although is_running returns True,
+    # the stream cannot accept messages yet
+    execute_and_fetch_all(cursor, f"START STREAM {stream_name}")
     assert get_is_running(cursor, stream_name)
+    if sleep:
+        time.sleep(5)
+
+
+def start_streams(cursor, stream_names):
+    # Start every stream but don't sleep after each creation
+    for stream_name in stream_names:
+        execute_and_fetch_all(cursor, f"START STREAM {stream_name}")
+        assert get_is_running(cursor, stream_name)
+    time.sleep(5)
 
 
 def start_stream_with_limit(cursor, stream_name, batch_limit, timeout=None):
@@ -123,13 +161,11 @@ def start_stream_with_limit(cursor, stream_name, batch_limit, timeout=None):
 
 def stop_stream(cursor, stream_name):
     execute_and_fetch_all(cursor, f"STOP STREAM {stream_name}")
-
     assert not get_is_running(cursor, stream_name)
 
 
 def drop_stream(cursor, stream_name):
     execute_and_fetch_all(cursor, f"DROP STREAM {stream_name}")
-
     assert get_stream_info(cursor, stream_name) is None
 
 
@@ -227,7 +263,7 @@ def test_start_and_stop_during_check(
     try:
         check_stream_proc.start()
 
-        time.sleep(0.5)
+        time.sleep(3)
 
         assert timed_wait(lambda: check_counter.value == CHECK_BEFORE_EXECUTE)
         assert timed_wait(lambda: get_is_running(cursor, "test_stream"))
@@ -260,25 +296,22 @@ def test_start_and_stop_during_check(
 
 def test_start_checked_stream_after_timeout(connection, stream_creator):
     cursor = connection.cursor()
-    execute_and_fetch_all(cursor, stream_creator("test_stream"))
+    stream_name = "test_start_checked_stream_after_timeout"
+    execute_and_fetch_all(cursor, stream_creator(stream_name))
 
-    TIMEOUT_IN_MS = 2000
-    TIMEOUT_IN_SECONDS = TIMEOUT_IN_MS / 1000
+    timeout_in_ms = 5000
 
     def call_check():
-        execute_and_fetch_all(connect().cursor(), f"CHECK STREAM test_stream TIMEOUT {TIMEOUT_IN_MS}")
+        execute_and_fetch_all(connect().cursor(), f"CHECK STREAM {stream_name} TIMEOUT {timeout_in_ms}")
 
     check_stream_proc = Process(target=call_check, daemon=True)
 
-    start = time.time()
     check_stream_proc.start()
-    assert timed_wait(lambda: get_is_running(cursor, "test_stream"))
-    start_stream(cursor, "test_stream")
-    end = time.time()
+    assert timed_wait(lambda: get_is_running(cursor, stream_name))
+    start_stream(cursor, stream_name)
 
-    assert (end - start) < 1.3 * TIMEOUT_IN_SECONDS, "The START STREAM was blocked too long"
-    assert get_is_running(cursor, "test_stream")
-    stop_stream(cursor, "test_stream")
+    assert get_is_running(cursor, stream_name)
+    stop_stream(cursor, stream_name)
 
 
 def test_check_stream_same_number_of_queries_than_messages(connection, stream_creator, message_sender):
@@ -313,7 +346,6 @@ def test_check_stream_same_number_of_queries_than_messages(connection, stream_cr
     # #            {parameters: {"value": "Parameter: 04"}, query: "Message: 04"}]
     # # -Batch 3: [{parameters: {"value": "Parameter: 05"}, query: "Message: 05"},
     # #            {parameters: {"value": "Parameter: 06"}, query: "Message: 06"}]
-
     assert len(test_results.value) == BATCH_LIMIT
 
     expected_queries_and_raw_messages_1 = (
@@ -351,7 +383,7 @@ def test_check_stream_different_number_of_queries_than_messages(connection, stre
     STREAM_NAME = "test_stream"
     cursor = connection.cursor()
     execute_and_fetch_all(cursor, stream_creator(STREAM_NAME, BATCH_SIZE))
-    time.sleep(2)
+    time.sleep(3)
 
     results = Manager().Namespace()
 
@@ -413,30 +445,30 @@ def test_check_stream_different_number_of_queries_than_messages(connection, stre
     assert expected_queries_and_raw_messages_3 == results.value[2]
 
 
-def test_start_stream_with_batch_limit(connection, stream_creator, messages_sender):
-    STREAM_NAME = "test"
+def test_start_stream_with_batch_limit(connection, stream_name, stream_creator, messages_sender):
     BATCH_LIMIT = 5
 
     cursor = connection.cursor()
-    execute_and_fetch_all(cursor, stream_creator(STREAM_NAME))
+    execute_and_fetch_all(cursor, stream_creator())
+    results = execute_and_fetch_all(connection.cursor(), "SHOW STREAMS")
+    assert len(results) == 1
 
-    def start_new_stream_with_limit(stream_name, batch_limit):
+    def start_new_stream_with_limit():
         connection = connect()
         cursor = connection.cursor()
-        start_stream_with_limit(cursor, stream_name, batch_limit)
-
-    thread_stream_running = Process(target=start_new_stream_with_limit, daemon=True, args=(STREAM_NAME, BATCH_LIMIT))
-    thread_stream_running.start()
+        start_stream_with_limit(cursor, stream_name, BATCH_LIMIT)
 
     def is_running():
-        return get_is_running(cursor, STREAM_NAME)
+        return get_is_running(cursor, stream_name)
+
+    thread_stream_running = Process(target=start_new_stream_with_limit)
+    thread_stream_running.start()
 
     assert mg_sleep_and_assert(True, is_running)
-
     messages_sender(BATCH_LIMIT - 1)
 
     # We have not sent enough batches to reach the limit. We check that the stream is still correctly running.
-    assert get_is_running(cursor, STREAM_NAME)
+    assert get_is_running(cursor, stream_name)
 
     # We send a last message to reach the batch_limit
     messages_sender(1)
@@ -549,7 +581,6 @@ def test_check_while_stream_with_batch_limit_running(connection, stream_creator,
     STREAM_NAME = "test_batch_limit_and_check"
     BATCH_LIMIT = 1
     TIMEOUT = 10000
-    TIMEOUT_IN_SECONDS = TIMEOUT / 1000
 
     cursor = connection.cursor()
     execute_and_fetch_all(cursor, stream_creator(STREAM_NAME))
@@ -625,7 +656,7 @@ def test_check_stream_with_batch_limit_with_invalid_batch_limit(connection, stre
 
     cursor = connection.cursor()
     execute_and_fetch_all(cursor, stream_creator(STREAM_NAME))
-    time.sleep(2)
+    time.sleep(3)
 
     # 1/ checking with batch_limit=-10
     batch_limit = -10

tests/e2e/streams/conftest.py

@@ -9,15 +9,14 @@
 # by the Apache License, Version 2.0, included in the file
 # licenses/APL.txt.
 
+import pulsar
 import pytest
+from common import NAME, PULSAR_SERVICE_URL, connect, execute_and_fetch_all
 from kafka import KafkaProducer
 from kafka.admin import KafkaAdminClient, NewTopic
 
-import pulsar
 import requests
 
-from common import NAME, connect, execute_and_fetch_all, PULSAR_SERVICE_URL
-
 # To run these test locally a running Kafka sever is necessery. The test tries
 # to connect on localhost:9092.
 
@@ -37,29 +36,22 @@ def connection():
 
 
 def get_topics(num):
-    return [f'topic_{i}' for i in range(num)]
+    return [f"topic_{i}" for i in range(num)]
 
 
 @pytest.fixture(scope="function")
 def kafka_topics():
-    admin_client = KafkaAdminClient(
-        bootstrap_servers="localhost:9092",
-        client_id="test")
+    admin_client = KafkaAdminClient(bootstrap_servers="localhost:29092", client_id="test")
     # The issue arises if we remove default kafka topics, e.g.
     # "__consumer_offsets"
-    previous_topics = [
-        topic for topic in admin_client.list_topics() if topic != "__consumer_offsets"]
+    previous_topics = [topic for topic in admin_client.list_topics() if topic != "__consumer_offsets"]
     if previous_topics:
         admin_client.delete_topics(topics=previous_topics, timeout_ms=5000)
 
     topics = get_topics(3)
     topics_to_create = []
     for topic in topics:
-        topics_to_create.append(
-            NewTopic(
-                name=topic,
-                num_partitions=1,
-                replication_factor=1))
+        topics_to_create.append(NewTopic(name=topic, num_partitions=1, replication_factor=1))
 
     admin_client.create_topics(new_topics=topics_to_create, timeout_ms=5000)
     yield topics
@@ -68,7 +60,7 @@ def kafka_topics():
 
 @pytest.fixture(scope="function")
 def kafka_producer():
-    yield KafkaProducer(bootstrap_servers="localhost:9092")
+    yield KafkaProducer(bootstrap_servers=["localhost:29092"], api_version_auto_timeout_ms=10000)
 
 
 @pytest.fixture(scope="function")
@@ -80,6 +72,5 @@ def pulsar_client():
 def pulsar_topics():
     topics = get_topics(3)
     for topic in topics:
-        requests.delete(
-            f'http://127.0.0.1:6652/admin/v2/persistent/public/default/{topic}?force=true')
+        requests.delete(f"http://localhost:6652/admin/v2/persistent/public/default/{topic}?force=true")
     yield topics

tests/e2e/streams/kafka.yml

@@ -1,20 +0,0 @@
-version: '3.7'
-services:
-  zookeeper:
-    image: 'bitnami/zookeeper:latest'
-    ports:
-      - '2181:2181'
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
-  kafka:
-    image: 'bitnami/kafka:latest'
-    ports:
-      - '9092:9092'
-    environment:
-      - KAFKA_BROKER_ID=1
-      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092
-      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
-      - ALLOW_PLAINTEXT_LISTENER=yes
-    depends_on:
-      - zookeeper

tests/e2e/streams/kafka_streams_tests.py

@@ -12,31 +12,31 @@
 # licenses/APL.txt.
 
 import sys
-import pytest
-import mgclient
 import time
-from mg_utils import mg_sleep_and_assert
-from multiprocessing import Process, Value
+from multiprocessing import Process
+
 import common
+import mgclient
+import pytest
+from mg_utils import mg_sleep_and_assert
 
 TRANSFORMATIONS_TO_CHECK_C = ["c_transformations.empty_transformation"]
-
 TRANSFORMATIONS_TO_CHECK_PY = ["kafka_transform.simple", "kafka_transform.with_parameters"]
+KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT = 60
+KAFKA_PRODUCER_SENDING_MSG_LARGE_TIMEOUT = 6000
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
 def test_simple(kafka_producer, kafka_topics, connection, transformation):
     assert len(kafka_topics) > 0
+    stream_name = "test_simple_" + transformation.split(".")[1]
+
     cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test TOPICS {','.join(kafka_topics)} TRANSFORM {transformation}",
-    )
-    common.start_stream(cursor, "test")
-    time.sleep(5)
+    common.create_stream(cursor, stream_name, ",".join(kafka_topics), transformation)
+    common.start_stream(cursor, stream_name)
 
     for topic in kafka_topics:
-        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=60)
+        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     for topic in kafka_topics:
         common.kafka_check_vertex_exists_with_topic_and_payload(cursor, topic, common.SIMPLE_MSG)
@@ -47,22 +47,15 @@ def test_separate_consumers(kafka_producer, kafka_topics, connection, transforma
     assert len(kafka_topics) > 0
     cursor = connection.cursor()
 
-    stream_names = []
-    for topic in kafka_topics:
-        stream_name = "stream_" + topic
-        stream_names.append(stream_name)
-        common.execute_and_fetch_all(
-            cursor,
-            f"CREATE KAFKA STREAM {stream_name} TOPICS {topic} TRANSFORM {transformation}",
-        )
+    stream_names = ["stream_" + transformation.split(".")[1] + "_" + topic for topic in kafka_topics]
 
-    for stream_name in stream_names:
-        common.start_stream(cursor, stream_name)
+    for stream_name, topic in zip(stream_names, kafka_topics):
+        common.create_stream(cursor, stream_name, topic, transformation)
 
-    time.sleep(5)
+    common.start_streams(cursor, stream_names)
 
     for topic in kafka_topics:
-        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=60)
+        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     for topic in kafka_topics:
         common.kafka_check_vertex_exists_with_topic_and_payload(cursor, topic, common.SIMPLE_MSG)
@@ -77,23 +70,20 @@ def test_start_from_last_committed_offset(kafka_producer, kafka_topics, connecti
     # restarting Memgraph during a single workload cannot be done currently.
     assert len(kafka_topics) > 0
     cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple",
-    )
-    common.start_stream(cursor, "test")
-    time.sleep(1)
 
-    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=60)
+    stream_name = "test_start_from_last_committed_offset"
+    common.create_stream(cursor, stream_name, kafka_topics[0], "kafka_transform.simple")
+    common.start_stream(cursor, stream_name)
 
+    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
     common.kafka_check_vertex_exists_with_topic_and_payload(cursor, kafka_topics[0], common.SIMPLE_MSG)
 
-    common.stop_stream(cursor, "test")
-    common.drop_stream(cursor, "test")
+    common.stop_stream(cursor, stream_name)
+    common.drop_stream(cursor, stream_name)
 
     messages = [b"second message", b"third message"]
     for message in messages:
-        kafka_producer.send(kafka_topics[0], message).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], message).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     for message in messages:
         vertices_with_msg = common.execute_and_fetch_all(
@@ -103,11 +93,8 @@ def test_start_from_last_committed_offset(kafka_producer, kafka_topics, connecti
 
         assert len(vertices_with_msg) == 0
 
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple",
-    )
-    common.start_stream(cursor, "test")
+    common.create_stream(cursor, stream_name, kafka_topics[0], "kafka_transform.simple")
+    common.start_stream(cursor, stream_name)
 
     for message in messages:
         common.kafka_check_vertex_exists_with_topic_and_payload(cursor, kafka_topics[0], message)
@@ -118,32 +105,26 @@ def test_check_stream(kafka_producer, kafka_topics, connection, transformation):
     assert len(kafka_topics) > 0
     BATCH_SIZE = 1
     INDEX_OF_FIRST_BATCH = 0
-    cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test TOPICS {kafka_topics[0]} TRANSFORM {transformation} BATCH_SIZE {BATCH_SIZE}",
-    )
-    common.start_stream(cursor, "test")
-    time.sleep(1)
+    stream_name = "test_check_stream_" + transformation.split(".")[1]
 
-    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=60)
-    common.stop_stream(cursor, "test")
+    cursor = connection.cursor()
+
+    common.create_stream(cursor, stream_name, kafka_topics[0], transformation, batch_size=BATCH_SIZE)
+    common.start_stream(cursor, stream_name)
+    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
+    common.stop_stream(cursor, stream_name)
 
     messages = [b"first message", b"second message", b"third message"]
     for message in messages:
-        kafka_producer.send(kafka_topics[0], message).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], message).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     def check_check_stream(batch_limit):
-        assert transformation == "kafka_transform.simple" or transformation == "kafka_transform.with_parameters"
-        test_results = common.execute_and_fetch_all(cursor, f"CHECK STREAM test BATCH_LIMIT {batch_limit}")
+        test_results = common.execute_and_fetch_all(cursor, f"CHECK STREAM {stream_name} BATCH_LIMIT {batch_limit}")
         assert len(test_results) == batch_limit
 
         for i in range(batch_limit):
             message_as_str = messages[i].decode("utf-8")
-            assert (
-                BATCH_SIZE == 1
-            )  # If batch size != 1, then the usage of INDEX_OF_FIRST_BATCH must change: the result will have a list of queries (pair<parameters,query>)
-
+            # If batch size != 1, then the usage of INDEX_OF_FIRST_BATCH must change: the result will have a list of queries (pair<parameters,query>)
             if transformation == "kafka_transform.simple":
                 assert (
                     f"payload: '{message_as_str}'"
@@ -165,41 +146,48 @@ def test_check_stream(kafka_producer, kafka_topics, connection, transformation):
     check_check_stream(1)
     check_check_stream(2)
     check_check_stream(3)
-    common.start_stream(cursor, "test")
+    common.start_stream(cursor, stream_name)
 
     for message in messages:
         common.kafka_check_vertex_exists_with_topic_and_payload(cursor, kafka_topics[0], message)
 
 
-def test_show_streams(kafka_producer, kafka_topics, connection):
+def test_show_streams(kafka_topics, connection):
     assert len(kafka_topics) > 1
     cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM default_values TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple BOOTSTRAP_SERVERS 'localhost:9092'",
-    )
 
     consumer_group = "my_special_consumer_group"
     BATCH_INTERVAL = 42
     BATCH_SIZE = 3
-    common.execute_and_fetch_all(
+    default_values_stream = "default_values"
+    complex_values_stream = "complex_values"
+
+    common.create_stream(
+        cursor, default_values_stream, kafka_topics[0], "kafka_transform.simple", bootstrap_servers="'localhost:29092'"
+    )
+    common.create_stream(
         cursor,
-        f"CREATE KAFKA STREAM complex_values TOPICS {','.join(kafka_topics)} TRANSFORM kafka_transform.with_parameters CONSUMER_GROUP {consumer_group} BATCH_INTERVAL {BATCH_INTERVAL} BATCH_SIZE {BATCH_SIZE} ",
+        complex_values_stream,
+        ",".join(kafka_topics),
+        "kafka_transform.with_parameters",
+        consumer_group=consumer_group,
+        batch_interval=BATCH_INTERVAL,
+        batch_size=BATCH_SIZE,
     )
 
     assert len(common.execute_and_fetch_all(cursor, "SHOW STREAMS")) == 2
 
     common.check_stream_info(
         cursor,
-        "default_values",
-        ("default_values", "kafka", 100, 1000, "kafka_transform.simple", None, False),
+        default_values_stream,
+        (default_values_stream, "kafka", 100, 1000, "kafka_transform.simple", None, False),
     )
 
     common.check_stream_info(
         cursor,
-        "complex_values",
+        complex_values_stream,
         (
-            "complex_values",
+            complex_values_stream,
             "kafka",
             BATCH_INTERVAL,
             BATCH_SIZE,
@@ -219,7 +207,7 @@ def test_start_and_stop_during_check(kafka_producer, kafka_topics, connection, o
         return f"CREATE KAFKA STREAM {stream_name} TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple BATCH_SIZE {BATCH_SIZE}"
 
     def message_sender(msg):
-        kafka_producer.send(kafka_topics[0], msg).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], msg).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     common.test_start_and_stop_during_check(
         operation,
@@ -235,14 +223,12 @@ def test_check_already_started_stream(kafka_topics, connection):
     assert len(kafka_topics) > 0
     cursor = connection.cursor()
 
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM started_stream TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple",
-    )
-    common.start_stream(cursor, "started_stream")
+    stream_name = "test_check_already_started_stream"
+    common.create_stream(cursor, stream_name, kafka_topics[0], "kafka_transform.simple")
+    common.start_stream(cursor, stream_name)
 
     with pytest.raises(mgclient.DatabaseError):
-        common.execute_and_fetch_all(cursor, "CHECK STREAM started_stream")
+        common.execute_and_fetch_all(cursor, f"CHECK STREAM {stream_name}")
 
 
 def test_start_checked_stream_after_timeout(kafka_topics, connection):
@@ -254,19 +240,14 @@ def test_start_checked_stream_after_timeout(kafka_topics, connection):
 
 def test_restart_after_error(kafka_producer, kafka_topics, connection):
     cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test_stream TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.query",
-    )
+    stream_name = "test_restart_after_error"
+    common.create_stream(cursor, stream_name, kafka_topics[0], "kafka_transform.query")
+    common.start_stream(cursor, stream_name)
 
-    common.start_stream(cursor, "test_stream")
-    time.sleep(1)
+    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
+    assert common.timed_wait(lambda: not common.get_is_running(cursor, stream_name))
 
-    kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=60)
-    assert common.timed_wait(lambda: not common.get_is_running(cursor, "test_stream"))
-
-    common.start_stream(cursor, "test_stream")
-    time.sleep(1)
+    common.start_stream(cursor, stream_name)
     kafka_producer.send(kafka_topics[0], b"CREATE (n:VERTEX { id : 42 })")
     assert common.check_one_result_row(cursor, "MATCH (n:VERTEX { id : 42 }) RETURN n")
 
@@ -275,23 +256,21 @@ def test_restart_after_error(kafka_producer, kafka_topics, connection):
 def test_bootstrap_server(kafka_producer, kafka_topics, connection, transformation):
     assert len(kafka_topics) > 0
     cursor = connection.cursor()
-    LOCAL = "localhost:9092"
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE KAFKA STREAM test TOPICS {','.join(kafka_topics)} TRANSFORM {transformation} BOOTSTRAP_SERVERS '{LOCAL}'",
-    )
-    common.start_stream(cursor, "test")
-    time.sleep(5)
+    local = "'localhost:29092'"
+    stream_name = "test_bootstrap_server_" + transformation.split(".")[1]
+
+    common.create_stream(cursor, stream_name, ",".join(kafka_topics), transformation, bootstrap_servers=local)
+    common.start_stream(cursor, stream_name)
 
     for topic in kafka_topics:
-        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=60)
+        kafka_producer.send(topic, common.SIMPLE_MSG).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     for topic in kafka_topics:
         common.kafka_check_vertex_exists_with_topic_and_payload(cursor, topic, common.SIMPLE_MSG)
 
 
 @pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
-def test_bootstrap_server_empty(kafka_producer, kafka_topics, connection, transformation):
+def test_bootstrap_server_empty(kafka_topics, connection, transformation):
     assert len(kafka_topics) > 0
     cursor = connection.cursor()
     with pytest.raises(mgclient.DatabaseError):
@@ -312,7 +291,7 @@ def test_set_offset(kafka_producer, kafka_topics, connection, transformation):
 
     messages = [f"{i} message" for i in range(1, 21)]
     for message in messages:
-        kafka_producer.send(kafka_topics[0], message.encode()).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], message.encode()).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     def consume(expected_msgs):
         common.start_stream(cursor, "test")
@@ -352,7 +331,7 @@ def test_set_offset(kafka_producer, kafka_topics, connection, transformation):
     res = execute_set_offset_and_consume(-2, [])
     assert len(res) == 0
     last_msg = "Final Message"
-    kafka_producer.send(kafka_topics[0], last_msg.encode()).get(timeout=60)
+    kafka_producer.send(kafka_topics[0], last_msg.encode()).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
     res = consume([last_msg])
     assert len(res) == 1
     assert comparison_check("Final Message", res[0])
@@ -361,21 +340,27 @@ def test_set_offset(kafka_producer, kafka_topics, connection, transformation):
 
 def test_info_procedure(kafka_topics, connection):
     cursor = connection.cursor()
-    STREAM_NAME = "test_stream"
-    CONFIGS = {"sasl.username": "michael.scott"}
-    LOCAL = "localhost:9092"
-    CREDENTIALS = {"sasl.password": "S3cr3tP4ssw0rd"}
-    CONSUMER_GROUP = "ConsumerGr"
-    common.execute_and_fetch_all(
+    stream_name = "test_stream"
+    configs = {"sasl.username": "michael.scott"}
+    local = "localhost:29092"
+    credentials = {"sasl.password": "S3cr3tP4ssw0rd"}
+    consumer_group = "ConsumerGr"
+
+    common.create_stream(
         cursor,
-        f"CREATE KAFKA STREAM {STREAM_NAME} TOPICS {','.join(kafka_topics)} TRANSFORM kafka_transform.simple CONSUMER_GROUP {CONSUMER_GROUP} BOOTSTRAP_SERVERS '{LOCAL}' CONFIGS {CONFIGS} CREDENTIALS {CREDENTIALS}",
+        stream_name,
+        ",".join(kafka_topics),
+        "kafka_transform.simple",
+        consumer_group=consumer_group,
+        bootstrap_servers=f"'{local}'",
+        configs=configs,
+        credentials=credentials,
     )
+    stream_info = common.execute_and_fetch_all(cursor, f"CALL mg.kafka_stream_info('{stream_name}') YIELD *")
 
-    stream_info = common.execute_and_fetch_all(cursor, f"CALL mg.kafka_stream_info('{STREAM_NAME}') YIELD *")
+    reducted_credentials = {key: "<REDUCTED>" for key in credentials.keys()}
 
-    reducted_credentials = {key: "<REDUCTED>" for key in CREDENTIALS.keys()}
-
-    expected_stream_info = [(LOCAL, CONFIGS, CONSUMER_GROUP, reducted_credentials, kafka_topics)]
+    expected_stream_info = [(local, configs, consumer_group, reducted_credentials, kafka_topics)]
     common.validate_info(stream_info, expected_stream_info)
 
 
@@ -398,7 +383,7 @@ def test_check_stream_same_number_of_queries_than_messages(kafka_producer, kafka
         return f"CREATE KAFKA STREAM {stream_name} TOPICS {kafka_topics[0]} TRANSFORM {TRANSFORMATION} BATCH_INTERVAL 3000 BATCH_SIZE {batch_size}"
 
     def message_sender(msg):
-        kafka_producer.send(kafka_topics[0], msg).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], msg).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     common.test_check_stream_same_number_of_queries_than_messages(connection, stream_creator, message_sender)
 
@@ -409,30 +394,33 @@ def test_check_stream_different_number_of_queries_than_messages(kafka_producer,
     TRANSFORMATION = "common_transform.check_stream_with_filtering"
 
     def stream_creator(stream_name, batch_size):
-        return f"CREATE KAFKA STREAM {stream_name} TOPICS {kafka_topics[0]} TRANSFORM {TRANSFORMATION} BATCH_INTERVAL 3000  BATCH_SIZE {batch_size}"
+        return f"CREATE KAFKA STREAM {stream_name} TOPICS {kafka_topics[0]} TRANSFORM {TRANSFORMATION} BATCH_INTERVAL 3000 BATCH_SIZE {batch_size}"
 
     def message_sender(msg):
-        kafka_producer.send(kafka_topics[0], msg).get(timeout=60)
+        kafka_producer.send(kafka_topics[0], msg).get(timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT)
 
     common.test_check_stream_different_number_of_queries_than_messages(connection, stream_creator, message_sender)
 
 
 def test_start_stream_with_batch_limit(kafka_producer, kafka_topics, connection):
     assert len(kafka_topics) > 0
+    STREAM_NAME = "test_start_stream_with_batch_limit"
 
-    def stream_creator(stream_name):
+    def stream_creator():
         return (
-            f"CREATE KAFKA STREAM {stream_name} TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple BATCH_SIZE 1"
+            f"CREATE KAFKA STREAM {STREAM_NAME} TOPICS {kafka_topics[0]} TRANSFORM kafka_transform.simple BATCH_SIZE 1"
         )
 
     def messages_sender(nof_messages):
-        for x in range(nof_messages):
-            kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(timeout=60)
+        for _ in range(nof_messages):
+            kafka_producer.send(kafka_topics[0], common.SIMPLE_MSG).get(
+                timeout=KAFKA_PRODUCER_SENDING_MSG_DEFAULT_TIMEOUT
+            )
 
-    common.test_start_stream_with_batch_limit(connection, stream_creator, messages_sender)
+    common.test_start_stream_with_batch_limit(connection, STREAM_NAME, stream_creator, messages_sender)
 
 
-def test_start_stream_with_batch_limit_timeout(kafka_producer, kafka_topics, connection):
+def test_start_stream_with_batch_limit_timeout(kafka_topics, connection):
     assert len(kafka_topics) > 0
 
     def stream_creator(stream_name):
@@ -443,7 +431,7 @@ def test_start_stream_with_batch_limit_timeout(kafka_producer, kafka_topics, con
     common.test_start_stream_with_batch_limit_timeout(connection, stream_creator)
 
 
-def test_start_stream_with_batch_limit_reaching_timeout(kafka_producer, kafka_topics, connection):
+def test_start_stream_with_batch_limit_reaching_timeout(kafka_topics, connection):
     assert len(kafka_topics) > 0
 
     def stream_creator(stream_name, batch_size):
@@ -461,7 +449,7 @@ def test_start_stream_with_batch_limit_while_check_running(kafka_producer, kafka
         )
 
     def message_sender(message):
-        kafka_producer.send(kafka_topics[0], message).get(timeout=6000)
+        kafka_producer.send(kafka_topics[0], message).get(timeout=KAFKA_PRODUCER_SENDING_MSG_LARGE_TIMEOUT)
 
     def setup_function(start_check_stream, cursor, stream_name, batch_limit, timeout):
         thread_stream_check = Process(target=start_check_stream, daemon=True, args=(stream_name, batch_limit, timeout))
@@ -488,12 +476,12 @@ def test_check_while_stream_with_batch_limit_running(kafka_producer, kafka_topic
         )
 
     def message_sender(message):
-        kafka_producer.send(kafka_topics[0], message).get(timeout=6000)
+        kafka_producer.send(kafka_topics[0], message).get(timeout=KAFKA_PRODUCER_SENDING_MSG_LARGE_TIMEOUT)
 
     common.test_check_while_stream_with_batch_limit_running(connection, stream_creator, message_sender)
 
 
-def test_start_stream_with_batch_limit_with_invalid_batch_limit(kafka_producer, kafka_topics, connection):
+def test_start_stream_with_batch_limit_with_invalid_batch_limit(kafka_topics, connection):
     assert len(kafka_topics) > 0
 
     def stream_creator(stream_name):
@@ -504,7 +492,7 @@ def test_start_stream_with_batch_limit_with_invalid_batch_limit(kafka_producer,
     common.test_start_stream_with_batch_limit_with_invalid_batch_limit(connection, stream_creator)
 
 
-def test_check_stream_with_batch_limit_with_invalid_batch_limit(kafka_producer, kafka_topics, connection):
+def test_check_stream_with_batch_limit_with_invalid_batch_limit(kafka_topics, connection):
     assert len(kafka_topics) > 0
 
     def stream_creator(stream_name):

tests/e2e/streams/pulsar.yml

@@ -1,8 +0,0 @@
-version: '3.7'
-services:
-  pulsar:
-    image: 'apachepulsar/pulsar:latest'
-    ports:
-      - '6652:8080'
-      - '6650:6650'
-    entrypoint: ['bin/pulsar', 'standalone']

tests/e2e/streams/pulsar_streams_tests.py

@@ -12,11 +12,12 @@
 # licenses/APL.txt.
 
 import sys
-import pytest
-import mgclient
 import time
 from multiprocessing import Process, Value
+
 import common
+import mgclient
+import pytest
 
 TRANSFORMATIONS_TO_CHECK = ["pulsar_transform.simple", "pulsar_transform.with_parameters"]
 
@@ -80,12 +81,6 @@ def test_start_from_latest_messages(pulsar_client, pulsar_topics, connection):
     # inbetween should be lost. Additionally, we check that consumer continues from the correct message
     # after stopping and starting again.
     assert len(pulsar_topics) > 0
-    cursor = connection.cursor()
-    common.execute_and_fetch_all(
-        cursor,
-        f"CREATE PULSAR STREAM test TOPICS {pulsar_topics[0]} TRANSFORM pulsar_transform.simple",
-    )
-    common.start_stream(cursor, "test")
 
     def assert_message_not_consumed(message):
         vertices_with_msg = common.execute_and_fetch_all(
@@ -95,6 +90,13 @@ def test_start_from_latest_messages(pulsar_client, pulsar_topics, connection):
 
         assert len(vertices_with_msg) == 0
 
+    cursor = connection.cursor()
+    common.execute_and_fetch_all(
+        cursor,
+        f"CREATE PULSAR STREAM test TOPICS {pulsar_topics[0]} TRANSFORM pulsar_transform.simple",
+    )
+    common.start_stream(cursor, "test")
+
     producer = pulsar_client.create_producer(
         common.pulsar_default_namespace_topic(pulsar_topics[0]), send_timeout_millis=60000
     )
@@ -131,7 +133,7 @@ def test_start_from_latest_messages(pulsar_client, pulsar_topics, connection):
         producer.send(message)
         assert_message_not_consumed(message)
 
-    common.start_stream(cursor, "test")
+    common.start_stream(cursor, "test", sleep=False)
 
     assert_message_not_consumed(LOST_MESSAGE)
 
@@ -338,19 +340,20 @@ def test_service_url(pulsar_client, pulsar_topics, connection, transformation):
 
 def test_start_stream_with_batch_limit(pulsar_client, pulsar_topics, connection):
     assert len(pulsar_topics) > 1
+    STREAM_NAME = "test_start_stream_with_batch_limit"
 
-    def stream_creator(stream_name):
-        return f"CREATE PULSAR STREAM {stream_name} TOPICS {pulsar_topics[0]} TRANSFORM pulsar_transform.simple BATCH_SIZE 1"
+    def stream_creator():
+        return f"CREATE PULSAR STREAM {STREAM_NAME} TOPICS {pulsar_topics[0]} TRANSFORM pulsar_transform.simple BATCH_SIZE 1"
 
     producer = pulsar_client.create_producer(
         common.pulsar_default_namespace_topic(pulsar_topics[0]), send_timeout_millis=60000
     )
 
     def messages_sender(nof_messages):
-        for x in range(nof_messages):
+        for _ in range(nof_messages):
             producer.send(common.SIMPLE_MSG)
 
-    common.test_start_stream_with_batch_limit(connection, stream_creator, messages_sender)
+    common.test_start_stream_with_batch_limit(connection, STREAM_NAME, stream_creator, messages_sender)
 
 
 def test_start_stream_with_batch_limit_timeout(pulsar_client, pulsar_topics, connection):
@@ -409,7 +412,7 @@ def test_check_stream_same_number_of_queries_than_messages(pulsar_client, pulsar
     TRANSFORMATION = "common_transform.check_stream_no_filtering"
 
     def stream_creator(stream_name, batch_size):
-        return f"CREATE PULSAR STREAM {stream_name} TOPICS {pulsar_topics[0]} TRANSFORM {TRANSFORMATION} BATCH_INTERVAL 3000 BATCH_SIZE  {batch_size} "
+        return f"CREATE PULSAR STREAM {stream_name} TOPICS {pulsar_topics[0]} TRANSFORM {TRANSFORMATION} BATCH_INTERVAL 3000 BATCH_SIZE {batch_size} "
 
     producer = pulsar_client.create_producer(
         common.pulsar_default_namespace_topic(pulsar_topics[0]), send_timeout_millis=60000

tests/stress/memory_limit.py

@@ -173,39 +173,49 @@ def run_writer(repetition_count: int, sleep_sec: float, worker_id: int) -> int:
     """
     This writer creates lot of nodes on each write.
     Also it checks that query failed if memory limit is tried to be broken
+
+    Return:
+        True if write suceeded
+        False otherwise
     """
 
     session = SessionCache.argument_session(args)
 
-    def create() -> bool:
+    def try_create() -> bool:
         """
-        Returns True if done, False if needs to continue
+        Function tries to create until memory limit is reached
+        Return:
+            True if it can continue creating (OOM not reached)
+            False otherwise
         """
-        memory_tracker_data_before_start = get_tracker_data(session)
-        should_fail = memory_tracker_data_before_start >= 2048
-        failed = False
+        should_continue = True
         try:
             try_execute(
                 session,
                 f"FOREACH (i in range(1,10000) | CREATE (:Node {{prop:'big string or something like that'}}))",
             )
         except Exception as ex:
-            failed = True
             output = str(ex)
-            log.info("Exception in create", output)
-            assert "Memory limit exceeded!" in output
+            memory_over_2048_mb = False
+            memory_tracker_data_after_start = get_tracker_data(session)
+            if memory_tracker_data_after_start:
+                memory_over_2048_mb = memory_tracker_data_after_start >= 2048
+            log.info(
+                "Exception in create, exception output:",
+                output,
+                f"Worker {worker_id} started iteration {curr_repetition}, memory over 2048MB: {memory_over_2048_mb}",
+            )
+            has_oom_happend = "Memory limit exceeded!" in output and memory_over_2048_mb
+            should_continue = not has_oom_happend
 
-        if should_fail:
-            assert failed, "Query should have failed"
-            return False
-        return True
+        return should_continue
 
     curr_repetition = 0
 
     while curr_repetition < repetition_count:
         log.info(f"Worker {worker_id} started iteration {curr_repetition}")
 
-        should_continue = create()
+        should_continue = try_create()
 
         if not should_continue:
             return True
@@ -214,6 +224,7 @@ def run_writer(repetition_count: int, sleep_sec: float, worker_id: int) -> int:
         log.info(f"Worker {worker_id} created chain in iteration {curr_repetition}")
 
         curr_repetition += 1
+    return False
 
 
 def execute_function(worker: Worker) -> Worker:

tests/stress/memory_tracker.py

@@ -260,10 +260,11 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
             # Problem with test using detach delete and memory tracker
             # is that memory tracker gets updated immediately
             # whereas RES takes some time
-            cnt_again = 3
+            # Tries 10 times or fails
+            cnt_again = 10
             skip_failure = False
-            # 10% is maximum increment, afterwards is fail
-            multiplier = 1
+            # 10% is maximum diff for this test to pass
+            multiplier = 1.10
             while cnt_again:
                 new_memory_tracker, new_res_data = get_storage_data(session)
 
@@ -277,7 +278,6 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
                         f"RES data: {new_res_data}, multiplier: {multiplier}"
                     )
                     break
-                multiplier += 0.05
                 cnt_again -= 1
             if not skip_failure:
                 log.info(memory_tracker, initial_diff, res_data)

tests/unit/dbms_handler.cpp

@@ -58,6 +58,7 @@ class TestEnvironment : public ::testing::Environment {
   void TearDown() override {
     ptr_.reset();
     auth.reset();
+    std::filesystem::remove_all(storage_directory);
   }
 
   static std::unique_ptr<memgraph::dbms::DbmsHandler> ptr_;

tests/unit/dbms_handler_community.cpp

@@ -58,6 +58,7 @@ class TestEnvironment : public ::testing::Environment {
   void TearDown() override {
     ptr_.reset();
     auth.reset();
+    std::filesystem::remove_all(storage_directory);
   }
 
   static std::unique_ptr<memgraph::dbms::DbmsHandler> ptr_;

tests/unit/interpreter.cpp

@@ -101,6 +101,8 @@ class InterpreterTest : public ::testing::Test {
       disk_test_utils::RemoveRocksDbDirs(testSuite);
       disk_test_utils::RemoveRocksDbDirs(testSuiteCsv);
     }
+
+    std::filesystem::remove_all(data_directory);
   }
 
   InterpreterFaker default_interpreter{&interpreter_context, db};

tests/unit/query_dump.cpp

@@ -700,6 +700,11 @@ TYPED_TEST(DumpTest, CheckStateVertexWithMultipleProperties) {
     config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s1").disk;
     config.force_on_disk = true;
   }
+  auto on_exit_s1 = memgraph::utils::OnScopeExit{[&]() {
+    if constexpr (std::is_same_v<TypeParam, memgraph::storage::DiskStorage>) {
+      disk_test_utils::RemoveRocksDbDirs("query-dump-s1");
+    }
+  }};
   memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(config));
 
   memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gk(config, repl_state);
@@ -814,7 +819,11 @@ TYPED_TEST(DumpTest, CheckStateSimpleGraph) {
     config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s2").disk;
     config.force_on_disk = true;
   }
-
+  auto on_exit_s2 = memgraph::utils::OnScopeExit{[&]() {
+    if constexpr (std::is_same_v<TypeParam, memgraph::storage::DiskStorage>) {
+      disk_test_utils::RemoveRocksDbDirs("query-dump-s2");
+    }
+  }};
   memgraph::replication::ReplicationState repl_state{ReplicationStateRootPath(config)};
   memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gk{config, repl_state};
   auto db_acc_opt = db_gk.access();

tests/unit/storage_v2_isolation_level.cpp

@@ -15,6 +15,7 @@
 #include "storage/v2/disk/storage.hpp"
 #include "storage/v2/inmemory/storage.hpp"
 #include "storage/v2/isolation_level.hpp"
+#include "utils/on_scope_exit.hpp"
 
 namespace {
 int64_t VerticesCount(memgraph::storage::Storage::Accessor *accessor) {
@@ -113,6 +114,7 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) {
 
   for (const auto override_isolation_level : isolation_levels) {
     std::unique_ptr<memgraph::storage::Storage> storage(new memgraph::storage::DiskStorage(config));
+    auto on_exit = memgraph::utils::OnScopeExit{[&]() { disk_test_utils::RemoveRocksDbDirs(testSuite); }};
     try {
       this->TestVisibility(storage, default_isolation_level, override_isolation_level);
     } catch (memgraph::utils::NotYetImplemented &) {
@@ -120,10 +122,8 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) {
           override_isolation_level != memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION) {
         continue;
       }
-      disk_test_utils::RemoveRocksDbDirs(testSuite);
       throw;
     }
-    disk_test_utils::RemoveRocksDbDirs(testSuite);
   }
 }
 

tests/unit/storage_v2_storage_mode.cpp

@@ -75,6 +75,8 @@ class StorageModeMultiTxTest : public ::testing::Test {
     return tmp;
   }();  // iile
 
+  void TearDown() override { std::filesystem::remove_all(data_directory); }
+
   memgraph::storage::Config config{.durability.storage_directory = data_directory,
                                    .disk.main_storage_directory = data_directory / "disk"};