Add OOM enabler in operator tree (#1379)

tests/e2e/memory/CMakeLists.txt

@@ -11,3 +11,12 @@ target_link_libraries(memgraph__e2e__memory__limit_global_alloc gflags mgclient
 add_executable(memgraph__e2e__memory__limit_global_alloc_proc memory_limit_global_alloc_proc.cpp)
 target_link_libraries(memgraph__e2e__memory__limit_global_alloc_proc gflags mgclient mg-utils mg-io Threads::Threads)
 
+add_executable(memgraph__e2e__memory__limit_delete memory_limit_delete.cpp)
+target_link_libraries(memgraph__e2e__memory__limit_delete gflags mgclient mg-utils mg-io)
+
+
+add_executable(memgraph__e2e__memory__limit_accumulation memory_limit_accumulation.cpp)
+target_link_libraries(memgraph__e2e__memory__limit_accumulation gflags mgclient mg-utils mg-io)
+
+add_executable(memgraph__e2e__memory__limit_edge_create memory_limit_edge_create.cpp)
+target_link_libraries(memgraph__e2e__memory__limit_edge_create gflags mgclient mg-utils mg-io)

tests/e2e/memory/memory_limit_accumulation.cpp

@@ -0,0 +1,80 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <gflags/gflags.h>
+#include <iostream>
+#include <mgclient.hpp>
+
+#include "utils/logging.hpp"
+#include "utils/timer.hpp"
+
+DEFINE_uint64(bolt_port, 7687, "Bolt port");
+DEFINE_bool(multi_db, false, "Run test in multi db environment");
+
+int main(int argc, char **argv) {
+  google::SetUsageMessage("Memgraph E2E Memory Control");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  memgraph::logging::RedirectToStderr();
+
+  mg::Client::Init();
+
+  auto client =
+      mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
+  if (!client) {
+    LOG_FATAL("Failed to connect!");
+  }
+
+  client->Execute("MATCH (n) DETACH DELETE n;");
+  client->DiscardAll();
+
+  if (FLAGS_multi_db) {
+    client->Execute("CREATE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("USE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("MATCH (n) DETACH DELETE n;");
+    client->DiscardAll();
+  }
+
+  const auto *create_query = "UNWIND range(1, 500000) as u CREATE (n {id:u%5, string: 'Some longer string'}) RETURN n;";
+
+  try {
+    client->Execute(create_query);
+    [[maybe_unused]] auto results = client->FetchAll();
+    if (results->empty()) {
+      assert(false);
+      return 0;
+    }
+  } catch (const mg::TransientException & /*unused*/) {
+    spdlog::info("Memgraph is out of memory");
+    assert(false);
+    return 0;
+  }
+
+  const auto *accumulate_query = "MATCH (n) WITH n.id as id, collect(n.string) as list RETURN id, list;";
+
+  try {
+    client->Execute(accumulate_query);
+    [[maybe_unused]] auto results = client->FetchAll();
+    if (results->empty()) {
+      assert(true);
+      return 0;
+    }
+  } catch (const mg::TransientException & /*unused*/) {
+    spdlog::info("Memgraph is out of memory");
+    assert(true);
+    return 0;
+  }
+
+  MG_ASSERT(false, "Query should have failed");
+
+  return 0;
+}

tests/e2e/memory/memory_limit_delete.cpp

@@ -0,0 +1,80 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <gflags/gflags.h>
+#include <iostream>
+#include <mgclient.hpp>
+
+#include "utils/logging.hpp"
+#include "utils/timer.hpp"
+
+DEFINE_uint64(bolt_port, 7687, "Bolt port");
+DEFINE_bool(multi_db, false, "Run test in multi db environment");
+
+int main(int argc, char **argv) {
+  google::SetUsageMessage("Memgraph E2E Memory Control");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  memgraph::logging::RedirectToStderr();
+
+  mg::Client::Init();
+
+  auto client =
+      mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
+  if (!client) {
+    LOG_FATAL("Failed to connect!");
+  }
+
+  client->Execute("MATCH (n) DETACH DELETE n;");
+  client->DiscardAll();
+
+  if (FLAGS_multi_db) {
+    client->Execute("CREATE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("USE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("MATCH (n) DETACH DELETE n;");
+    client->DiscardAll();
+  }
+
+  const auto *create_query = "UNWIND range(1, 500000) as u CREATE (n {string: 'Some longer string'}) RETURN n;";
+
+  try {
+    client->Execute(create_query);
+    [[maybe_unused]] auto results = client->FetchAll();
+    if (results->empty()) {
+      assert(false);
+      return 0;
+    }
+  } catch (const mg::TransientException & /*unused*/) {
+    spdlog::info("Memgraph is out of memory");
+    assert(false);
+    return 0;
+  }
+
+  const auto *delete_query = "MATCH (n) DETACH DELETE n RETURN count(n);";
+
+  try {
+    client->Execute(delete_query);
+    [[maybe_unused]] auto results = client->FetchAll();
+    if (results->empty()) {
+      assert(true);
+      return 0;
+    }
+  } catch (const mg::TransientException & /*unused*/) {
+    spdlog::info("Memgraph is out of memory");
+    assert(true);
+    return 0;
+  }
+
+  MG_ASSERT(false, "Query should have failed");
+
+  return 0;
+}

tests/e2e/memory/memory_limit_edge_create.cpp

@@ -0,0 +1,66 @@
+// Copyright 2023 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <gflags/gflags.h>
+#include <iostream>
+#include <mgclient.hpp>
+
+#include "utils/logging.hpp"
+#include "utils/timer.hpp"
+
+DEFINE_uint64(bolt_port, 7687, "Bolt port");
+DEFINE_bool(multi_db, false, "Run test in multi db environment");
+
+int main(int argc, char **argv) {
+  google::SetUsageMessage("Memgraph E2E Memory Control");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  memgraph::logging::RedirectToStderr();
+
+  mg::Client::Init();
+
+  auto client =
+      mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
+  if (!client) {
+    LOG_FATAL("Failed to connect!");
+  }
+
+  client->Execute("MATCH (n) DETACH DELETE n;");
+  client->DiscardAll();
+
+  if (FLAGS_multi_db) {
+    client->Execute("CREATE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("USE DATABASE clean;");
+    client->DiscardAll();
+    client->Execute("MATCH (n) DETACH DELETE n;");
+    client->DiscardAll();
+  }
+  client->Execute("CREATE INDEX ON :Node(id);");
+  client->DiscardAll();
+
+  const auto *create_edge_query = "UNWIND range(1, 10000) as u MERGE (n:Node {id:u})-[:CONNECTED]->(m: Node {id:u+1});";
+
+  try {
+    client->Execute(create_edge_query);
+    [[maybe_unused]] auto results = client->FetchAll();
+    if (results->empty()) {
+      assert(true);
+      return 0;
+    }
+  } catch (const mg::TransientException & /*unused*/) {
+    spdlog::info("Memgraph is out of memory");
+    assert(true);
+    return 0;
+  }
+  MG_ASSERT(false, "Query should have failed");
+
+  return 0;
+}

tests/e2e/memory/workloads.yaml

@@ -23,6 +23,30 @@ disk_cluster: &disk_cluster
         - "STORAGE MODE ON_DISK_TRANSACTIONAL"
       validation_queries: []
 
+args_450_MiB_limit: &args_450_MiB_limit
+  - "--bolt-port"
+  - *bolt_port
+  - "--memory-limit=450"
+  - "--storage-gc-cycle-sec=180"
+  - "--log-level=INFO"
+
+in_memory_450_MiB_limit_cluster: &in_memory_450_MiB_limit_cluster
+  cluster:
+    main:
+      args: *args_450_MiB_limit
+      log_file: "memory-e2e.log"
+      setup_queries: []
+      validation_queries: []
+
+
+disk_450_MiB_limit_cluster: &disk_450_MiB_limit_cluster
+  cluster:
+    main:
+      args: *args_450_MiB_limit
+      log_file: "memory-e2e.log"
+      setup_queries: []
+      validation_queries: []
+
 
 workloads:
   - name: "Memory control"
@@ -70,3 +94,33 @@ workloads:
     args: ["--bolt-port", *bolt_port, "--timeout", "180"]
     proc: "tests/e2e/memory/procedures/"
     <<: *disk_cluster
+
+  - name: "Memory control for detach delete"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_delete"
+    args: ["--bolt-port", *bolt_port]
+    <<: *in_memory_450_MiB_limit_cluster
+
+  - name: "Memory control for detach delete on disk storage"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_delete"
+    args: ["--bolt-port", *bolt_port]
+    <<: *disk_450_MiB_limit_cluster
+
+  - name: "Memory control for accumulation"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_accumulation"
+    args: ["--bolt-port", *bolt_port]
+    <<: *in_memory_450_MiB_limit_cluster
+
+  - name: "Memory control for accumulation on disk storage"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_accumulation"
+    args: ["--bolt-port", *bolt_port]
+    <<: *disk_450_MiB_limit_cluster
+
+  - name: "Memory control for edge create"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_edge_create"
+    args: ["--bolt-port", *bolt_port]
+    <<: *in_memory_450_MiB_limit_cluster
+
+  - name: "Memory control for edge create on disk storage"
+    binary: "tests/e2e/memory/memgraph__e2e__memory__limit_edge_create"
+    args: ["--bolt-port", *bolt_port]
+    <<: *disk_450_MiB_limit_cluster