Compare commits
1 Commits
v2.12.1
...
print-stor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e3ad2af0fb |
@@ -64,7 +64,7 @@ option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
|
||||
# Set the current version here to override the automatic version detection. The
|
||||
# version must be specified as `X.Y.Z`. Primarily used when building new patch
|
||||
# versions.
|
||||
set(MEMGRAPH_OVERRIDE_VERSION "2.12.1")
|
||||
set(MEMGRAPH_OVERRIDE_VERSION "")
|
||||
|
||||
# Custom suffix that this version should have. The suffix can be any arbitrary
|
||||
# string. Primarily used when building a version for a specific customer.
|
||||
@@ -292,7 +292,7 @@ if (MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
option(ENABLE_JEMALLOC "Use jemalloc" ON)
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
|
||||
if (ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
|
||||
@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
|
||||
3. using the Licensed Work to create a work or solution
|
||||
which competes (or might reasonably be expected to
|
||||
compete) with the Licensed Work.
|
||||
CHANGE DATE: 2027-17-11
|
||||
CHANGE DATE: 2027-30-10
|
||||
CHANGE LICENSE: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.
|
||||
|
||||
@@ -220,8 +220,7 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle
|
||||
|
||||
spdlog::trace("Recovering indices and constraints from snapshot.");
|
||||
storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_,
|
||||
&storage->constraints_, &storage->vertices_,
|
||||
storage->name_id_mapper_.get());
|
||||
&storage->constraints_, &storage->vertices_);
|
||||
} catch (const storage::durability::RecoveryFailure &e) {
|
||||
LOG_FATAL("Couldn't load the snapshot because of: {}", e.what());
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed i
|
||||
// Storage flags.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).",
|
||||
FLAG_IN_RANGE(1, 24UL * 3600));
|
||||
FLAG_IN_RANGE(1, 24 * 3600));
|
||||
// NOTE: The `storage_properties_on_edges` flag must be the same here and in
|
||||
// `mg_import_csv`. If you change it, make sure to change it there as well.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
|
||||
@@ -3,14 +3,14 @@ set(memory_src_files
|
||||
global_memory_control.cpp
|
||||
query_memory_control.cpp)
|
||||
|
||||
|
||||
|
||||
find_package(jemalloc REQUIRED)
|
||||
|
||||
add_library(mg-memory STATIC ${memory_src_files})
|
||||
target_link_libraries(mg-memory mg-utils fmt)
|
||||
|
||||
message(STATUS "ENABLE_JEMALLOC: ${ENABLE_JEMALLOC}")
|
||||
if (ENABLE_JEMALLOC)
|
||||
find_package(jemalloc REQUIRED)
|
||||
target_link_libraries(mg-memory Jemalloc::Jemalloc ${CMAKE_DL_LIBS})
|
||||
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
|
||||
else()
|
||||
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=0)
|
||||
endif()
|
||||
|
||||
@@ -1216,6 +1216,10 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
|
||||
call_proc->memory_limit_ = memory_limit_info->first;
|
||||
call_proc->memory_scale_ = memory_limit_info->second;
|
||||
}
|
||||
} else {
|
||||
// Default to 100 MB
|
||||
call_proc->memory_limit_ = storage_->Create<PrimitiveLiteral>(TypedValue(100));
|
||||
call_proc->memory_scale_ = 1024U * 1024U;
|
||||
}
|
||||
|
||||
const auto &maybe_found =
|
||||
@@ -1236,13 +1240,11 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
|
||||
throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
|
||||
}
|
||||
}
|
||||
if (maybe_found) {
|
||||
call_proc->is_write_ = maybe_found->second->info.is_write;
|
||||
}
|
||||
call_proc->is_write_ = maybe_found->second->info.is_write;
|
||||
|
||||
auto *yield_ctx = ctx->yieldProcedureResults();
|
||||
if (!yield_ctx) {
|
||||
if ((maybe_found && !maybe_found->second->results.empty()) && !call_proc->void_procedure_) {
|
||||
if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) {
|
||||
throw SemanticException(
|
||||
"CALL without YIELD may only be used on procedures which do not "
|
||||
"return any result fields.");
|
||||
|
||||
@@ -3086,6 +3086,10 @@ PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_
|
||||
const auto vm_max_map_count = utils::GetVmMaxMapCount();
|
||||
const int64_t vm_max_map_count_storage_info =
|
||||
vm_max_map_count.has_value() ? vm_max_map_count.value() : memgraph::utils::VM_MAX_MAP_COUNT_DEFAULT;
|
||||
auto memory_res = utils::GetReadableSize(static_cast<double>(info.memory_res));
|
||||
auto memory_tracked = utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.Amount()));
|
||||
|
||||
spdlog::trace("Memory res: {}, memory tracked: {}", memory_res, memory_tracked);
|
||||
std::vector<std::vector<TypedValue>> results{
|
||||
{TypedValue("name"), TypedValue(storage->id())},
|
||||
{TypedValue("vertex_count"), TypedValue(static_cast<int64_t>(info.vertex_count))},
|
||||
|
||||
@@ -57,7 +57,6 @@
|
||||
#include "utils/likely.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/pmr/deque.hpp"
|
||||
@@ -4860,7 +4859,6 @@ class CallProcedureCursor : public Cursor {
|
||||
result_signature_size_ = result_->signature->size();
|
||||
result_->signature = nullptr;
|
||||
if (result_->error_msg) {
|
||||
memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker;
|
||||
throw QueryRuntimeException("{}: {}", self_->procedure_name_, *result_->error_msg);
|
||||
}
|
||||
result_row_it_ = result_->rows.begin();
|
||||
|
||||
@@ -176,10 +176,7 @@ class RuleBasedPlanner {
|
||||
PlanResult Plan(const QueryParts &query_parts) {
|
||||
auto &context = *context_;
|
||||
std::unique_ptr<LogicalOperator> final_plan;
|
||||
// procedures need to start from 1
|
||||
// due to swapping mechanism of procedure
|
||||
// tracking
|
||||
uint64_t procedure_id = 1;
|
||||
|
||||
for (const auto &query_part : query_parts.query_parts) {
|
||||
std::unique_ptr<LogicalOperator> input_op;
|
||||
|
||||
@@ -189,6 +186,10 @@ class RuleBasedPlanner {
|
||||
|
||||
uint64_t merge_id = 0;
|
||||
uint64_t subquery_id = 0;
|
||||
// procedures need to start from 1
|
||||
// due to swapping mechanism of procedure
|
||||
// tracking
|
||||
uint64_t procedure_id = 1;
|
||||
|
||||
for (const auto &clause : single_query_part.remaining_clauses) {
|
||||
MG_ASSERT(!utils::IsSubtype(*clause, Match::kType), "Unexpected Match in remaining clauses");
|
||||
|
||||
@@ -38,7 +38,6 @@
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/math.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/temporal.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
@@ -159,7 +158,6 @@ template <typename TFunc, typename... Args>
|
||||
[[nodiscard]] mgp_error WrapExceptions(TFunc &&func, Args &&...args) noexcept {
|
||||
static_assert(sizeof...(args) <= 1, "WrapExceptions should have only one or zero parameter!");
|
||||
try {
|
||||
memgraph::utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_enabler;
|
||||
WrapExceptionsHelper(std::forward<TFunc>(func), std::forward<Args>(args)...);
|
||||
} catch (const DeletedObjectException &neoe) {
|
||||
spdlog::error("Deleted object error during mg API call: {}", neoe.what());
|
||||
@@ -1546,7 +1544,6 @@ mgp_error mgp_duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory
|
||||
|
||||
mgp_error mgp_result_set_error_msg(mgp_result *res, const char *msg) {
|
||||
return WrapExceptions([=] {
|
||||
memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker blocker{};
|
||||
auto *memory = res->rows.get_allocator().GetMemoryResource();
|
||||
res->error_msg.emplace(msg, memory);
|
||||
});
|
||||
|
||||
@@ -41,4 +41,4 @@ add_library(mg-storage-v2 STATIC
|
||||
replication/replication_storage_state.cpp
|
||||
inmemory/replication/replication_client.cpp
|
||||
)
|
||||
target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory)
|
||||
target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events)
|
||||
|
||||
@@ -130,17 +130,16 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
|
||||
// recovery process.
|
||||
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
|
||||
Constraints *constraints, utils::SkipList<Vertex> *vertices,
|
||||
NameIdMapper *name_id_mapper,
|
||||
const std::optional<ParallelizedIndexCreationInfo> ¶llel_exec_info) {
|
||||
spdlog::info("Recreating indices from metadata.");
|
||||
// Recover label indices.
|
||||
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
|
||||
auto *mem_label_index = static_cast<InMemoryLabelIndex *>(indices->label_index_.get());
|
||||
for (const auto &item : indices_constraints.indices.label) {
|
||||
if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info)) {
|
||||
if (!mem_label_index->CreateIndex(item, vertices->access(), parallel_exec_info))
|
||||
throw RecoveryFailure("The label index must be created here!");
|
||||
}
|
||||
spdlog::info("Index on :{} is recreated from metadata", name_id_mapper->IdToName(item.AsUint()));
|
||||
|
||||
spdlog::info("A label index is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label indices are recreated.");
|
||||
|
||||
@@ -149,8 +148,7 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
|
||||
spdlog::info("Recreating {} label index statistics from metadata.", indices_constraints.indices.label_stats.size());
|
||||
for (const auto &item : indices_constraints.indices.label_stats) {
|
||||
mem_label_index->SetIndexStats(item.first, item.second);
|
||||
spdlog::info("Statistics for index on :{} are recreated from metadata",
|
||||
name_id_mapper->IdToName(item.first.AsUint()));
|
||||
spdlog::info("A label index statistics is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label indices statistics are recreated.");
|
||||
|
||||
@@ -161,8 +159,7 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
|
||||
for (const auto &item : indices_constraints.indices.label_property) {
|
||||
if (!mem_label_property_index->CreateIndex(item.first, item.second, vertices->access(), parallel_exec_info))
|
||||
throw RecoveryFailure("The label+property index must be created here!");
|
||||
spdlog::info("Index on :{}({}) is recreated from metadata", name_id_mapper->IdToName(item.first.AsUint()),
|
||||
name_id_mapper->IdToName(item.second.AsUint()));
|
||||
spdlog::info("A label+property index is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label+property indices are recreated.");
|
||||
|
||||
@@ -174,8 +171,7 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
|
||||
const auto property_id = item.second.first;
|
||||
const auto &stats = item.second.second;
|
||||
mem_label_property_index->SetIndexStats({label_id, property_id}, stats);
|
||||
spdlog::info("Statistics for index on :{}({}) are recreated from metadata",
|
||||
name_id_mapper->IdToName(label_id.AsUint()), name_id_mapper->IdToName(property_id.AsUint()));
|
||||
spdlog::info("A label+property index statistics is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label+property indices statistics are recreated.");
|
||||
|
||||
@@ -195,8 +191,8 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
|
||||
}
|
||||
|
||||
constraints->existence_constraints_->InsertConstraint(label, property);
|
||||
spdlog::info("Existence constraint on :{}({}) is recreated from metadata", name_id_mapper->IdToName(label.AsUint()),
|
||||
name_id_mapper->IdToName(property.AsUint()));
|
||||
|
||||
spdlog::info("A existence constraint is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Existence constraints are recreated from metadata.");
|
||||
|
||||
@@ -207,15 +203,7 @@ void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_
|
||||
auto ret = mem_unique_constraints->CreateConstraint(item.first, item.second, vertices->access());
|
||||
if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
|
||||
throw RecoveryFailure("The unique constraint must be created here!");
|
||||
|
||||
std::vector<std::string> property_names;
|
||||
property_names.reserve(item.second.size());
|
||||
for (const auto &prop : item.second) {
|
||||
property_names.emplace_back(name_id_mapper->IdToName(prop.AsUint()));
|
||||
}
|
||||
const auto property_names_joined = utils::Join(property_names, ",");
|
||||
spdlog::info("Unique constraint on :{}({}) is recreated from metadata",
|
||||
name_id_mapper->IdToName(item.first.AsUint()), property_names_joined);
|
||||
spdlog::info("A unique constraint is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Unique constraints are recreated from metadata.");
|
||||
spdlog::info("Constraints are recreated from metadata.");
|
||||
@@ -282,7 +270,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
? std::make_optional(std::make_pair(recovery_info.vertex_batches,
|
||||
config.durability.recovery_thread_count))
|
||||
: std::nullopt;
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info);
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info);
|
||||
return recovered_snapshot->recovery_info;
|
||||
}
|
||||
} else {
|
||||
@@ -414,7 +402,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
? std::make_optional(std::make_pair(recovery_info.vertex_batches, config.durability.recovery_thread_count))
|
||||
: std::nullopt;
|
||||
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, name_id_mapper, par_exec_info);
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info);
|
||||
|
||||
memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us,
|
||||
std::chrono::duration_cast<std::chrono::microseconds>(timer.Elapsed()).count());
|
||||
|
||||
@@ -104,7 +104,7 @@ using ParallelizedIndexCreationInfo =
|
||||
/// @throw RecoveryFailure
|
||||
void RecoverIndicesAndConstraints(
|
||||
const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints,
|
||||
utils::SkipList<Vertex> *vertices, NameIdMapper *name_id_mapper,
|
||||
utils::SkipList<Vertex> *vertices,
|
||||
const std::optional<ParallelizedIndexCreationInfo> ¶llel_exec_info = std::nullopt);
|
||||
|
||||
/// Recovers data either from a snapshot and/or WAL files.
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "memory/global_memory_control.hpp"
|
||||
#include "storage/v2/durability/durability.hpp"
|
||||
#include "storage/v2/durability/snapshot.hpp"
|
||||
#include "storage/v2/metadata_delta.hpp"
|
||||
@@ -102,13 +101,8 @@ InMemoryStorage::InMemoryStorage(Config config, StorageMode storage_mode)
|
||||
"those files into a .backup directory inside the storage directory.");
|
||||
}
|
||||
}
|
||||
|
||||
if (config_.gc.type == Config::Gc::Type::PERIODIC) {
|
||||
// TODO: move out of storage have one global gc_runner_
|
||||
gc_runner_.Run("Storage GC", config_.gc.interval, [this] {
|
||||
this->FreeMemory(std::unique_lock<utils::ResourceLock>{main_lock_, std::defer_lock});
|
||||
});
|
||||
gc_jemalloc_runner_.Run("Jemalloc GC", config_.gc.interval, [] { memory::PurgeUnusedMemory(); });
|
||||
gc_runner_.Run("Storage GC", config_.gc.interval, [this] { this->CollectGarbage<false>(); });
|
||||
}
|
||||
if (timestamp_ == kTimestampInitialId) {
|
||||
commit_log_.emplace();
|
||||
@@ -122,7 +116,6 @@ InMemoryStorage::InMemoryStorage(Config config) : InMemoryStorage(config, Storag
|
||||
InMemoryStorage::~InMemoryStorage() {
|
||||
if (config_.gc.type == Config::Gc::Type::PERIODIC) {
|
||||
gc_runner_.Stop();
|
||||
gc_jemalloc_runner_.Stop();
|
||||
}
|
||||
{
|
||||
// Stop replication (Stop all clients or stop the REPLICA server)
|
||||
@@ -1217,7 +1210,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock<utils::ResourceLock> main_
|
||||
main_lock_.lock_shared();
|
||||
}
|
||||
} else {
|
||||
DMG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_");
|
||||
MG_ASSERT(main_guard.mutex() == std::addressof(main_lock_), "main_guard should be only for the main_lock_");
|
||||
}
|
||||
|
||||
utils::OnScopeExit lock_releaser{[&] {
|
||||
|
||||
@@ -418,7 +418,6 @@ class InMemoryStorage final : public Storage {
|
||||
std::optional<CommitLog> commit_log_;
|
||||
|
||||
utils::Scheduler gc_runner_;
|
||||
utils::Scheduler gc_jemalloc_runner_;
|
||||
std::mutex gc_lock_;
|
||||
|
||||
using BondPmrLd = Bond<utils::pmr::list<Delta>>;
|
||||
|
||||
@@ -71,7 +71,6 @@ add_subdirectory(query_modules)
|
||||
add_subdirectory(constraints)
|
||||
add_subdirectory(inspect_query)
|
||||
add_subdirectory(queries)
|
||||
add_subdirectory(garbage_collection)
|
||||
|
||||
copy_e2e_python_files(pytest_runner pytest_runner.sh "")
|
||||
copy_e2e_python_files(x x.sh "")
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
function(garbage_collection_e2e_python_files FILE_NAME)
|
||||
copy_e2e_python_files(garbage_collection ${FILE_NAME})
|
||||
endfunction()
|
||||
|
||||
garbage_collection_e2e_python_files(common.py)
|
||||
garbage_collection_e2e_python_files(conftest.py)
|
||||
garbage_collection_e2e_python_files(gc_periodic.py)
|
||||
@@ -1,25 +0,0 @@
|
||||
# Copyright 2023 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import typing
|
||||
|
||||
import mgclient
|
||||
|
||||
|
||||
def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]:
|
||||
cursor.execute(query, params)
|
||||
return cursor.fetchall()
|
||||
|
||||
|
||||
def connect(**kwargs) -> mgclient.Connection:
|
||||
connection = mgclient.connect(host="localhost", port=7687, **kwargs)
|
||||
connection.autocommit = True
|
||||
return connection
|
||||
@@ -1,21 +0,0 @@
|
||||
# Copyright 2023 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import pytest
|
||||
from common import connect, execute_and_fetch_all
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def connection():
|
||||
connection = connect()
|
||||
yield connection
|
||||
cursor = connection.cursor()
|
||||
execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n")
|
||||
@@ -1,59 +0,0 @@
|
||||
# Copyright 2023 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from common import execute_and_fetch_all
|
||||
|
||||
|
||||
def remove_non_numeric_suffix(text):
|
||||
match = re.search(r"\D*$", text)
|
||||
if match:
|
||||
non_numeric_suffix = match.group(0)
|
||||
return text[: -len(non_numeric_suffix)]
|
||||
else:
|
||||
return text
|
||||
|
||||
|
||||
def get_memory_from_list(list):
|
||||
for list_item in list:
|
||||
if list_item[0] == "memory_tracked":
|
||||
return float(remove_non_numeric_suffix(list_item[1]))
|
||||
return None
|
||||
|
||||
|
||||
def get_memory(cursor):
|
||||
return get_memory_from_list(execute_and_fetch_all(cursor, "SHOW STORAGE INFO"))
|
||||
|
||||
|
||||
def test_gc_periodic(connection):
|
||||
"""
|
||||
This test checks that periodic gc works.
|
||||
It does so by checking that the allocated memory is lowered by at least 1/4 of the memory allocated by creating nodes.
|
||||
If we choose a number a high number the test will become flaky because the memory only gets fully cleared after a while
|
||||
due to jemalloc holding some memory for a while. If we'd wait for jemalloc to fully release the memory the test would take too long.
|
||||
"""
|
||||
cursor = connection.cursor()
|
||||
|
||||
memory_pre_creation = get_memory(cursor)
|
||||
execute_and_fetch_all(cursor, "UNWIND range(1, 1000) AS index CREATE (:Node);")
|
||||
memory_after_creation = get_memory(cursor)
|
||||
time.sleep(2)
|
||||
memory_after_gc = get_memory(cursor)
|
||||
|
||||
assert memory_after_gc < memory_pre_creation + (memory_after_creation - memory_pre_creation) / 4 * 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-rA"]))
|
||||
@@ -1,19 +0,0 @@
|
||||
args: &args
|
||||
- "--bolt-port"
|
||||
- "7687"
|
||||
- "--log-level=TRACE"
|
||||
- "--storage-gc-cycle-sec=2"
|
||||
|
||||
in_memory_cluster: &in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "garbage_collection-e2e.log"
|
||||
setup_queries: []
|
||||
validation_queries: []
|
||||
|
||||
workloads:
|
||||
- name: "Garbage collection"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
args: ["garbage_collection/gc_periodic.py"]
|
||||
<<: *in_memory_cluster
|
||||
@@ -2,8 +2,6 @@ add_subdirectory(procedures)
|
||||
|
||||
find_package(gflags REQUIRED)
|
||||
|
||||
# Global memory limit
|
||||
|
||||
add_executable(memgraph__e2e__memory__control memory_control.cpp)
|
||||
target_link_libraries(memgraph__e2e__memory__control gflags mgclient mg-utils mg-io Threads::Threads)
|
||||
|
||||
@@ -22,12 +20,6 @@ target_link_libraries(memgraph__e2e__memory__limit_accumulation gflags mgclient
|
||||
add_executable(memgraph__e2e__memory__limit_edge_create memory_limit_edge_create.cpp)
|
||||
target_link_libraries(memgraph__e2e__memory__limit_edge_create gflags mgclient mg-utils mg-io)
|
||||
|
||||
add_executable(memgraph__e2e__memory_limit_global_multi_thread_proc_create memory_limit_global_multi_thread_proc_create.cpp)
|
||||
target_link_libraries(memgraph__e2e__memory_limit_global_multi_thread_proc_create gflags mgclient mg-utils mg-io)
|
||||
|
||||
add_executable(memgraph__e2e__memory_limit_global_thread_alloc_proc memory_limit_global_thread_alloc_proc.cpp)
|
||||
target_link_libraries(memgraph__e2e__memory_limit_global_thread_alloc_proc gflags mgclient mg-utils mg-io)
|
||||
|
||||
# Query memory limit tests
|
||||
|
||||
add_executable(memgraph__e2e__memory__limit_query_alloc_proc_multi_thread query_memory_limit_proc_multi_thread.cpp)
|
||||
@@ -42,6 +34,7 @@ target_link_libraries(memgraph__e2e__memory__limit_query_alloc_proc gflags mgcli
|
||||
add_executable(memgraph__e2e__memory__limit_query_alloc_create_multi_thread query_memory_limit_multi_thread.cpp)
|
||||
target_link_libraries(memgraph__e2e__memory__limit_query_alloc_create_multi_thread gflags mgclient mg-utils mg-io Threads::Threads)
|
||||
|
||||
|
||||
# Procedure memory limit tests
|
||||
|
||||
add_executable(memgraph__e2e__procedure_memory_limit procedure_memory_limit.cpp)
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include <algorithm>
|
||||
#include <exception>
|
||||
#include <ios>
|
||||
#include <iostream>
|
||||
#include <mgclient.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/timer.hpp"
|
||||
|
||||
DEFINE_uint64(bolt_port, 7687, "Bolt port");
|
||||
DEFINE_uint64(timeout, 120, "Timeout seconds");
|
||||
DEFINE_bool(multi_db, false, "Run test in multi db environment");
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread Create For Local Allocators");
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
memgraph::logging::RedirectToStderr();
|
||||
|
||||
mg::Client::Init();
|
||||
|
||||
auto client =
|
||||
mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
|
||||
if (!client) {
|
||||
LOG_FATAL("Failed to connect!");
|
||||
}
|
||||
|
||||
if (FLAGS_multi_db) {
|
||||
client->Execute("CREATE DATABASE clean;");
|
||||
client->DiscardAll();
|
||||
client->Execute("USE DATABASE clean;");
|
||||
client->DiscardAll();
|
||||
client->Execute("MATCH (n) DETACH DELETE n;");
|
||||
client->DiscardAll();
|
||||
}
|
||||
|
||||
bool error{false};
|
||||
try {
|
||||
client->Execute(
|
||||
"CALL libglobal_memory_limit_multi_thread_create_proc.multi_create() PROCEDURE MEMORY UNLIMITED YIELD "
|
||||
"allocated_all RETURN allocated_all "
|
||||
"QUERY MEMORY LIMIT 50MB;");
|
||||
auto result_rows = client->FetchAll();
|
||||
if (result_rows) {
|
||||
auto row = *result_rows->begin();
|
||||
error = row[0].ValueBool() == false;
|
||||
}
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
error = true;
|
||||
}
|
||||
|
||||
MG_ASSERT(error, "Error should have happend");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include <algorithm>
|
||||
#include <exception>
|
||||
#include <ios>
|
||||
#include <iostream>
|
||||
#include <mgclient.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/timer.hpp"
|
||||
|
||||
DEFINE_uint64(bolt_port, 7687, "Bolt port");
|
||||
DEFINE_uint64(timeout, 120, "Timeout seconds");
|
||||
DEFINE_bool(multi_db, false, "Run test in multi db environment");
|
||||
|
||||
// Test checks path of throwing error from different thread
|
||||
// than main thread which started test
|
||||
int main(int argc, char **argv) {
|
||||
google::SetUsageMessage("Memgraph E2E Global Memory Limit In Multi-Thread For Procedures For Local Allocators");
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
memgraph::logging::RedirectToStderr();
|
||||
|
||||
mg::Client::Init();
|
||||
|
||||
auto client =
|
||||
mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
|
||||
if (!client) {
|
||||
LOG_FATAL("Failed to connect!");
|
||||
}
|
||||
|
||||
if (FLAGS_multi_db) {
|
||||
client->Execute("CREATE DATABASE clean;");
|
||||
client->DiscardAll();
|
||||
client->Execute("USE DATABASE clean;");
|
||||
client->DiscardAll();
|
||||
client->Execute("MATCH (n) DETACH DELETE n;");
|
||||
client->DiscardAll();
|
||||
}
|
||||
|
||||
bool error{false};
|
||||
try {
|
||||
client->Execute(
|
||||
"CALL libglobal_memory_limit_thread_proc.thread() YIELD allocated_all RETURN allocated_all QUERY MEMORY LIMIT "
|
||||
"100MB;");
|
||||
auto result_rows = client->FetchAll();
|
||||
if (result_rows) {
|
||||
auto row = *result_rows->begin();
|
||||
error = row[0].ValueBool() == false;
|
||||
}
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
error = true;
|
||||
}
|
||||
|
||||
MG_ASSERT(error, "Error should have happend");
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -14,7 +14,6 @@
|
||||
#include <exception>
|
||||
#include <ios>
|
||||
#include <iostream>
|
||||
#include <mgclient-value.hpp>
|
||||
#include <mgclient.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
@@ -54,9 +53,11 @@ int main(int argc, char **argv) {
|
||||
"CALL libproc_memory_limit.alloc_32_mib() PROCEDURE MEMORY LIMIT 10MB YIELD allocated AS allocated_2 RETURN "
|
||||
"allocated_1, allocated_2");
|
||||
auto result_rows = client->FetchAll();
|
||||
if (result_rows && result_rows->empty()) {
|
||||
test_passed = true;
|
||||
if (result_rows) {
|
||||
auto row = *result_rows->begin();
|
||||
test_passed = row[0].ValueBool() == true && row[0].ValueBool() == false;
|
||||
}
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
test_passed = true;
|
||||
}
|
||||
|
||||
@@ -4,21 +4,16 @@ target_include_directories(global_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/inclu
|
||||
add_library(global_memory_limit_proc SHARED global_memory_limit_proc.c)
|
||||
target_include_directories(global_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
|
||||
|
||||
add_library(query_memory_limit_proc_multi_thread SHARED query_memory_limit_proc_multi_thread.cpp)
|
||||
target_include_directories(query_memory_limit_proc_multi_thread PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(query_memory_limit_proc_multi_thread mg-utils)
|
||||
|
||||
|
||||
add_library(query_memory_limit_proc SHARED query_memory_limit_proc.cpp)
|
||||
target_include_directories(query_memory_limit_proc PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(query_memory_limit_proc mg-utils)
|
||||
|
||||
add_library(global_memory_limit_thread_proc SHARED global_memory_limit_thread_proc.cpp)
|
||||
target_include_directories(global_memory_limit_thread_proc PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(global_memory_limit_thread_proc mg-utils)
|
||||
|
||||
add_library(global_memory_limit_multi_thread_create_proc SHARED global_memory_limit_multi_thread_create_proc.cpp)
|
||||
target_include_directories(global_memory_limit_multi_thread_create_proc PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(global_memory_limit_multi_thread_create_proc mg-utils)
|
||||
|
||||
add_library(proc_memory_limit SHARED proc_memory_limit.cpp)
|
||||
target_include_directories(proc_memory_limit PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <exception>
|
||||
#include <functional>
|
||||
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mg_procedure.h"
|
||||
#include "mgp.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
|
||||
// change communication between threads with feature and promise
|
||||
std::atomic<int> created_vertices{0};
|
||||
constexpr int num_vertices_per_thread{100'000};
|
||||
constexpr int num_threads{2};
|
||||
|
||||
void CallCreate(mgp_graph *graph, mgp_memory *memory) {
|
||||
[[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph);
|
||||
for (int i = 0; i < num_vertices_per_thread; i++) {
|
||||
struct mgp_vertex *vertex{nullptr};
|
||||
auto enum_error = mgp_graph_create_vertex(graph, memory, &vertex);
|
||||
if (enum_error != mgp_error::MGP_ERROR_NO_ERROR) {
|
||||
break;
|
||||
}
|
||||
created_vertices.fetch_add(1, std::memory_order_acq_rel);
|
||||
}
|
||||
[[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph);
|
||||
}
|
||||
|
||||
void AllocFunc(mgp_graph *graph, mgp_memory *memory) {
|
||||
try {
|
||||
CallCreate(graph, memory);
|
||||
} catch (const std::exception &e) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void MultiCreate(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto arguments = mgp::List(args);
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
std::vector<std::thread> threads;
|
||||
|
||||
for (int i = 0; i < 2; i++) {
|
||||
threads.emplace_back(AllocFunc, memgraph_graph, memory);
|
||||
}
|
||||
|
||||
for (int i = 0; i < num_threads; i++) {
|
||||
threads[i].join();
|
||||
}
|
||||
if (created_vertices.load(std::memory_order_acquire) != num_vertices_per_thread * num_threads) {
|
||||
record_factory.SetErrorMessage("Unable to allocate");
|
||||
return;
|
||||
}
|
||||
|
||||
auto new_record = record_factory.NewRecord();
|
||||
new_record.Insert("allocated_all",
|
||||
created_vertices.load(std::memory_order_acquire) == num_vertices_per_thread * num_threads);
|
||||
} catch (std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
|
||||
try {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
|
||||
AddProcedure(MultiCreate, std::string("multi_create").c_str(), mgp::ProcedureType::Write, {},
|
||||
{mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory);
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern "C" int mgp_shutdown_module() { return 0; }
|
||||
@@ -1,92 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <exception>
|
||||
#include <functional>
|
||||
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mg_procedure.h"
|
||||
#include "mgp.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
|
||||
enum mgp_error Alloc(mgp_memory *memory, void *ptr) {
|
||||
const size_t mb_size_512 = 1 << 29;
|
||||
|
||||
return mgp_alloc(memory, mb_size_512, (void **)(&ptr));
|
||||
}
|
||||
|
||||
// change communication between threads with feature and promise
|
||||
std::atomic<int> num_allocations{0};
|
||||
void *ptr_;
|
||||
|
||||
void AllocFunc(mgp_memory *memory, mgp_graph *graph) {
|
||||
try {
|
||||
[[maybe_unused]] const enum mgp_error tracking_error = mgp_track_current_thread_allocations(graph);
|
||||
enum mgp_error alloc_err { mgp_error::MGP_ERROR_NO_ERROR };
|
||||
alloc_err = Alloc(memory, ptr_);
|
||||
if (alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE) {
|
||||
num_allocations.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) {
|
||||
assert(false);
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
[[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph);
|
||||
assert(false);
|
||||
}
|
||||
[[maybe_unused]] const enum mgp_error untracking_error = mgp_untrack_current_thread_allocations(graph);
|
||||
}
|
||||
|
||||
void Thread(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto arguments = mgp::List(args);
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
num_allocations.store(0, std::memory_order_relaxed);
|
||||
try {
|
||||
std::thread thread{AllocFunc, memory, memgraph_graph};
|
||||
|
||||
thread.join();
|
||||
|
||||
if (ptr_ != nullptr) {
|
||||
mgp_free(memory, ptr_);
|
||||
}
|
||||
|
||||
auto new_record = record_factory.NewRecord();
|
||||
|
||||
new_record.Insert("allocated_all", num_allocations.load(std::memory_order_relaxed) == 1);
|
||||
} catch (std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
|
||||
try {
|
||||
mgp::memory = memory;
|
||||
|
||||
AddProcedure(Thread, std::string("thread").c_str(), mgp::ProcedureType::Read, {},
|
||||
{mgp::Return(std::string("allocated_all").c_str(), mgp::Type::Bool)}, module, memory);
|
||||
|
||||
} catch (const std::exception &e) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern "C" int mgp_shutdown_module() { return 0; }
|
||||
@@ -75,9 +75,6 @@ void Alloc_32_MiB(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result,
|
||||
}};
|
||||
|
||||
const enum mgp_error alloc_err = Alloc_32(memory, ptr);
|
||||
if (alloc_err != mgp_error::MGP_ERROR_NO_ERROR) {
|
||||
record_factory.SetErrorMessage("Unable to allocate");
|
||||
}
|
||||
auto new_record = record_factory.NewRecord();
|
||||
new_record.Insert("allocated", alloc_err != mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE);
|
||||
} catch (std::exception &e) {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#include <cassert>
|
||||
#include <exception>
|
||||
#include <functional>
|
||||
|
||||
#include <mgp.hpp>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
@@ -22,7 +22,6 @@
|
||||
#include <vector>
|
||||
|
||||
#include "mg_procedure.h"
|
||||
#include "mgp.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
|
||||
enum mgp_error Alloc(void *ptr) {
|
||||
|
||||
@@ -144,7 +144,6 @@ workloads:
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create_multi_thread"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_query_limit_cluster
|
||||
|
||||
- name: "Memory control for detach delete"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_delete"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
@@ -175,18 +174,6 @@ workloads:
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *disk_450_MiB_limit_cluster
|
||||
|
||||
- name: "Memory control for create from multi thread proc create"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_multi_thread_proc_create"
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Memory control for memory limit global thread alloc"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory_limit_global_thread_alloc_proc"
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Procedure memory control for single procedure"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit"
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
@@ -194,7 +181,7 @@ workloads:
|
||||
<<: *in_memory_limited_global_limit_cluster
|
||||
|
||||
- name: "Procedure memory control for multiple procedures"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit_multi_proc"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__procedure_memory_limit"
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_limited_global_limit_cluster
|
||||
|
||||
@@ -2833,6 +2833,18 @@ TEST_P(CypherMainVisitorTest, DumpDatabase) {
|
||||
ASSERT_TRUE(query);
|
||||
}
|
||||
|
||||
namespace {
|
||||
template <class TAst>
|
||||
void CheckCallProcedureDefaultMemoryLimit(const TAst &ast, const CallProcedure &call_proc) {
|
||||
// Should be 100 MB
|
||||
auto *literal = dynamic_cast<PrimitiveLiteral *>(call_proc.memory_limit_);
|
||||
ASSERT_TRUE(literal);
|
||||
TypedValue value(literal->value_);
|
||||
ASSERT_TRUE(TypedValue::BoolEqual{}(value, TypedValue(100)));
|
||||
ASSERT_EQ(call_proc.memory_scale_, 1024 * 1024);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) {
|
||||
AddProc(*mock_module_with_dots_in_name, "proc", {}, {"res"}, ProcedureType::WRITE);
|
||||
auto &ast_generator = *GetParam();
|
||||
@@ -2856,6 +2868,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDotsInName) {
|
||||
std::vector<std::string> expected_names{"res"};
|
||||
ASSERT_EQ(identifier_names, expected_names);
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) {
|
||||
@@ -2881,6 +2894,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithDashesInName) {
|
||||
std::vector<std::string> expected_names{"res"};
|
||||
ASSERT_EQ(identifier_names, expected_names);
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) {
|
||||
@@ -2912,6 +2926,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldSomeFields) {
|
||||
std::vector<std::string> expected_names{"fst", "field-with-dashes", "last_field"};
|
||||
ASSERT_EQ(identifier_names, expected_names);
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
};
|
||||
check_proc(ProcedureType::READ);
|
||||
check_proc(ProcedureType::WRITE);
|
||||
@@ -2944,6 +2959,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithYieldAliasedFields) {
|
||||
ASSERT_EQ(identifier_names, aliased_names);
|
||||
std::vector<std::string> field_names{"fst", "snd", "thrd"};
|
||||
ASSERT_EQ(call_proc->result_fields_, field_names);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) {
|
||||
@@ -2970,6 +2986,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithArguments) {
|
||||
std::vector<std::string> expected_names{"res"};
|
||||
ASSERT_EQ(identifier_names, expected_names);
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) {
|
||||
@@ -2991,6 +3008,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsterisk) {
|
||||
}
|
||||
ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable"));
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) {
|
||||
@@ -3015,6 +3033,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureYieldAsteriskReturnAsterisk) {
|
||||
}
|
||||
ASSERT_THAT(identifier_names, UnorderedElementsAre("name", "signature", "is_write", "path", "is_editable"));
|
||||
ASSERT_EQ(identifier_names, call_proc->result_fields_);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) {
|
||||
@@ -3030,6 +3049,7 @@ TEST_P(CypherMainVisitorTest, CallProcedureWithoutYield) {
|
||||
ASSERT_TRUE(call_proc->arguments_.empty());
|
||||
ASSERT_TRUE(call_proc->result_fields_.empty());
|
||||
ASSERT_TRUE(call_proc->result_identifiers_.empty());
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
TEST_P(CypherMainVisitorTest, CallProcedureWithMemoryLimitWithoutYield) {
|
||||
@@ -3163,6 +3183,7 @@ void CheckParsedCallProcedure(const CypherQuery &query, Base &ast_generator,
|
||||
EXPECT_EQ(identifier_names, args_as_str);
|
||||
EXPECT_EQ(identifier_names, call_proc->result_fields_);
|
||||
ASSERT_EQ(call_proc->is_write_, type == ProcedureType::WRITE);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
};
|
||||
} // namespace
|
||||
|
||||
@@ -3556,6 +3577,7 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) {
|
||||
auto *single_query = query->single_query_;
|
||||
ASSERT_EQ(single_query->clauses_.size(), 2U);
|
||||
auto *call_proc = dynamic_cast<CallProcedure *>(single_query->clauses_[0]);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -3616,6 +3638,7 @@ TEST_P(CypherMainVisitorTest, MemoryLimit) {
|
||||
auto *single_query = query->single_query_;
|
||||
ASSERT_EQ(single_query->clauses_.size(), 1U);
|
||||
auto *call_proc = dynamic_cast<CallProcedure *>(single_query->clauses_[0]);
|
||||
CheckCallProcedureDefaultMemoryLimit(ast_generator, *call_proc);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user