Compare commits

..

1 Commits

Author SHA1 Message Date
Andi Skrgat
43a9a5b82b Fix max RAM usage 2023-12-07 10:50:20 +01:00
23 changed files with 63 additions and 357 deletions

View File

@@ -75,8 +75,6 @@ jobs:
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results cartesian.json cartesian
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results order_by.json order_by
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
@@ -106,9 +104,3 @@ jobs:
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "order_by" \
--benchmark-results "../../tests/mgbench/order_by.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -178,7 +178,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}

View File

@@ -259,7 +259,7 @@ repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "
# jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756
JEMALLOC_COMMIT_VERSION="5.2.1"
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
# this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
pushd jemalloc

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2027-08-12
CHANGE DATE: 2027-30-10
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -108,83 +108,31 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
}
struct Property {
std::string name;
mgp::Value value;
Property(const std::string &name, mgp::Value &&value) : name(name), value(std::move(value)) {}
};
struct LabelsHash {
std::size_t operator()(const std::set<std::string> &set) const {
std::size_t seed = set.size();
for (const auto &i : set) {
seed ^= std::hash<std::string>{}(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}
return seed;
}
};
struct LabelsComparator {
bool operator()(const std::set<std::string> &lhs, const std::set<std::string> &rhs) const { return lhs == rhs; }
};
struct PropertyComparator {
bool operator()(const Property &lhs, const Property &rhs) const { return lhs.name < rhs.name; }
};
struct PropertyInfo {
std::set<Property, PropertyComparator> properties;
bool mandatory;
};
void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
const auto record_factory = mgp::RecordFactory(result);
try {
std::unordered_map<std::set<std::string>, PropertyInfo, LabelsHash, LabelsComparator> node_types_properties;
for (auto node : mgp::Graph(memgraph_graph).Nodes()) {
std::set<std::string> labels_set = {};
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto node : graph.Nodes()) {
std::string type;
mgp::List labels = mgp::List();
for (auto label : node.Labels()) {
labels_set.emplace(label);
}
if (node_types_properties.find(labels_set) == node_types_properties.end()) {
node_types_properties[labels_set] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
labels.AppendExtend(mgp::Value(label));
type += ":`" + std::string(label) + "`";
}
if (node.Properties().empty()) {
node_types_properties[labels_set].mandatory = false; // if there is node with no property, it is not mandatory
auto record = record_factory.NewRecord();
ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
continue;
}
auto &property_info = node_types_properties.at(labels_set);
for (auto &[key, prop] : node.Properties()) {
property_info.properties.emplace(key, std::move(prop));
if (property_info.mandatory) {
property_info.mandatory =
property_info.properties.size() == 1; // if there is only one property, it is mandatory
}
}
}
for (auto &[labels, property_info] : node_types_properties) {
std::string label_type;
mgp::List labels_list = mgp::List();
for (auto const &label : labels) {
label_type += ":`" + std::string(label) + "`";
labels_list.AppendExtend(mgp::Value(label));
}
for (auto const &prop : property_info.properties) {
auto property_type = mgp::List();
auto record = record_factory.NewRecord();
ProcessPropertiesNode(record, label_type, labels_list, prop.name, TypeOf(prop.value.Type()),
property_info.mandatory);
}
if (property_info.properties.empty()) {
auto record = record_factory.NewRecord();
ProcessPropertiesNode<std::string>(record, label_type, labels_list, "", "", false);
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
ProcessPropertiesNode<mgp::List>(record, type, labels, key, property_type, true);
}
}
@@ -196,41 +144,23 @@ void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph,
void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
std::unordered_map<std::string, PropertyInfo> rel_types_properties;
const auto record_factory = mgp::RecordFactory(result);
try {
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto rel : graph.Relationships()) {
std::string rel_type = std::string(rel.Type());
if (rel_types_properties.find(rel_type) == rel_types_properties.end()) {
rel_types_properties[rel_type] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
}
for (auto rel : graph.Relationships()) {
std::string type = ":`" + std::string(rel.Type()) + "`";
if (rel.Properties().empty()) {
rel_types_properties[rel_type].mandatory = false; // if there is rel with no property, it is not mandatory
auto record = record_factory.NewRecord();
ProcessPropertiesRel<std::string>(record, type, "", "", false);
continue;
}
auto &property_info = rel_types_properties.at(rel_type);
for (auto &[key, prop] : rel.Properties()) {
property_info.properties.emplace(key, std::move(prop));
if (property_info.mandatory) {
property_info.mandatory =
property_info.properties.size() == 1; // if there is only one property, it is mandatory
}
}
}
for (auto &[type, property_info] : rel_types_properties) {
std::string type_str = ":`" + std::string(type) + "`";
for (auto const &prop : property_info.properties) {
auto property_type = mgp::List();
auto record = record_factory.NewRecord();
ProcessPropertiesRel(record, type_str, prop.name, TypeOf(prop.value.Type()), property_info.mandatory);
}
if (property_info.properties.empty()) {
auto record = record_factory.NewRecord();
ProcessPropertiesRel<std::string>(record, type_str, "", "", false);
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
ProcessPropertiesRel<mgp::List>(record, type, key, property_type, true);
}
}

View File

@@ -119,7 +119,7 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
return false;

View File

@@ -238,16 +238,6 @@ std::optional<std::string> GetOptionalStringValue(query::Expression *expression,
return {};
};
bool IsOrderByQuery(const std::vector<memgraph::query::Clause *> &clauses) {
for (const auto &clause : clauses) {
if (clause->GetTypeInfo() == Return::kType) {
auto *return_clause = utils::Downcast<Return>(clause);
return !return_clause->body_.order_by.empty();
}
}
return false;
}
bool IsAllShortestPathsQuery(const std::vector<memgraph::query::Clause *> &clauses) {
for (const auto &clause : clauses) {
if (clause->GetTypeInfo() != Match::kType) {
@@ -1601,8 +1591,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
}
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
auto use_monotonic_memory = !contains_csv && !IsCallBatchedProcedureQuery(clauses) &&
!IsAllShortestPathsQuery(clauses) && !IsOrderByQuery(clauses);
auto use_monotonic_memory =
!contains_csv && !IsCallBatchedProcedureQuery(clauses) && !IsAllShortestPathsQuery(clauses);
MG_ASSERT(current_db.execution_db_accessor_, "Cypher query expects a current DB transaction");
auto *dba =
@@ -1764,8 +1754,8 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// If this is LOAD CSV, BatchedProcedure or AllShortest query, use PoolResource without MonotonicMemoryResource as we
// want to reuse allocated memory
auto use_monotonic_memory = !contains_csv && !IsCallBatchedProcedureQuery(clauses) &&
!IsAllShortestPathsQuery(clauses) && !IsOrderByQuery(clauses);
auto use_monotonic_memory =
!contains_csv && !IsCallBatchedProcedureQuery(clauses) && !IsAllShortestPathsQuery(clauses);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
EvaluationContext evaluation_context;
@@ -3726,8 +3716,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
auto const &clauses = cypher_query->single_query_->clauses_;
bool hasAllShortestPaths = IsAllShortestPathsQuery(clauses);
// Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage.
bool usePool = hasAllShortestPaths || IsCallBatchedProcedureQuery(clauses) || IsLoadCsvQuery(clauses) ||
IsOrderByQuery(clauses);
bool usePool = hasAllShortestPaths || IsCallBatchedProcedureQuery(clauses) || IsLoadCsvQuery(clauses);
return {usePool, hasAllShortestPaths};
}(); // IILE

View File

@@ -110,13 +110,7 @@ void Telemetry::CollectData(const std::string &event) {
{
std::lock_guard<std::mutex> guard(lock_);
for (auto &collector : collectors_) {
try {
data[collector.first] = collector.second();
} catch (std::exception &e) {
spdlog::warn(fmt::format(
"Unknwon exception occured on in telemetry server {}, please contact support on https://memgr.ph/unknown ",
e.what()));
}
data[collector.first] = collector.second();
}
}
if (event == "") {

View File

@@ -25,32 +25,17 @@ namespace memgraph::utils {
static constexpr int64_t VM_MAX_MAP_COUNT_DEFAULT{-1};
/// Returns the number of bytes a directory is using on disk. If the given path
/// isn't a directory, zero will be returned. If there are some files with
/// wrong permission, it will be skipped
/// isn't a directory, zero will be returned.
template <bool IgnoreSymlink = true>
inline uint64_t GetDirDiskUsage(const std::filesystem::path &path) {
if (!std::filesystem::is_directory(path)) return 0;
if (!utils::HasReadAccess(path)) {
spdlog::warn(
"Skipping directory path on collecting directory disk usage '{}' because it is not readable, check file "
"ownership and read permissions!",
path);
return 0;
}
uint64_t size = 0;
for (const auto &p : std::filesystem::directory_iterator(path)) {
for (auto &p : std::filesystem::directory_iterator(path)) {
if (IgnoreSymlink && std::filesystem::is_symlink(p)) continue;
if (std::filesystem::is_directory(p)) {
size += GetDirDiskUsage(p);
} else if (std::filesystem::is_regular_file(p)) {
if (!utils::HasReadAccess(p)) {
spdlog::warn(
"Skipping file path on collecting directory disk usage '{}' because it is not readable, check file "
"ownership and read permissions!",
p);
continue;
}
size += std::filesystem::file_size(p);
}
}

View File

@@ -9,8 +9,8 @@ fi
if [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then
export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
fi
if [ -d "/opt/apache-maven-3.9.3" ]; then
export M2_HOME="/opt/apache-maven-3.9.3"
if [ -d "/opt/apache-maven-3.9.2" ]; then
export M2_HOME="/opt/apache-maven-3.9.2"
fi
export PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH"

View File

@@ -80,8 +80,6 @@ ACTIONS = {
"quit": lambda _: sys.exit(1),
}
CLEANUP_DIRECTORIES_ON_EXIT = False
log = logging.getLogger("memgraph.tests.e2e")
@@ -111,11 +109,10 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_
assert not is_port_in_use(
extract_bolt_port(args)
), "If this raises, you are trying to start an instance on a port already used by one already running instance."
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl)
MEMGRAPH_INSTANCES[name] = mg_instance
log_file_path = os.path.join(BUILD_DIR, "logs", log_file)
data_directory_path = os.path.join(BUILD_DIR, data_directory)
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY, use_ssl, {data_directory_path})
MEMGRAPH_INSTANCES[name] = mg_instance
binary_args = args + ["--log-file", log_file_path] + ["--data-directory", data_directory_path]
if len(procdir) != 0:
@@ -125,43 +122,39 @@ def _start_instance(name, args, log_file, setup_queries, use_ssl, procdir, data_
assert mg_instance.is_running(), "An error occured after starting Memgraph instance: application stopped running."
def stop_all(keep_directories=True):
def stop_all():
for mg_instance in MEMGRAPH_INSTANCES.values():
mg_instance.stop(keep_directories)
mg_instance.stop()
MEMGRAPH_INSTANCES.clear()
def stop_instance(context, name, keep_directories=True):
def stop_instance(context, name):
for key, _ in context.items():
if key != name:
continue
MEMGRAPH_INSTANCES[name].stop(keep_directories)
MEMGRAPH_INSTANCES[name].stop()
MEMGRAPH_INSTANCES.pop(name)
def stop(context, name, keep_directories=True):
def stop(context, name):
if name != "all":
stop_instance(context, name, keep_directories)
stop_instance(context, name)
return
stop_all()
def kill(context, name, keep_directories=True):
def kill(context, name):
for key in context.keys():
if key != name:
continue
MEMGRAPH_INSTANCES[name].kill(keep_directories)
MEMGRAPH_INSTANCES[name].kill()
MEMGRAPH_INSTANCES.pop(name)
def cleanup_directories_on_exit(value=True):
CLEANUP_DIRECTORIES_ON_EXIT = value
@atexit.register
def cleanup():
stop_all(CLEANUP_DIRECTORIES_ON_EXIT)
stop_all()
def start_instance(context, name, procdir):
@@ -191,8 +184,8 @@ def start_instance(context, name, procdir):
assert len(mg_instances) == 1
def start_all(context, procdir="", keep_directories=True):
stop_all(keep_directories)
def start_all(context, procdir=""):
stop_all()
for key, _ in context.items():
start_instance(context, key, procdir)

View File

@@ -11,7 +11,6 @@
import copy
import os
import shutil
import subprocess
import sys
import time
@@ -57,14 +56,13 @@ def replace_paths(path):
class MemgraphInstanceRunner:
def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False, delete_on_stop=None):
def __init__(self, binary_path=MEMGRAPH_BINARY, use_ssl=False):
self.host = "127.0.0.1"
self.bolt_port = None
self.binary_path = binary_path
self.args = None
self.proc_mg = None
self.ssl = use_ssl
self.delete_on_stop = delete_on_stop
def execute_setup_queries(self, setup_queries):
if setup_queries is None:
@@ -130,7 +128,7 @@ class MemgraphInstanceRunner:
return False
return True
def stop(self, keep_directories=False):
def stop(self):
if not self.is_running():
return
@@ -142,16 +140,9 @@ class MemgraphInstanceRunner:
time.sleep(1)
if not keep_directories:
for folder in self.delete_on_stop or {}:
shutil.rmtree(folder)
def kill(self, keep_directories=False):
def kill(self):
if not self.is_running():
return
self.proc_mg.kill()
code = self.proc_mg.wait()
if not keep_directories:
for folder in self.delete_on_stop or {}:
shutil.rmtree(folder)
assert code == -9, "The killed Memgraph process exited with non-nine!"

View File

@@ -431,7 +431,7 @@ def test_node_type_properties1():
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)[0]
)
assert (result) == [":`Activity`", ["Activity"], "location", "String", False]
assert (result) == [":`Activity`", ["Activity"], "location", ["String"], True]
result = list(
execute_and_fetch_all(
@@ -439,7 +439,7 @@ def test_node_type_properties1():
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)[1]
)
assert (result) == [":`Activity`", ["Activity"], "name", "String", False]
assert (result) == [":`Activity`", ["Activity"], "name", ["String"], True]
result = list(
execute_and_fetch_all(
@@ -447,7 +447,7 @@ def test_node_type_properties1():
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)[2]
)
assert (result) == [":`Dog`", ["Dog"], "name", "String", False]
assert (result) == [":`Dog`", ["Dog"], "name", ["String"], True]
result = list(
execute_and_fetch_all(
@@ -455,81 +455,7 @@ def test_node_type_properties1():
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)[3]
)
assert (result) == [":`Dog`", ["Dog"], "owner", "String", False]
def test_node_type_properties2():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (d:MyNode)
CREATE (n:MyNode)
""",
)
result = execute_and_fetch_all(
cursor,
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)
assert (list(result[0])) == [":`MyNode`", ["MyNode"], "", "", False]
assert (result.__len__()) == 1
def test_node_type_properties3():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (d:Dog {name: 'Rex', owner: 'Carl'})
CREATE (n:Dog)
""",
)
result = execute_and_fetch_all(
cursor,
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)
assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", False]
assert (list(result[1])) == [":`Dog`", ["Dog"], "owner", "String", False]
assert (result.__len__()) == 2
def test_node_type_properties4():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (n:Label1:Label2 {property1: 'value1', property2: 'value2'})
CREATE (m:Label2:Label1 {property3: 'value3'})
""",
)
result = list(
execute_and_fetch_all(
cursor,
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)
)
assert (list(result[0])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property1", "String", False]
assert (list(result[1])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property2", "String", False]
assert (list(result[2])) == [":`Label1`:`Label2`", ["Label1", "Label2"], "property3", "String", False]
assert (result.__len__()) == 3
def test_node_type_properties5():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (d:Dog {name: 'Rex'})
""",
)
result = execute_and_fetch_all(
cursor,
f"CALL libschema.node_type_properties() YIELD nodeType, nodeLabels, propertyName, propertyTypes , mandatory RETURN nodeType, nodeLabels, propertyName, propertyTypes , mandatory ORDER BY propertyName, nodeLabels[0];",
)
assert (list(result[0])) == [":`Dog`", ["Dog"], "name", "String", True]
assert (result.__len__()) == 1
assert (result) == [":`Dog`", ["Dog"], "owner", ["String"], True]
def test_rel_type_properties1():
@@ -547,38 +473,5 @@ def test_rel_type_properties1():
assert (result) == [":`LOVES`", "", "", False]
def test_rel_type_properties2():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (d:Dog {name: 'Rex', owner: 'Carl'})-[l:LOVES]->(a:Activity {name: 'Running', location: 'Zadar'})
CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'})
""",
)
result = execute_and_fetch_all(
cursor,
f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;",
)
assert (list(result[0])) == [":`LOVES`", "duration", "Int", False]
assert (result.__len__()) == 1
def test_rel_type_properties3():
cursor = connect().cursor()
execute_and_fetch_all(
cursor,
"""
CREATE (n:Dog {name: 'Simba', owner: 'Lucy'})-[j:LOVES {duration: 30}]->(b:Activity {name: 'Running', location: 'Zadar'})
""",
)
result = execute_and_fetch_all(
cursor,
f"CALL libschema.rel_type_properties() YIELD relType,propertyName, propertyTypes , mandatory RETURN relType, propertyName, propertyTypes , mandatory;",
)
assert (list(result[0])) == [":`LOVES`", "duration", "Int", True]
assert (result.__len__()) == 1
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

View File

@@ -53,8 +53,8 @@ def run(args):
# Setup.
@atexit.register
def cleanup(keep_directories=True):
interactive_mg_runner.stop_all(keep_directories)
def cleanup():
interactive_mg_runner.stop_all()
if "pre_set_workload" in workload:
binary = os.path.join(BUILD_DIR, workload["pre_set_workload"])
@@ -92,7 +92,7 @@ def run(args):
data = mg_instance.query(validation["query"], conn)[0][0]
assert data == validation["expected"]
conn.close()
cleanup(keep_directories=False)
cleanup()
log.info("%s PASSED.", workload_name)

View File

@@ -1,46 +0,0 @@
# Copyright 2023 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
from workloads.base import Workload
class OrderBy(Workload):
NAME = "order_by"
CARDINALITY = 10000
def indexes_generator(self):
return [
("CREATE INDEX ON :Node;", {}),
("CREATE INDEX ON :Node(prop1);", {}),
("CREATE INDEX ON :Node(prop2);", {}),
("CREATE INDEX ON :Node(prop3);", {}),
]
def dataset_generator(self):
queries = []
for i in range(0, OrderBy.CARDINALITY):
queries.append(
(
"""CREATE
(:Node {prop1: $id, prop2: $id, prop3: $id
});
""",
{"id": i},
)
)
return queries
def benchmark__test__order_by(self):
return (
"MATCH (n:Node) RETURN n ORDER BY n.prop1",
{},
)

View File

@@ -260,11 +260,10 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
# Problem with test using detach delete and memory tracker
# is that memory tracker gets updated immediately
# whereas RES takes some time
# Tries 10 times or fails
cnt_again = 10
cnt_again = 3
skip_failure = False
# 10% is maximum diff for this test to pass
multiplier = 1.10
# 10% is maximum increment, afterwards is fail
multiplier = 1
while cnt_again:
new_memory_tracker, new_res_data = get_storage_data(session)
@@ -278,6 +277,7 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
f"RES data: {new_res_data}, multiplier: {multiplier}"
)
break
multiplier += 0.05
cnt_again -= 1
if not skip_failure:
log.info(memory_tracker, initial_diff, res_data)

View File

@@ -179,7 +179,7 @@ LARGE_DATASET = (
"--vertex-count",
"200000",
"--edge-count",
"1000000",
"500000",
"--max-time",
"480",
"--verify",

View File

@@ -58,7 +58,6 @@ class TestEnvironment : public ::testing::Environment {
void TearDown() override {
ptr_.reset();
auth.reset();
std::filesystem::remove_all(storage_directory);
}
static std::unique_ptr<memgraph::dbms::DbmsHandler> ptr_;

View File

@@ -58,7 +58,6 @@ class TestEnvironment : public ::testing::Environment {
void TearDown() override {
ptr_.reset();
auth.reset();
std::filesystem::remove_all(storage_directory);
}
static std::unique_ptr<memgraph::dbms::DbmsHandler> ptr_;

View File

@@ -101,8 +101,6 @@ class InterpreterTest : public ::testing::Test {
disk_test_utils::RemoveRocksDbDirs(testSuite);
disk_test_utils::RemoveRocksDbDirs(testSuiteCsv);
}
std::filesystem::remove_all(data_directory);
}
InterpreterFaker default_interpreter{&interpreter_context, db};

View File

@@ -700,11 +700,6 @@ TYPED_TEST(DumpTest, CheckStateVertexWithMultipleProperties) {
config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s1").disk;
config.force_on_disk = true;
}
auto on_exit_s1 = memgraph::utils::OnScopeExit{[&]() {
if constexpr (std::is_same_v<TypeParam, memgraph::storage::DiskStorage>) {
disk_test_utils::RemoveRocksDbDirs("query-dump-s1");
}
}};
memgraph::replication::ReplicationState repl_state(ReplicationStateRootPath(config));
memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gk(config, repl_state);
@@ -819,11 +814,7 @@ TYPED_TEST(DumpTest, CheckStateSimpleGraph) {
config.disk = disk_test_utils::GenerateOnDiskConfig("query-dump-s2").disk;
config.force_on_disk = true;
}
auto on_exit_s2 = memgraph::utils::OnScopeExit{[&]() {
if constexpr (std::is_same_v<TypeParam, memgraph::storage::DiskStorage>) {
disk_test_utils::RemoveRocksDbDirs("query-dump-s2");
}
}};
memgraph::replication::ReplicationState repl_state{ReplicationStateRootPath(config)};
memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gk{config, repl_state};
auto db_acc_opt = db_gk.access();

View File

@@ -15,7 +15,6 @@
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/isolation_level.hpp"
#include "utils/on_scope_exit.hpp"
namespace {
int64_t VerticesCount(memgraph::storage::Storage::Accessor *accessor) {
@@ -114,7 +113,6 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) {
for (const auto override_isolation_level : isolation_levels) {
std::unique_ptr<memgraph::storage::Storage> storage(new memgraph::storage::DiskStorage(config));
auto on_exit = memgraph::utils::OnScopeExit{[&]() { disk_test_utils::RemoveRocksDbDirs(testSuite); }};
try {
this->TestVisibility(storage, default_isolation_level, override_isolation_level);
} catch (memgraph::utils::NotYetImplemented &) {
@@ -122,8 +120,10 @@ TEST_P(StorageIsolationLevelTest, VisibilityOnDiskStorage) {
override_isolation_level != memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION) {
continue;
}
disk_test_utils::RemoveRocksDbDirs(testSuite);
throw;
}
disk_test_utils::RemoveRocksDbDirs(testSuite);
}
}

View File

@@ -75,8 +75,6 @@ class StorageModeMultiTxTest : public ::testing::Test {
return tmp;
}(); // iile
void TearDown() override { std::filesystem::remove_all(data_directory); }
memgraph::storage::Config config{.durability.storage_directory = data_directory,
.disk.main_storage_directory = data_directory / "disk"};