From 2c6a55775dbda2ce7e51ce72799eaa319d6a4ffc Mon Sep 17 00:00:00 2001 From: Antonio Filipovic <61245998+antoniofilipovic@users.noreply.github.com> Date: Wed, 19 Apr 2023 16:10:20 +0200 Subject: [PATCH 1/3] Fix max block size bug on LOAD CSV(#877) --- src/query/interpreter.cpp | 2 +- src/query/interpreter.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index 6d47f8633..1c2273c6e 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -2729,7 +2729,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, // Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage. // QueryExecution MemoryResource is mostly used for allocations done on Frame and storing `row`s query_executions_[query_executions_.size() - 1] = std::make_unique( - utils::PoolResource(1, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource())); + utils::PoolResource(8, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource())); query_execution_ptr = &query_executions_.back(); } } diff --git a/src/query/interpreter.hpp b/src/query/interpreter.hpp index 53d8b7e60..ba91ccf0d 100644 --- a/src/query/interpreter.hpp +++ b/src/query/interpreter.hpp @@ -51,7 +51,7 @@ extern const Event FailedQuery; namespace memgraph::query { inline constexpr size_t kExecutionMemoryBlockSize = 1UL * 1024UL * 1024UL; -inline constexpr size_t kExecutionPoolMaxBlockSize = 32768UL; // 2 ^ 15 +inline constexpr size_t kExecutionPoolMaxBlockSize = 2048UL; // 2 ^ 11 class AuthQueryHandler { public: From b02b2011299b25042e2fdfbdf8060a79931d9936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Sun, 23 Apr 2023 16:16:49 +0200 Subject: [PATCH 2/3] Improve Jepsen setup (#893) --- tests/jepsen/run.sh | 46 ++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/tests/jepsen/run.sh b/tests/jepsen/run.sh index 56cbae3ea..f0764c327 100755 --- a/tests/jepsen/run.sh +++ b/tests/jepsen/run.sh @@ -3,6 +3,23 @@ set -Eeuo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +MEMGRAPH_BINARY_PATH="../../build/memgraph" +# NOTE: On Ubuntu 22.04 0.3.2 uses non-existing docker compose --compatibility flag. +# NOTE: On Ubuntu 22.04 0.3.1 seems to be working. +JEPSEN_VERSION="${JEPSEN_VERSION:-v0.3.0}" +JEPSEN_ACTIVE_NODES_NO=5 +CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn" +CONTROL_LEIN_RUN_STDOUT_LOGS=1 +CONTROL_LEIN_RUN_STDERR_LOGS=1 +PRINT_CONTEXT() { + echo -e "MEMGRAPH_BINARY_PATH:\t\t $MEMGRAPH_BINARY_PATH" + echo -e "JEPSEN_VERSION:\t\t\t $JEPSEN_VERSION" + echo -e "JEPSEN_ACTIVE_NODES_NO:\t\t $JEPSEN_ACTIVE_NODES_NO" + echo -e "CONTROL_LEIN_RUN_ARGS:\t\t $CONTROL_LEIN_RUN_ARGS" + echo -e "CONTROL_LEIN_RUN_STDOUT_LOGS:\t $CONTROL_LEIN_RUN_STDOUT_LOGS" + echo -e "CONTROL_LEIN_RUN_STDERR_LOGS:\t $CONTROL_LEIN_RUN_STDERR_LOGS" +} + HELP_EXIT() { echo "" echo "HELP: $0 help|cluster-up|test [args]" @@ -28,15 +45,10 @@ if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/nu ERROR "docker and docker-compose have to be installed." exit 1 fi - -MEMGRAPH_BINARY_PATH="../../build/memgraph" -JEPSEN_ACTIVE_NODES_NO=5 -CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn" -CONTROL_LEIN_RUN_STDOUT_LOGS=1 -CONTROL_LEIN_RUN_STDERR_LOGS=1 +PRINT_CONTEXT if [ ! -d "$script_dir/jepsen" ]; then - git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen" + git clone https://github.com/jepsen-io/jepsen.git -b "$JEPSEN_VERSION" "$script_dir/jepsen" fi if [ "$#" -lt 1 ]; then @@ -96,21 +108,25 @@ case $1 in esac done - # Resolve binary path if it is a link. + # Copy Memgraph binary, handles both cases, when binary is a sym link + # or a regular file. binary_path="$MEMGRAPH_BINARY_PATH" if [ -L "$binary_path" ]; then binary_path=$(readlink "$binary_path") fi binary_name=$(basename -- "$binary_path") - - # Copy Memgraph binary. for iter in $(seq 1 "$JEPSEN_ACTIVE_NODES_NO"); do jepsen_node_name="jepsen-n$iter" - # Cleanup the node folder with previous binaries. - docker exec "$jepsen_node_name" rm -rf /opt/memgraph/ - docker exec "$jepsen_node_name" mkdir -p /opt/memgraph - docker cp "$binary_path" "$jepsen_node_name":/opt/memgraph/"$binary_name" - docker exec "$jepsen_node_name" bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + docker_exec="docker exec $jepsen_node_name bash -c" + if [ "$binary_name" == "memgraph" ]; then + _binary_name="memgraph_tmp" + else + _binary_name="$binary_name" + fi + $docker_exec "rm -rf /opt/memgraph/ && mkdir -p /opt/memgraph" + docker cp "$binary_path" "$jepsen_node_name":/opt/memgraph/"$_binary_name" + $docker_exec "ln -s /opt/memgraph/$_binary_name /opt/memgraph/memgraph" + $docker_exec "touch /opt/memgraph/memgraph.log" INFO "Copying $binary_name to $jepsen_node_name DONE." done From 97e250129ecdf7bc5d62ea6fdb7001fb5d932aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Mon, 24 Apr 2023 16:22:22 +0200 Subject: [PATCH 3/3] Change `AccumulateCursor` to use `utils::pmr::deque` (#888) * Increase performance by eliminating unnecessary `TypedValue` copies --- src/query/plan/operator.cpp | 3 ++- src/utils/pmr/deque.hpp | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/utils/pmr/deque.hpp diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 2203f36aa..fb729e392 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -53,6 +53,7 @@ #include "utils/likely.hpp" #include "utils/logging.hpp" #include "utils/memory.hpp" +#include "utils/pmr/deque.hpp" #include "utils/pmr/list.hpp" #include "utils/pmr/unordered_map.hpp" #include "utils/pmr/unordered_set.hpp" @@ -3248,7 +3249,7 @@ class AccumulateCursor : public Cursor { private: const Accumulate &self_; const UniqueCursorPtr input_cursor_; - utils::pmr::vector> cache_; + utils::pmr::deque> cache_; decltype(cache_.begin()) cache_it_ = cache_.begin(); bool pulled_all_input_{false}; }; diff --git a/src/utils/pmr/deque.hpp b/src/utils/pmr/deque.hpp new file mode 100644 index 000000000..ecfbf5fc0 --- /dev/null +++ b/src/utils/pmr/deque.hpp @@ -0,0 +1,23 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "utils/memory.hpp" + +namespace memgraph::utils::pmr { + +template +using deque = std::deque>; + +} // namespace memgraph::utils::pmr