From 9966ba1d52d8147fdd67cf4ee8bec422c6ae14e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Tue, 22 Dec 2020 18:18:41 +0100 Subject: [PATCH] Improve Jepsen run script (#64) Co-authored-by: Antonio Andelic --- .github/workflows/diff.yaml | 9 +- src/io/network/endpoint.cpp | 2 +- src/query/interpreter.cpp | 28 ++-- tests/jepsen/run.sh | 179 +++++++++++++++++++++ tests/jepsen/setup-local-docker-cluster.sh | 55 ------- tests/jepsen/src/jepsen/memgraph/core.clj | 2 +- 6 files changed, 197 insertions(+), 78 deletions(-) create mode 100755 tests/jepsen/run.sh delete mode 100755 tests/jepsen/setup-local-docker-cluster.sh diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index a292641d4..7ba4747b0 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -352,16 +352,11 @@ jobs: - name: Run Jepsen tests run: | cd tests/jepsen - ./setup-local-docker-cluster.sh copy ../../build/memgraph - # NOTE: docker exec -t is NOT ok because gh user does NOT have TTY. - # NOTE: ~/.bashrc has to be manually sourced when bash -c is used - # because some Jepsen config is there. - docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run test-all --local-binary /opt/memgraph/memgraph --node-configs resources/node-config.edn" - docker exec jepsen-control bash -c 'tar -czvf /jepsen/memgraph/Jepsen.tar.gz $(readlink -f /jepsen/memgraph/store/latest)' - docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./ + ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs - name: Save Jepsen report uses: actions/upload-artifact@v2 + if: ${{ always() }} with: name: "Jepsen Report" path: tests/jepsen/Jepsen.tar.gz diff --git a/src/io/network/endpoint.cpp b/src/io/network/endpoint.cpp index b48180a51..83f0e096b 100644 --- a/src/io/network/endpoint.cpp +++ b/src/io/network/endpoint.cpp @@ -6,8 +6,8 @@ #include "glog/logging.h" -#include "utils/string.hpp" #include "io/network/endpoint.hpp" +#include "utils/string.hpp" namespace io::network { diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index ed17255ab..0550319e6 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -5,11 +5,11 @@ #include #include "glue/communication.hpp" +#include "query/constants.hpp" #include "query/context.hpp" #include "query/db_accessor.hpp" #include "query/dump.hpp" #include "query/exceptions.hpp" -#include "query/constants.hpp" #include "query/frontend/ast/cypher_main_visitor.hpp" #include "query/frontend/opencypher/parser.hpp" #include "query/frontend/semantic/required_privileges.hpp" @@ -222,20 +222,20 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { } } - auto maybe_ip_and_port = io::network::Endpoint::ParseSocketOrIpAddress( - socket_address, query::kDefaultReplicationPort); - if (maybe_ip_and_port) { - auto [ip, port] = *maybe_ip_and_port; - auto ret = - db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, - {.timeout = timeout, .ssl = std::nullopt}); - if (ret.HasError()) { - throw QueryRuntimeException( - fmt::format("Couldn't register replica '{}'!", name)); - } - } else { - throw QueryRuntimeException("Invalid socket address!"); + auto maybe_ip_and_port = io::network::Endpoint::ParseSocketOrIpAddress( + socket_address, query::kDefaultReplicationPort); + if (maybe_ip_and_port) { + auto [ip, port] = *maybe_ip_and_port; + auto ret = + db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, + {.timeout = timeout, .ssl = std::nullopt}); + if (ret.HasError()) { + throw QueryRuntimeException( + fmt::format("Couldn't register replica '{}'!", name)); } + } else { + throw QueryRuntimeException("Invalid socket address!"); + } } /// @throw QueryRuntimeException if an error ocurred. diff --git a/tests/jepsen/run.sh b/tests/jepsen/run.sh new file mode 100755 index 000000000..85af53874 --- /dev/null +++ b/tests/jepsen/run.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +set -Eeuo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +HELP_EXIT() { + echo "" + echo "HELP: $0 help|cluster-up|test [args]" + echo "" + echo " test args --binary MEMGRAPH_BINARY_PATH" + echo " --ignore-run-stdout-logs Ignore lein run stdout logs." + echo " --ignore-run-stderr-logs Ignore lein run stderr logs." + echo " --nodes-no JEPSEN_ACTIVE_NODES_NO" + echo " --run-args \"CONTROL_LEIN_RUN_ARGS\" (NOTE: quotes)" + echo "" + exit 1 +} + +ERROR() { + /bin/echo -e "\e[101m\e[97m[ERROR]\e[49m\e[39m" "$@" +} + +INFO() { + /bin/echo -e "\e[104m\e[97m[INFO]\e[49m\e[39m" "$@" +} + +if ! command -v docker > /dev/null 2>&1 || ! command -v docker-compose > /dev/null 2>&1; then + ERROR "docker and docker-compose have to be installed." + exit 1 +fi + +MEMGRAPH_BINARY_PATH="../../build/memgraph" +JEPSEN_ACTIVE_NODES_NO=5 +CONTROL_LEIN_RUN_ARGS="test-all --node-configs resources/node-config.edn" +CONTROL_LEIN_RUN_STDOUT_LOGS=1 +CONTROL_LEIN_RUN_STDERR_LOGS=1 + +if [ ! -d "$script_dir/jepsen" ]; then + git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen" +fi + +if [ "$#" -lt 1 ]; then + HELP_EXIT +fi + +# Initialize testing context by copying source/binary files. Inside CI, +# Memgraph is tested on a single machine cluster based on Docker containers. +# Once these tests will be part of the official Jepsen repo, the majority of +# functionalities inside this script won't be needed because each node clones +# the public repo. +case $1 in + help) + HELP_EXIT + ;; + # Start Jepsen Docker cluster of 5 nodes. To configure the cluster please + # take a look under jepsen/docker/docker-compose.yml. + # NOTE: If you delete the jepsen folder where docker config is located, + # the current cluster is broken because it relies on the folder. That can + # happen easiliy because the jepsen folder is git ignored. + cluster-up) + "$script_dir/jepsen/docker/bin/up" --daemon + ;; + # Run tests against the specified Memgraph binary. + test) + shift + while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --binary) + shift + MEMGRAPH_BINARY_PATH="$1" + shift + ;; + --ignore-run-stdout-logs) + CONTROL_LEIN_RUN_STDOUT_LOGS=0 + shift + ;; + --ignore-run-stderr-logs) + CONTROL_LEIN_RUN_STDERR_LOGS=0 + shift + ;; + --nodes-no) + shift + JEPSEN_ACTIVE_NODES_NO="$1" + shift + ;; + --run-args) + shift + CONTROL_LEIN_RUN_ARGS="$1" + shift + ;; + *) + ERROR "Unknown option $1." + HELP_EXIT + ;; + esac + done + + # Resolve binary path if it is a link. + binary_path="$MEMGRAPH_BINARY_PATH" + if [ -L "$binary_path" ]; then + binary_path=$(readlink "$binary_path") + fi + binary_name=$(basename -- "$binary_path") + + # Copy Memgraph binary. + for iter in $(seq 1 "$JEPSEN_ACTIVE_NODES_NO"); do + jepsen_node_name="jepsen-n$iter" + docker exec "$jepsen_node_name" mkdir -p /opt/memgraph + docker cp "$binary_path" "$jepsen_node_name":/opt/memgraph/"$binary_name" + docker exec "$jepsen_node_name" bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + INFO "Copying $binary_name to $jepsen_node_name DONE." + done + + # Copy test files into the control node. + docker exec jepsen-control mkdir -p /jepsen/memgraph + docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/ + docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/ + docker cp "$script_dir/resources/." jepsen-control:/jepsen/memgraph/resources/ + docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj + INFO "Copying test files to jepsen-control DONE." + + start_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z" + + # Run the test. + # NOTE: docker exec -t is NOT ok because gh CI user does NOT have TTY. + # NOTE: ~/.bashrc has to be manually sourced when bash -c is used + # because some Jepsen config is there. + set +e + if [ "$CONTROL_LEIN_RUN_STDOUT_LOGS" -eq 0 ]; then + redirect_stdout_logs="/dev/null" + else + redirect_stdout_logs="/dev/stdout" + fi + if [ "$CONTROL_LEIN_RUN_STDERR_LOGS" -eq 0 ]; then + redirect_stderr_logs="/dev/null" + else + redirect_stderr_logs="/dev/stderr" + fi + INFO "Jepsen run in progress... START_TIME: $start_time" + docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run $CONTROL_LEIN_RUN_ARGS" 1> $redirect_stdout_logs 2> $redirect_stderr_logs + # To be able to archive the run result even if the run fails. + jepsen_run_exit_status=$? + end_time="$(docker exec jepsen-control bash -c 'date -u +"%Y%m%dT%H%M%S"').000Z" + INFO "Jepsen run DONE. END_TIME: $end_time" + set -e + + # Pack all test workload runs between start and end time. + all_workloads=$(docker exec jepsen-control bash -c 'ls /jepsen/memgraph/store/' | grep test-) + all_workload_run_folders="" + for workload in $all_workloads; do + for time_folder in $(docker exec jepsen-control bash -c "ls /jepsen/memgraph/store/$workload"); do + if [[ "$time_folder" == "latest" ]]; then + continue + fi + # The early continue pattern here is nice because bash doesn't + # have >= for the string comparison (marginal values). + if [[ "$time_folder" < "$start_time" ]]; then + continue + fi + if [[ "$time_folder" > "$end_time" ]]; then + continue + fi + all_workload_run_folders="$all_workload_run_folders /jepsen/memgraph/store/$workload/$time_folder" + done + done + docker exec jepsen-control bash -c "tar -czvf /jepsen/memgraph/Jepsen.tar.gz $all_workload_run_folders" + docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./ + INFO "Test and results packing DONE." + + # If the run has failed, this script also has to return non-zero status. + if [ "$jepsen_run_exit_status" -ne 0 ]; then + exit "$jepsen_run_exit_status" + fi + ;; + *) + HELP_EXIT + ;; +esac diff --git a/tests/jepsen/setup-local-docker-cluster.sh b/tests/jepsen/setup-local-docker-cluster.sh deleted file mode 100755 index dcc0f98b2..000000000 --- a/tests/jepsen/setup-local-docker-cluster.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -set -Eeuo pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# NOTE: docker and docker-compose have to be installed on the system. - -if [ ! -d "$script_dir/jepsen" ]; then - git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen" -fi - -# Initialize testing context by copying source/binary files. Inside CI, -# Memgraph is tested on a single machine cluster based on Docker containers. -# Once these tests will be part of the official Jepsen repo, the majority of -# functionalities inside this script won't be needed because each node clones -# the public repo. -case $1 in - # Start Jepsen Docker cluster. - up) - "$script_dir/jepsen/docker/bin/up" --daemon - ;; - # Copy Memgraph Jepsen project files and Memgraph binary if specified. - copy) - shift 1 - binary_path=$1 - if [ -L "$binary_path" ]; then - binary_path=$(readlink "$binary_path") - fi - binary_name=$(basename -- "$binary_path") - - # Copy Memgraph build binary. - docker exec jepsen-n1 mkdir -p /opt/memgraph - docker exec jepsen-n2 mkdir -p /opt/memgraph - docker exec jepsen-n3 mkdir -p /opt/memgraph - docker exec jepsen-n4 mkdir -p /opt/memgraph - docker exec jepsen-n5 mkdir -p /opt/memgraph - docker cp "$binary_path" jepsen-n1:/opt/memgraph/"$binary_name" - docker exec jepsen-n1 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" - docker cp "$binary_path" jepsen-n2:/opt/memgraph/"$binary_name" - docker exec jepsen-n2 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" - docker cp "$binary_path" jepsen-n3:/opt/memgraph/"$binary_name" - docker exec jepsen-n3 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" - docker cp "$binary_path" jepsen-n4:/opt/memgraph/"$binary_name" - docker exec jepsen-n4 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" - docker cp "$binary_path" jepsen-n5:/opt/memgraph/"$binary_name" - docker exec jepsen-n5 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" - - # Copy tests/jepsen/memgraph required files into the control node. - docker exec jepsen-control mkdir -p /jepsen/memgraph - docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/ - docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/ - docker cp "$script_dir/resources/." jepsen-control:/jepsen/memgraph/resources/ - docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj - ;; -esac diff --git a/tests/jepsen/src/jepsen/memgraph/core.clj b/tests/jepsen/src/jepsen/memgraph/core.clj index fcf456a71..69a4a0aa0 100644 --- a/tests/jepsen/src/jepsen/memgraph/core.clj +++ b/tests/jepsen/src/jepsen/memgraph/core.clj @@ -164,7 +164,7 @@ workloads (if-let [w (:workload opts)] [w] (keys workloads)) node-configs (if (:node-configs opts) (merge-node-configurations (:nodes opts) (:node-configs opts)) - [(resolve-all-node-hostnames (default-node-configuration (:nodes opts)))]) + (throw (Exception. "Node config is missing"))) test-opts (for [i counts c node-configs w workloads] (assoc opts :node-config c