diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index 73290a0b4..9175e9c13 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -320,3 +320,47 @@ jobs: with: name: "Enterprise DEB package" path: build/output/memgraph*.deb + + release_jepsen_test: + name: "Release Jepsen Test" + runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl] + env: + THREADS: 24 + + steps: + - name: Set up repository + uses: actions/checkout@v2 + with: + # Number of commits to fetch. `0` indicates all history for all + # branches and tags. (default: 1) + fetch-depth: 0 + + - name: Build release binaries + run: | + # Activate toolchain. + source /opt/toolchain-v2/activate + + # Initialize dependencies. + ./init + + # Build only memgraph release binarie. + cd build + cmake -DCMAKE_BUILD_TYPE=release .. + make -j$THREADS memgraph + + - name: Run Jepsen tests + run: | + cd tests/jepsen + ./setup-local-docker-cluster.sh copy ../../build/memgraph + # NOTE: docker exec -t is NOT ok because gh user does NOT have TTY. + # NOTE: ~/.bashrc has to be manually sourced when bash -c is used + # because some Jepsen config is there. + docker exec jepsen-control bash -c "source ~/.bashrc && cd memgraph && lein run test-all --local-binary /opt/memgraph/memgraph --node n1" + docker exec jepsen-control bash -c 'tar -czvf /jepsen/memgraph/Jepsen.tar.gz $(readlink -f /jepsen/memgraph/store/latest)' + docker cp jepsen-control:/jepsen/memgraph/Jepsen.tar.gz ./ + + - name: Save Jepsen report + uses: actions/upload-artifact@v2 + with: + name: "Jepsen Report" + path: tests/jepsen/Jepsen.tar.gz diff --git a/.github/workflows/test_all_workers.yaml b/.github/workflows/test_all_workers.yaml index 0a4a0cb7b..854fa5dd8 100644 --- a/.github/workflows/test_all_workers.yaml +++ b/.github/workflows/test_all_workers.yaml @@ -36,6 +36,8 @@ jobs: run: | source /opt/toolchain-v2/activate ./tools/check-build-system + docker --version + docker ps | grep jepsen HP-DL360G6-v2-1: name: "HP-DL360G6-v2-1" diff --git a/tests/jepsen/.gitignore b/tests/jepsen/.gitignore new file mode 100644 index 000000000..863ad65af --- /dev/null +++ b/tests/jepsen/.gitignore @@ -0,0 +1,13 @@ +/target +/classes +/checkouts +profiles.clj +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +/store +/jepsen +/Jepsen.tar.gz diff --git a/tests/jepsen/CHANGELOG.md b/tests/jepsen/CHANGELOG.md new file mode 100644 index 000000000..f93aefd8b --- /dev/null +++ b/tests/jepsen/CHANGELOG.md @@ -0,0 +1,5 @@ +# Change Log + +All notable changes to this project will be documented in this file. This +change log follows the conventions of +[keepachangelog.com](http://keepachangelog.com/). diff --git a/tests/jepsen/README.md b/tests/jepsen/README.md new file mode 100644 index 000000000..292e4964a --- /dev/null +++ b/tests/jepsen/README.md @@ -0,0 +1,4 @@ +# Jepsen Memgraph Test Suite + +NOTE: Jepsen can only connect to the SSH server on the default 22 port. +`--node` flag only takes the actual address (:port doesn't work). diff --git a/tests/jepsen/doc/intro.md b/tests/jepsen/doc/intro.md new file mode 100644 index 000000000..e79e1fd79 --- /dev/null +++ b/tests/jepsen/doc/intro.md @@ -0,0 +1,3 @@ +# Introduction to Jepsen Memgraph Test Suite + +TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) diff --git a/tests/jepsen/project.clj b/tests/jepsen/project.clj new file mode 100644 index 000000000..f07bda16f --- /dev/null +++ b/tests/jepsen/project.clj @@ -0,0 +1,11 @@ +(defproject jepsen.memgraph "0.1.0-SNAPSHOT" + :description "A Jepsen test for Memgraph" + :url "https://memgraph.com/" + :license {:name "Memgraph Enterprise" + :url "https://github.com/memgraph/memgraph/blob/master/release/LICENSE_ENTERPRISE.md"} + :main jepsen.memgraph.core + :dependencies [[org.clojure/clojure "1.10.0"] + [jepsen "0.2.1-SNAPSHOT"] + [gorillalabs/neo4j-clj "4.1.0"]] + :profiles {:test {:dependencies [#_[org.neo4j.test/neo4j-harness "4.1.0"]]}} + :repl-options {:init-ns jepsen.memgraph.core}) diff --git a/tests/jepsen/setup-local-docker-cluster.sh b/tests/jepsen/setup-local-docker-cluster.sh new file mode 100755 index 000000000..418284c59 --- /dev/null +++ b/tests/jepsen/setup-local-docker-cluster.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -Eeuo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# NOTE: docker and docker-compose have to be installed on the system. + +if [ ! -d "$script_dir/jepsen" ]; then + git clone https://github.com/jepsen-io/jepsen.git -b "0.2.1" "$script_dir/jepsen" +fi + +# Initialize testing context by copying source/binary files. Inside CI, +# Memgraph is tested on a single machine cluster based on Docker containers. +# Once these tests will be part of the official Jepsen repo, the majority of +# functionalities inside this script won't be needed because each node clones +# the public repo. +case $1 in + # Start Jepsen Docker cluster. + up) + "$script_dir/jepsen/docker/bin/up" --daemon + ;; + # Copy Memgraph Jepsen project files and Memgraph binary if specified. + copy) + shift 1 + binary_path=$1 + if [ -L "$binary_path" ]; then + binary_path=$(readlink "$binary_path") + fi + binary_name=$(basename -- "$binary_path") + + # Copy Memgraph build binary. + docker exec jepsen-n1 mkdir -p /opt/memgraph + docker exec jepsen-n2 mkdir -p /opt/memgraph + docker exec jepsen-n3 mkdir -p /opt/memgraph + docker exec jepsen-n4 mkdir -p /opt/memgraph + docker exec jepsen-n5 mkdir -p /opt/memgraph + docker cp "$binary_path" jepsen-n1:/opt/memgraph/"$binary_name" + docker exec jepsen-n1 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + docker cp "$binary_path" jepsen-n2:/opt/memgraph/"$binary_name" + docker exec jepsen-n2 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + docker cp "$binary_path" jepsen-n3:/opt/memgraph/"$binary_name" + docker exec jepsen-n3 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + docker cp "$binary_path" jepsen-n4:/opt/memgraph/"$binary_name" + docker exec jepsen-n4 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + docker cp "$binary_path" jepsen-n5:/opt/memgraph/"$binary_name" + docker exec jepsen-n5 bash -c "rm -f /opt/memgraph/memgraph && ln -s /opt/memgraph/$binary_name /opt/memgraph/memgraph" + + # Copy tests/jepsen/memgraph required files into the control node. + docker exec jepsen-control mkdir -p /jepsen/memgraph + docker cp "$script_dir/src/." jepsen-control:/jepsen/memgraph/src/ + docker cp "$script_dir/test/." jepsen-control:/jepsen/memgraph/test/ + docker cp "$script_dir/project.clj" jepsen-control:/jepsen/memgraph/project.clj + ;; +esac diff --git a/tests/jepsen/src/jepsen/memgraph/basic.clj b/tests/jepsen/src/jepsen/memgraph/basic.clj new file mode 100644 index 000000000..626ff742a --- /dev/null +++ b/tests/jepsen/src/jepsen/memgraph/basic.clj @@ -0,0 +1,75 @@ +(ns jepsen.memgraph.basic + "Basic Memgraph test" + (:require [neo4j-clj.core :as dbclient] + [jepsen [client :as client] + [checker :as checker] + [generator :as gen]] + [jepsen.checker.timeline :as timeline] + [knossos.model :as model] + [slingshot.slingshot :refer [try+ throw+]] + [jepsen.memgraph.client :as c])) + +;; Operations specific to the tested system (Bolt and Cypher). + +(dbclient/defquery create-node + "CREATE (n:Node {id: $id, value: $value});") +(dbclient/defquery get-node + "MATCH (n:Node {id: $id}) RETURN n;") +(dbclient/defquery get-all-nodes + "MATCH (n:Node) RETURN n;") +(dbclient/defquery update-node + "MATCH (n:Node {id: $id}) SET n.value = $value;") + +(defn compare-and-set-node + [conn o n] + (dbclient/with-transaction conn tx + (if (= (-> (get-node tx {:id "0"}) first :n :value) (str o)) + (update-node tx {:id "0" :value n}) + (throw+ "Unable to alter something that does NOT exist.")))) +(dbclient/defquery detach-delete-all + "MATCH (n) DETACH DELETE n;") + +;; Client specific to the tested system (Bolt and Cypher). +;; TODO (gitbuda): Write an impl targeting Memgraph replicated cluster. +(defrecord Client [conn] + client/Client + (open! [this test node] + (assoc this :conn (c/open node))) + (setup! [this test] + (c/with-session conn session + (create-node session {:id "0" :value 0}))) + (invoke! [this test op] + (case (:f op) + :read (assoc op :type :ok + :value (c/with-session conn session + (-> (get-node session {:id "0"}) first :n :value))) + :write (do (c/with-session conn session + (update-node session {:id "0" :value (:value op)})) + (assoc op :type :ok)) + :cas (try+ + (let [[o n] (:value op)] + (assoc op :type (if (compare-and-set-node conn o n) :ok :fail))) + (catch Object _ + (assoc op :type :fail, :error :not-found))))) + (teardown! [this test] + (c/with-session conn session + (detach-delete-all session))) + (close! [_ est] + (dbclient/disconnect conn))) + +;; Abstract operations executed against the tested system. +(defn r [_ _] {:type :invoke, :f :read, :value nil}) +(defn w [_ _] {:type :invoke, :f :write, :value (rand-int 5)}) +(defn cas [_ _] {:type :invoke, :f :cas, :value [(rand-int 5) (rand-int 5)]}) + +(defn workload + "Basic test workload" + [opts] + {:client (Client. nil) + :checker (checker/compose + {:linear (checker/linearizable + {:model (model/cas-register 0) + :algorithm :linear}) + :timeline (timeline/html)}) + :generator (gen/mix [r w cas])}) + diff --git a/tests/jepsen/src/jepsen/memgraph/client.clj b/tests/jepsen/src/jepsen/memgraph/client.clj new file mode 100644 index 000000000..d3d5734d1 --- /dev/null +++ b/tests/jepsen/src/jepsen/memgraph/client.clj @@ -0,0 +1,23 @@ +(ns jepsen.memgraph.client + "Neo4j Clojure driver helper functions/macros" + (:require [neo4j-clj.core :as dbclient]) + (:import (java.net URI))) + +;; Jepsen related utils. +(defn instance-url + "An URL for connecting to an instance on a particular port" + [node port] + (str "bolt://" node ":" port)) + +;; neo4j-clj related utils. +(defmacro with-session + "Execute body expressions by using the same session. Useful when executing + multiple queries, each as a separete transaction." + [connection session & body] + `(with-open [~session (dbclient/get-session ~connection)] + ~@body)) + +(defn open + "Open client connection to the node" + [node] + (dbclient/connect (URI. (instance-url node 7687)) "" "")) diff --git a/tests/jepsen/src/jepsen/memgraph/core.clj b/tests/jepsen/src/jepsen/memgraph/core.clj new file mode 100644 index 000000000..2ff58ebb0 --- /dev/null +++ b/tests/jepsen/src/jepsen/memgraph/core.clj @@ -0,0 +1,78 @@ +(ns jepsen.memgraph.core + (:gen-class) + (:require [clojure.tools.logging :refer :all] + [clojure.string :as str] + [jepsen [cli :as cli] + [core :as jepsen] + [checker :as checker] + [nemesis :as nemesis] + [generator :as gen] + [tests :as tests]] + [slingshot.slingshot :refer [try+ throw+]] + [jepsen.memgraph [basic :as basic] + [support :as s]])) + +(def workloads + "A map of workload names to functions that can take opts and construct + workloads." + {:basic basic/workload}) + +(defn memgraph-test + "Given an options map from the command line runner (e.g. :nodes, :ssh, + :concurrency, ...), constructs a test map." + [opts] + (let [workload ((get workloads (:workload opts)) opts)] + (merge tests/noop-test + opts + {:pure-generators true + :name (str "test-" (name (:workload opts))) + :db (s/db (:package-url opts) (:local-binary opts)) + :client (:client workload) + :checker (checker/compose + ;; Fails on a cluster of independent Memgraphs. + {;; :stats (checker/stats) CAS always fails + ;; so enable this + ;; if all test have + ;; at least 1 ok op + :exceptions (checker/unhandled-exceptions) + :perf (checker/perf) + :workload (:checker workload)}) + :nemesis (nemesis/partition-random-halves) + :generator (->> (:generator workload) + (gen/nemesis + (cycle [(gen/sleep 5) + {:type :info, :f :start} + (gen/sleep 5) + {:type :info, :f :stop}])) + (gen/time-limit (:time-limit opts)))}))) + +(def cli-opts + "CLI options for tests." + [[nil "--package-url URL" "What package of Memgraph should we test?" + :default nil] + [nil "--local-binary PATH" "Ignore package; use this local binary instead." + :default "/opt/memgraph/memgraph"] + ["-w" "--workload NAME" "Test workload to run" + :parse-fn keyword + :validate [workloads (cli/one-of workloads)]]]) + +(defn all-tests + "Takes base CLI options and constructs a sequence of test options." + [opts] + (let [counts (range (:test-count opts)) + workloads (if-let [w (:workload opts)] [w] (keys workloads)) + test-opts (for [i counts, w workloads] + (assoc opts + :workload w))] + (map memgraph-test test-opts))) + +(defn -main + "Handles command line arguments. Can either run a test, or a web server for + browsing results." + [& args] + (cli/run! (merge (cli/test-all-cmd {:tests-fn all-tests + :opt-spec cli-opts}) + (cli/single-test-cmd {:test-fn memgraph-test + :opt-spec cli-opts}) + (cli/serve-cmd)) + args)) diff --git a/tests/jepsen/src/jepsen/memgraph/support.clj b/tests/jepsen/src/jepsen/memgraph/support.clj new file mode 100644 index 000000000..df8a847ef --- /dev/null +++ b/tests/jepsen/src/jepsen/memgraph/support.clj @@ -0,0 +1,44 @@ +(ns jepsen.memgraph.support + (:require [clojure.string :as str] + [clojure.tools.logging :refer [info]] + [jepsen [db :as db] + [control :as c] + [util :as util :refer [meh]]] + [jepsen.control.util :as cu] + [jepsen.os.debian :as debian])) + +;; Memgraph database config and setup. +(def mgdir "/opt/memgraph") +(def mgdata (str mgdir "/mg_data")) +(def mglog (str mgdir "/memgraph.log")) +(def mgpid (str mgdir "/memgraph.pid")) +(defn db + "Manage Memgraph DB on each node." + [package-url local-binary] + (reify db/DB + (setup! [_ test node] + (c/su (debian/install ['python3 'python3-dev])) + (c/su (meh (c/exec :killall :memgraph))) + (when-not (nil? package-url) + (throw (Exception. "Memgraph package-url setup not yet implemented."))) + (when (nil? local-binary) + (throw (Exception. "Memgraph local-binary has to be defined."))) + (try (c/exec :command :-v local-binary) + (catch Exception e + (throw (Exception. (str local-binary " is not there."))))) + (info node "Memgraph binary is there" local-binary) + (cu/start-daemon! + {:logfile mglog + :pidfile mgpid + :chdir mgdir} + local-binary) + (Thread/sleep 2000)) + (teardown! [_ test node] + (info node "Tearing down Memgraph") + (when (and local-binary mgpid) (cu/stop-daemon! local-binary mgpid)) + (c/su + (c/exec :rm :-rf mgdata) + (c/exec :rm :-rf mglog))) + db/LogFiles + (log-files [_ test node] + [mglog]))) diff --git a/tests/jepsen/test/jepsen/memgraph_test.clj b/tests/jepsen/test/jepsen/memgraph_test.clj new file mode 100644 index 000000000..64c35c013 --- /dev/null +++ b/tests/jepsen/test/jepsen/memgraph_test.clj @@ -0,0 +1,7 @@ +(ns jepsen.memgraph-test + (:require [clojure.test :refer :all] + [jepsen.memgraph :refer :all])) + +(deftest a-test + (testing "FIXME, I fail." + (is (= 0 1))))