From 14c9e6845630e71ea7d141b4447e80230ffcc275 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 12 Aug 2022 08:24:32 +0200 Subject: [PATCH 01/39] Transport prototype (#466) --- .github/workflows/diff.yaml | 11 +- src/CMakeLists.txt | 1 + src/io/address.hpp | 60 ++++ src/io/errors.hpp | 26 ++ src/io/future.hpp | 262 ++++++++++++++++++ src/io/simulator/CMakeLists.txt | 8 + src/io/simulator/message_conversion.hpp | 177 ++++++++++++ src/io/simulator/simulator.hpp | 45 +++ src/io/simulator/simulator_config.hpp | 30 ++ src/io/simulator/simulator_handle.cpp | 154 ++++++++++ src/io/simulator/simulator_handle.hpp | 206 ++++++++++++++ src/io/simulator/simulator_stats.hpp | 25 ++ src/io/simulator/simulator_transport.hpp | 65 +++++ src/io/time.hpp | 21 ++ src/io/transport.hpp | 130 +++++++++ tests/CMakeLists.txt | 3 + tests/benchmark/CMakeLists.txt | 3 + tests/benchmark/future.cpp | 30 ++ tests/simulation/CMakeLists.txt | 30 ++ tests/simulation/basic_request.cpp | 87 ++++++ .../trial_query_storage/messages.hpp | 34 +++ .../query_storage_test.cpp | 85 ++++++ tests/unit/CMakeLists.txt | 4 + tests/unit/future.cpp | 55 ++++ 24 files changed, 1551 insertions(+), 1 deletion(-) create mode 100644 src/io/address.hpp create mode 100644 src/io/errors.hpp create mode 100644 src/io/future.hpp create mode 100644 src/io/simulator/CMakeLists.txt create mode 100644 src/io/simulator/message_conversion.hpp create mode 100644 src/io/simulator/simulator.hpp create mode 100644 src/io/simulator/simulator_config.hpp create mode 100644 src/io/simulator/simulator_handle.cpp create mode 100644 src/io/simulator/simulator_handle.hpp create mode 100644 src/io/simulator/simulator_stats.hpp create mode 100644 src/io/simulator/simulator_transport.hpp create mode 100644 src/io/time.hpp create mode 100644 src/io/transport.hpp create mode 100644 tests/benchmark/future.cpp create mode 100644 tests/simulation/CMakeLists.txt create mode 100644 tests/simulation/basic_request.cpp create mode 100644 tests/simulation/trial_query_storage/messages.hpp create mode 100644 tests/simulation/trial_query_storage/query_storage_test.cpp create mode 100644 tests/unit/future.cpp diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml index bf6a39147..ce2caea75 100644 --- a/.github/workflows/diff.yaml +++ b/.github/workflows/diff.yaml @@ -171,7 +171,7 @@ jobs: # Run leftover CTest tests (all except unit and benchmark tests). cd build - ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure + ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure - name: Run drivers tests run: | @@ -262,6 +262,15 @@ jobs: cd build ctest -R memgraph__unit --output-on-failure -j$THREADS + - name: Run simulation tests + run: | + # Activate toolchain. + source /opt/toolchain-v4/activate + + # Run unit tests. + cd build + ctest -R memgraph__simulation --output-on-failure -j$THREADS + - name: Run e2e tests run: | # TODO(gitbuda): Setup mgclient and pymgclient properly. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index efc653b9a..f4c303daf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(lisp) add_subdirectory(utils) add_subdirectory(requests) add_subdirectory(io) +add_subdirectory(io/simulator) add_subdirectory(kvstore) add_subdirectory(telemetry) add_subdirectory(communication) diff --git a/src/io/address.hpp b/src/io/address.hpp new file mode 100644 index 000000000..94a231e07 --- /dev/null +++ b/src/io/address.hpp @@ -0,0 +1,60 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace memgraph::io { +struct Address { + // It's important for all participants to have a + // unique identifier - IP and port alone are not + // enough, and may change over the lifecycle of + // the nodes. Particularly storage nodes may change + // their IP addresses over time, and the system + // should gracefully update its information + // about them. + boost::uuids::uuid unique_id; + boost::asio::ip::address last_known_ip; + uint16_t last_known_port; + + static Address TestAddress(uint16_t port) { + Address ret; + ret.last_known_port = port; + return ret; + } + + friend bool operator==(const Address &lhs, const Address &rhs) = default; + + /// unique_id is most dominant for ordering, then last_known_ip, then last_known_port + friend bool operator<(const Address &lhs, const Address &rhs) { + if (lhs.unique_id != rhs.unique_id) { + return lhs.unique_id < rhs.unique_id; + } + + if (lhs.last_known_ip != rhs.last_known_ip) { + return lhs.last_known_ip < rhs.last_known_ip; + } + + return lhs.last_known_port < rhs.last_known_port; + } + + std::string ToString() const { + return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}", + boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port); + } +}; +}; // namespace memgraph::io diff --git a/src/io/errors.hpp b/src/io/errors.hpp new file mode 100644 index 000000000..7df2171d9 --- /dev/null +++ b/src/io/errors.hpp @@ -0,0 +1,26 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +namespace memgraph::io { +// Signifies that a retriable operation was unable to +// complete after a configured number of retries. +struct RetriesExhausted {}; + +// Signifies that a request was unable to receive a response +// within some configured timeout duration. It is important +// to remember that in distributed systems, a timeout does +// not signify that a request was not received or processed. +// It may be the case that the request was fully processed +// but that the response was not received. +struct TimedOut {}; +}; // namespace memgraph::io diff --git a/src/io/future.hpp b/src/io/future.hpp new file mode 100644 index 000000000..7b9a4461c --- /dev/null +++ b/src/io/future.hpp @@ -0,0 +1,262 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "io/errors.hpp" +#include "utils/logging.hpp" + +namespace memgraph::io { + +// Shared is in an anonymous namespace, and the only way to +// construct a Promise or Future is to pass a Shared in. This +// ensures that Promises and Futures can only be constructed +// in this translation unit. +namespace details { +template +class Shared { + mutable std::condition_variable cv_; + mutable std::mutex mu_; + std::optional item_; + bool consumed_ = false; + bool waiting_ = false; + std::function simulator_notifier_ = nullptr; + + public: + explicit Shared(std::function simulator_notifier) : simulator_notifier_(simulator_notifier) {} + Shared() = default; + Shared(Shared &&) = delete; + Shared &operator=(Shared &&) = delete; + Shared(const Shared &) = delete; + Shared &operator=(const Shared &) = delete; + ~Shared() = default; + + /// Takes the item out of our optional item_ and returns it. + T Take() { + MG_ASSERT(item_, "Take called without item_ being present"); + MG_ASSERT(!consumed_, "Take called on already-consumed Future"); + + T ret = std::move(item_).value(); + item_.reset(); + + consumed_ = true; + + return ret; + } + + T Wait() { + std::unique_lock lock(mu_); + waiting_ = true; + + while (!item_) { + bool simulator_progressed = false; + if (simulator_notifier_) [[unlikely]] { + // We can't hold our own lock while notifying + // the simulator because notifying the simulator + // involves acquiring the simulator's mutex + // to guarantee that our notification linearizes + // with the simulator's condition variable. + // However, the simulator may acquire our + // mutex to check if we are being awaited, + // while determining system quiescence, + // so we have to get out of its way to avoid + // a cyclical deadlock. + lock.unlock(); + simulator_progressed = std::invoke(simulator_notifier_); + lock.lock(); + if (item_) { + // item may have been filled while we + // had dropped our mutex while notifying + // the simulator of our waiting_ status. + break; + } + } + if (!simulator_progressed) [[likely]] { + cv_.wait(lock); + } + MG_ASSERT(!consumed_, "Future consumed twice!"); + } + + waiting_ = false; + + return Take(); + } + + bool IsReady() const { + std::unique_lock lock(mu_); + return item_; + } + + std::optional TryGet() { + std::unique_lock lock(mu_); + + if (item_) { + return Take(); + } + + return std::nullopt; + } + + void Fill(T item) { + { + std::unique_lock lock(mu_); + + MG_ASSERT(!consumed_, "Promise filled after it was already consumed!"); + MG_ASSERT(!item_, "Promise filled twice!"); + + item_ = item; + } // lock released before condition variable notification + + cv_.notify_all(); + } + + bool IsAwaited() const { + std::unique_lock lock(mu_); + return waiting_; + } +}; +} // namespace details + +template +class Future { + bool consumed_or_moved_ = false; + std::shared_ptr> shared_; + + public: + explicit Future(std::shared_ptr> shared) : shared_(shared) {} + + Future() = delete; + Future(Future &&old) noexcept { + MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed."); + shared_ = std::move(old.shared_); + consumed_or_moved_ = old.consumed_or_moved_; + old.consumed_or_moved_ = true; + } + + Future &operator=(Future &&old) noexcept { + MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed."); + shared_ = std::move(old.shared_); + old.consumed_or_moved_ = true; + } + + Future(const Future &) = delete; + Future &operator=(const Future &) = delete; + ~Future() = default; + + /// Returns true if the Future is ready to + /// be consumed using TryGet or Wait (prefer Wait + /// if you know it's ready, because it doesn't + /// return an optional. + bool IsReady() { + MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!"); + return shared_->IsReady(); + } + + /// Non-blocking method that returns the inner + /// item if it's already ready, or std::nullopt + /// if it is not ready yet. + std::optional TryGet() { + MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!"); + std::optional ret = shared_->TryGet(); + if (ret) { + consumed_or_moved_ = true; + } + return ret; + } + + /// Block on the corresponding promise to be filled, + /// returning the inner item when ready. + T Wait() && { + MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!"); + T ret = shared_->Wait(); + consumed_or_moved_ = true; + return ret; + } + + /// Marks this Future as canceled. + void Cancel() { + MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!"); + consumed_or_moved_ = true; + } +}; + +template +class Promise { + std::shared_ptr> shared_; + bool filled_or_moved_ = false; + + public: + explicit Promise(std::shared_ptr> shared) : shared_(shared) {} + + Promise() = delete; + Promise(Promise &&old) noexcept { + MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled."); + shared_ = std::move(old.shared_); + old.filled_or_moved_ = true; + } + + Promise &operator=(Promise &&old) noexcept { + MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled."); + shared_ = std::move(old.shared_); + old.filled_or_moved_ = true; + } + Promise(const Promise &) = delete; + Promise &operator=(const Promise &) = delete; + + ~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); } + + // Fill the expected item into the Future. + void Fill(T item) { + MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!"); + shared_->Fill(item); + filled_or_moved_ = true; + } + + bool IsAwaited() { return shared_->IsAwaited(); } + + /// Moves this Promise into a unique_ptr. + std::unique_ptr> ToUnique() && { + std::unique_ptr> up = std::make_unique>(std::move(shared_)); + + filled_or_moved_ = true; + + return up; + } +}; + +template +std::pair, Promise> FuturePromisePair() { + std::shared_ptr> shared = std::make_shared>(); + + Future future = Future(shared); + Promise promise = Promise(shared); + + return std::make_pair(std::move(future), std::move(promise)); +} + +template +std::pair, Promise> FuturePromisePairWithNotifier(std::function simulator_notifier) { + std::shared_ptr> shared = std::make_shared>(simulator_notifier); + + Future future = Future(shared); + Promise promise = Promise(shared); + + return std::make_pair(std::move(future), std::move(promise)); +} + +}; // namespace memgraph::io diff --git a/src/io/simulator/CMakeLists.txt b/src/io/simulator/CMakeLists.txt new file mode 100644 index 000000000..1cb61d8d9 --- /dev/null +++ b/src/io/simulator/CMakeLists.txt @@ -0,0 +1,8 @@ +set(io_simulator_sources + simulator_handle.cpp) + +find_package(fmt REQUIRED) +find_package(Threads REQUIRED) + +add_library(mg-io-simulator STATIC ${io_simulator_sources}) +target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils) diff --git a/src/io/simulator/message_conversion.hpp b/src/io/simulator/message_conversion.hpp new file mode 100644 index 000000000..f16c60f65 --- /dev/null +++ b/src/io/simulator/message_conversion.hpp @@ -0,0 +1,177 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "io/transport.hpp" + +namespace memgraph::io::simulator { + +using memgraph::io::Duration; +using memgraph::io::Message; +using memgraph::io::Time; + +struct OpaqueMessage { + Address from_address; + uint64_t request_id; + std::any message; + + /// Recursively tries to match a specific type from the outer + /// variant's parameter pack against the type of the std::any, + /// and if it matches, make it concrete and return it. Otherwise, + /// move on and compare the any with the next type from the + /// parameter pack. + /// + /// Return is the full std::variant type that holds the + /// full parameter pack without interfering with recursive + /// narrowing expansion. + template + std::optional Unpack(std::any &&a) { + if (typeid(Head) == a.type()) { + Head concrete = std::any_cast(std::move(a)); + return concrete; + } + + if constexpr (sizeof...(Rest) > 0) { + return Unpack(std::move(a)); + } else { + return std::nullopt; + } + } + + /// High level "user-facing" conversion function that lets + /// people interested in conversion only supply a single + /// parameter pack for the types that they want to compare + /// with the any and potentially include in the returned + /// variant. + template + requires(sizeof...(Ms) > 0) std::optional> VariantFromAny(std::any &&a) { + return Unpack, Ms...>(std::move(a)); + } + + template + requires(sizeof...(Ms) > 0) std::optional> Take() && { + std::optional> m_opt = VariantFromAny(std::move(message)); + + if (m_opt) { + return RequestEnvelope{ + .message = std::move(*m_opt), + .request_id = request_id, + .from_address = from_address, + }; + } + + return std::nullopt; + } +}; + +class OpaquePromiseTraitBase { + public: + virtual const std::type_info *TypeInfo() const = 0; + virtual bool IsAwaited(void *ptr) const = 0; + virtual void Fill(void *ptr, OpaqueMessage &&) const = 0; + virtual void TimeOut(void *ptr) const = 0; + + virtual ~OpaquePromiseTraitBase() = default; + OpaquePromiseTraitBase() = default; + OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete; + OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete; + OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete; + OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete; +}; + +template +class OpaquePromiseTrait : public OpaquePromiseTraitBase { + public: + const std::type_info *TypeInfo() const override { return &typeid(T); }; + + bool IsAwaited(void *ptr) const override { return static_cast *>(ptr)->IsAwaited(); }; + + void Fill(void *ptr, OpaqueMessage &&opaque_message) const override { + T message = std::any_cast(std::move(opaque_message.message)); + auto response_envelope = ResponseEnvelope{.message = std::move(message), + .request_id = opaque_message.request_id, + .from_address = opaque_message.from_address}; + auto promise = static_cast *>(ptr); + auto unique_promise = std::unique_ptr>(promise); + unique_promise->Fill(std::move(response_envelope)); + }; + + void TimeOut(void *ptr) const override { + auto promise = static_cast *>(ptr); + auto unique_promise = std::unique_ptr>(promise); + ResponseResult result = TimedOut{}; + unique_promise->Fill(std::move(result)); + } +}; + +class OpaquePromise { + void *ptr_; + std::unique_ptr trait_; + + public: + OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) { + MG_ASSERT(old.ptr_ != nullptr); + old.ptr_ = nullptr; + } + + OpaquePromise &operator=(OpaquePromise &&old) noexcept { + MG_ASSERT(ptr_ == nullptr); + MG_ASSERT(old.ptr_ != nullptr); + MG_ASSERT(this != &old); + ptr_ = old.ptr_; + trait_ = std::move(old.trait_); + old.ptr_ = nullptr; + return *this; + } + + OpaquePromise(const OpaquePromise &) = delete; + OpaquePromise &operator=(const OpaquePromise &) = delete; + + template + std::unique_ptr> Take() && { + MG_ASSERT(typeid(T) == *trait_->TypeInfo()); + MG_ASSERT(ptr_ != nullptr); + + auto ptr = static_cast *>(ptr_); + + ptr_ = nullptr; + + return std::unique_ptr(ptr); + } + + template + explicit OpaquePromise(std::unique_ptr> promise) + : ptr_(static_cast(promise.release())), trait_(std::make_unique>()) {} + + bool IsAwaited() { + MG_ASSERT(ptr_ != nullptr); + return trait_->IsAwaited(ptr_); + } + + void TimeOut() { + MG_ASSERT(ptr_ != nullptr); + trait_->TimeOut(ptr_); + ptr_ = nullptr; + } + + void Fill(OpaqueMessage &&opaque_message) { + MG_ASSERT(ptr_ != nullptr); + trait_->Fill(ptr_, std::move(opaque_message)); + ptr_ = nullptr; + } + + ~OpaquePromise() { + MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled"); + } +}; + +} // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator.hpp b/src/io/simulator/simulator.hpp new file mode 100644 index 000000000..354aae6ac --- /dev/null +++ b/src/io/simulator/simulator.hpp @@ -0,0 +1,45 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include + +#include "io/address.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/simulator/simulator_handle.hpp" +#include "io/simulator/simulator_transport.hpp" + +namespace memgraph::io::simulator { +class Simulator { + std::mt19937 rng_; + std::shared_ptr simulator_handle_; + + public: + explicit Simulator(SimulatorConfig config) + : rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared(config)} {} + + void ShutDown() { simulator_handle_->ShutDown(); } + + Io Register(Address address) { + std::uniform_int_distribution seed_distrib; + uint64_t seed = seed_distrib(rng_); + return Io{SimulatorTransport{simulator_handle_, address, seed}, address}; + } + + void IncrementServerCountAndWaitForQuiescentState(Address address) { + simulator_handle_->IncrementServerCountAndWaitForQuiescentState(address); + } + + SimulatorStats Stats() { return simulator_handle_->Stats(); } +}; +}; // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_config.hpp b/src/io/simulator/simulator_config.hpp new file mode 100644 index 000000000..4719488d2 --- /dev/null +++ b/src/io/simulator/simulator_config.hpp @@ -0,0 +1,30 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +#include "io/time.hpp" + +namespace memgraph::io::simulator { + +using memgraph::io::Time; + +struct SimulatorConfig { + uint8_t drop_percent = 0; + bool perform_timeouts = false; + bool scramble_messages = true; + uint64_t rng_seed = 0; + Time start_time = Time::min(); + Time abort_time = Time::max(); +}; +}; // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_handle.cpp b/src/io/simulator/simulator_handle.cpp new file mode 100644 index 000000000..05585f551 --- /dev/null +++ b/src/io/simulator/simulator_handle.cpp @@ -0,0 +1,154 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include "io/simulator/simulator_handle.hpp" +#include "io/address.hpp" +#include "io/errors.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/simulator/simulator_stats.hpp" +#include "io/time.hpp" +#include "io/transport.hpp" + +namespace memgraph::io::simulator { + +using memgraph::io::Duration; +using memgraph::io::Time; + +void SimulatorHandle::ShutDown() { + std::unique_lock lock(mu_); + should_shut_down_ = true; + cv_.notify_all(); +} + +bool SimulatorHandle::ShouldShutDown() const { + std::unique_lock lock(mu_); + return should_shut_down_; +} + +void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) { + std::unique_lock lock(mu_); + server_addresses_.insert(address); + + while (true) { + const size_t blocked_servers = BlockedServers(); + + const bool all_servers_blocked = blocked_servers == server_addresses_.size(); + + if (all_servers_blocked) { + return; + } + + cv_.wait(lock); + } +} + +size_t SimulatorHandle::BlockedServers() { + size_t blocked_servers = blocked_on_receive_; + + for (auto &[promise_key, opaque_promise] : promises_) { + if (opaque_promise.promise.IsAwaited() && server_addresses_.contains(promise_key.requester_address)) { + blocked_servers++; + } + } + + return blocked_servers; +} + +bool SimulatorHandle::MaybeTickSimulator() { + std::unique_lock lock(mu_); + + const size_t blocked_servers = BlockedServers(); + + if (blocked_servers < server_addresses_.size()) { + // we only need to advance the simulator when all + // servers have reached a quiescent state, blocked + // on their own futures or receive methods. + return false; + } + + stats_.simulator_ticks++; + + cv_.notify_all(); + + TimeoutPromisesPastDeadline(); + + if (in_flight_.empty()) { + // return early here because there are no messages to schedule + + // We tick the clock forward when all servers are blocked but + // there are no in-flight messages to schedule delivery of. + std::poisson_distribution<> time_distrib(50); + Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)}; + cluster_wide_time_microseconds_ += clock_advance; + + MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time, + "Cluster has executed beyond its configured abort_time, and something may be failing to make progress " + "in an expected amount of time."); + return true; + } + + if (config_.scramble_messages) { + // scramble messages + std::uniform_int_distribution swap_distrib(0, in_flight_.size() - 1); + const size_t swap_index = swap_distrib(rng_); + std::swap(in_flight_[swap_index], in_flight_.back()); + } + + auto [to_address, opaque_message] = std::move(in_flight_.back()); + in_flight_.pop_back(); + + std::uniform_int_distribution drop_distrib(0, 99); + const int drop_threshold = drop_distrib(rng_); + const bool should_drop = drop_threshold < config_.drop_percent; + + if (should_drop) { + stats_.dropped_messages++; + } + + PromiseKey promise_key{.requester_address = to_address, + .request_id = opaque_message.request_id, + .replier_address = opaque_message.from_address}; + + if (promises_.contains(promise_key)) { + // complete waiting promise if it's there + DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key)); + promises_.erase(promise_key); + + const bool normal_timeout = config_.perform_timeouts && (dop.deadline < cluster_wide_time_microseconds_); + + if (should_drop || normal_timeout) { + stats_.timed_out_requests++; + dop.promise.TimeOut(); + } else { + stats_.total_responses++; + dop.promise.Fill(std::move(opaque_message)); + } + } else if (should_drop) { + // don't add it anywhere, let it drop + } else { + // add to can_receive_ if not + const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector()); + om_vec->second.emplace_back(std::move(opaque_message)); + } + + return true; +} + +Time SimulatorHandle::Now() const { + std::unique_lock lock(mu_); + return cluster_wide_time_microseconds_; +} + +SimulatorStats SimulatorHandle::Stats() { + std::unique_lock lock(mu_); + return stats_; +} +} // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_handle.hpp b/src/io/simulator/simulator_handle.hpp new file mode 100644 index 000000000..6abaa129d --- /dev/null +++ b/src/io/simulator/simulator_handle.hpp @@ -0,0 +1,206 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/address.hpp" +#include "io/errors.hpp" +#include "io/simulator/message_conversion.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/simulator/simulator_stats.hpp" +#include "io/time.hpp" +#include "io/transport.hpp" + +namespace memgraph::io::simulator { + +using memgraph::io::Duration; +using memgraph::io::Time; + +struct PromiseKey { + Address requester_address; + uint64_t request_id; + // TODO(tyler) possibly remove replier_address from promise key + // once we want to support DSR. + Address replier_address; + + public: + friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) { + if (lhs.requester_address != rhs.requester_address) { + return lhs.requester_address < rhs.requester_address; + } + + if (lhs.request_id != rhs.request_id) { + return lhs.request_id < rhs.request_id; + } + + return lhs.replier_address < rhs.replier_address; + } +}; + +struct DeadlineAndOpaquePromise { + Time deadline; + OpaquePromise promise; +}; + +class SimulatorHandle { + mutable std::mutex mu_{}; + mutable std::condition_variable cv_; + + // messages that have not yet been scheduled or dropped + std::vector> in_flight_; + + // the responses to requests that are being waited on + std::map promises_; + + // messages that are sent to servers that may later receive them + std::map> can_receive_; + + Time cluster_wide_time_microseconds_; + bool should_shut_down_ = false; + SimulatorStats stats_; + size_t blocked_on_receive_ = 0; + std::set
server_addresses_; + std::mt19937 rng_; + SimulatorConfig config_; + + /// Returns the number of servers currently blocked on Receive, plus + /// the servers that are blocked on Futures that were created through + /// SimulatorTransport::Request. + /// + /// TODO(tyler) investigate whether avoiding consideration of Futures + /// increases determinism. + size_t BlockedServers(); + + void TimeoutPromisesPastDeadline() { + const Time now = cluster_wide_time_microseconds_; + + for (auto &[promise_key, dop] : promises_) { + if (dop.deadline < now) { + spdlog::debug("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(), + promise_key.replier_address.ToString()); + std::move(dop).promise.TimeOut(); + promises_.erase(promise_key); + + stats_.timed_out_requests++; + } + } + } + + public: + explicit SimulatorHandle(SimulatorConfig config) + : cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {} + + void IncrementServerCountAndWaitForQuiescentState(Address address); + + /// This method causes most of the interesting simulation logic to happen, wrt network behavior. + /// It checks to see if all background "server" threads are blocked on new messages, and if so, + /// it will decide whether to drop, reorder, or deliver in-flight messages based on the SimulatorConfig + /// that was used to create the Simulator. + bool MaybeTickSimulator(); + + void ShutDown(); + + bool ShouldShutDown() const; + + template + void SubmitRequest(Address to_address, Address from_address, uint64_t request_id, Request &&request, Duration timeout, + ResponsePromise &&promise) { + std::unique_lock lock(mu_); + + const Time deadline = cluster_wide_time_microseconds_ + timeout; + + std::any message(request); + OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message)}; + in_flight_.emplace_back(std::make_pair(to_address, std::move(om))); + + PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address}; + OpaquePromise opaque_promise(std::move(promise).ToUnique()); + DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)}; + promises_.emplace(std::move(promise_key), std::move(dop)); + + stats_.total_messages++; + stats_.total_requests++; + + cv_.notify_all(); + } + + template + requires(sizeof...(Ms) > 0) RequestResult Receive(const Address &receiver, Duration timeout) { + std::unique_lock lock(mu_); + + blocked_on_receive_ += 1; + + const Time deadline = cluster_wide_time_microseconds_ + timeout; + + while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) { + if (can_receive_.contains(receiver)) { + std::vector &can_rx = can_receive_.at(receiver); + if (!can_rx.empty()) { + OpaqueMessage message = std::move(can_rx.back()); + can_rx.pop_back(); + + // TODO(tyler) search for item in can_receive_ that matches the desired types, rather + // than asserting that the last item in can_rx matches. + auto m_opt = std::move(message).Take(); + + blocked_on_receive_ -= 1; + + return std::move(m_opt).value(); + } + } + + lock.unlock(); + bool made_progress = MaybeTickSimulator(); + lock.lock(); + if (!should_shut_down_ && !made_progress) { + cv_.wait(lock); + } + } + + blocked_on_receive_ -= 1; + + return TimedOut{}; + } + + template + void Send(Address to_address, Address from_address, uint64_t request_id, M message) { + std::unique_lock lock(mu_); + std::any message_any(std::move(message)); + OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message_any)}; + in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om))); + + stats_.total_messages++; + + cv_.notify_all(); + } + + Time Now() const; + + template , class Return = uint64_t> + Return Rand(D distrib) { + std::unique_lock lock(mu_); + return distrib(rng_); + } + + SimulatorStats Stats(); +}; +}; // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_stats.hpp b/src/io/simulator/simulator_stats.hpp new file mode 100644 index 000000000..7f529a456 --- /dev/null +++ b/src/io/simulator/simulator_stats.hpp @@ -0,0 +1,25 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +namespace memgraph::io::simulator { +struct SimulatorStats { + uint64_t total_messages = 0; + uint64_t dropped_messages = 0; + uint64_t timed_out_requests = 0; + uint64_t total_requests = 0; + uint64_t total_responses = 0; + uint64_t simulator_ticks = 0; +}; +}; // namespace memgraph::io::simulator diff --git a/src/io/simulator/simulator_transport.hpp b/src/io/simulator/simulator_transport.hpp new file mode 100644 index 000000000..b67371ff0 --- /dev/null +++ b/src/io/simulator/simulator_transport.hpp @@ -0,0 +1,65 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include + +#include "io/address.hpp" +#include "io/simulator/simulator_handle.hpp" +#include "io/time.hpp" + +namespace memgraph::io::simulator { + +using memgraph::io::Duration; +using memgraph::io::Time; + +class SimulatorTransport { + std::shared_ptr simulator_handle_; + const Address address_; + std::mt19937 rng_; + + public: + SimulatorTransport(std::shared_ptr simulator_handle, Address address, uint64_t seed) + : simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {} + + template + ResponseFuture Request(Address address, uint64_t request_id, Request request, Duration timeout) { + std::function maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); }; + auto [future, promise] = + memgraph::io::FuturePromisePairWithNotifier>(maybe_tick_simulator); + + simulator_handle_->SubmitRequest(address, address_, request_id, std::move(request), timeout, std::move(promise)); + + return std::move(future); + } + + template + requires(sizeof...(Ms) > 0) RequestResult Receive(Duration timeout) { + return simulator_handle_->template Receive(address_, timeout); + } + + template + void Send(Address address, uint64_t request_id, M message) { + return simulator_handle_->template Send(address, address_, request_id, message); + } + + Time Now() const { return simulator_handle_->Now(); } + + bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); } + + template , class Return = uint64_t> + Return Rand(D distrib) { + return distrib(rng_); + } +}; +}; // namespace memgraph::io::simulator diff --git a/src/io/time.hpp b/src/io/time.hpp new file mode 100644 index 000000000..57f58cab1 --- /dev/null +++ b/src/io/time.hpp @@ -0,0 +1,21 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include + +namespace memgraph::io { + +using Duration = std::chrono::microseconds; +using Time = std::chrono::time_point; + +} // namespace memgraph::io diff --git a/src/io/transport.hpp b/src/io/transport.hpp new file mode 100644 index 000000000..a9e550434 --- /dev/null +++ b/src/io/transport.hpp @@ -0,0 +1,130 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include +#include + +#include "io/address.hpp" +#include "io/errors.hpp" +#include "io/future.hpp" +#include "io/time.hpp" +#include "utils/result.hpp" + +namespace memgraph::io { + +using memgraph::utils::BasicResult; + +// TODO(tyler) ensure that Message continues to represent +// reasonable constraints around message types over time, +// as we adapt things to use Thrift-generated message types. +template +concept Message = std::same_as>; + +template +struct ResponseEnvelope { + M message; + uint64_t request_id; + Address from_address; +}; + +template +using ResponseResult = BasicResult>; + +template +using ResponseFuture = memgraph::io::Future>; + +template +using ResponsePromise = memgraph::io::Promise>; + +template +struct RequestEnvelope { + std::variant message; + uint64_t request_id; + Address from_address; +}; + +template +using RequestResult = BasicResult>; + +template +class Io { + I implementation_; + Address address_; + uint64_t request_id_counter_ = 0; + Duration default_timeout_ = std::chrono::microseconds{50000}; + + public: + Io(I io, Address address) : implementation_(io), address_(address) {} + + /// Set the default timeout for all requests that are issued + /// without an explicit timeout set. + void SetDefaultTimeout(Duration timeout) { default_timeout_ = timeout; } + + /// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients. + template + ResponseFuture RequestWithTimeout(Address address, Request request, Duration timeout) { + const uint64_t request_id = ++request_id_counter_; + return implementation_.template Request(address, request_id, request, timeout); + } + + /// Issue a request that times out after the default timeout. This tends + /// to be used by clients. + template + ResponseFuture Request(Address address, Request request) { + const uint64_t request_id = ++request_id_counter_; + const Duration timeout = default_timeout_; + return implementation_.template Request(address, request_id, std::move(request), timeout); + } + + /// Wait for an explicit number of microseconds for a request of one of the + /// provided types to arrive. This tends to be used by servers. + template + RequestResult ReceiveWithTimeout(Duration timeout) { + return implementation_.template Receive(timeout); + } + + /// Wait the default number of microseconds for a request of one of the + /// provided types to arrive. This tends to be used by servers. + template + requires(sizeof...(Ms) > 0) RequestResult Receive() { + const Duration timeout = default_timeout_; + return implementation_.template Receive(timeout); + } + + /// Send a message in a best-effort fashion. This is used for messaging where + /// responses are not necessarily expected, and for servers to respond to requests. + /// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases. + template + void Send(Address address, uint64_t request_id, M message) { + return implementation_.template Send(address, request_id, std::move(message)); + } + + /// The current system time. This time source should be preferred over any other, + /// because it lets us deterministically control clocks from tests for making + /// things like timeouts deterministic. + Time Now() const { return implementation_.Now(); } + + /// Returns true if the system should shut-down. + bool ShouldShutDown() const { return implementation_.ShouldShutDown(); } + + /// Returns a random number within the specified distribution. + template , class Return = uint64_t> + Return Rand(D distrib) { + return implementation_.template Rand(distrib); + } + + Address GetAddress() { return address_; } +}; +}; // namespace memgraph::io diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 02535dcc7..664c010c8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -10,6 +10,9 @@ add_subdirectory(stress) # concurrent test binaries add_subdirectory(concurrent) +# simulation test binaries +add_subdirectory(simulation) + # manual test binaries add_subdirectory(manual) diff --git a/tests/benchmark/CMakeLists.txt b/tests/benchmark/CMakeLists.txt index 4bf8374b0..31f0eebc0 100644 --- a/tests/benchmark/CMakeLists.txt +++ b/tests/benchmark/CMakeLists.txt @@ -62,3 +62,6 @@ target_link_libraries(${test_prefix}storage_v2_gc mg-storage-v2) add_benchmark(storage_v2_property_store.cpp) target_link_libraries(${test_prefix}storage_v2_property_store mg-storage-v2) + +add_benchmark(future.cpp) +target_link_libraries(${test_prefix}future mg-io) diff --git a/tests/benchmark/future.cpp b/tests/benchmark/future.cpp new file mode 100644 index 000000000..abbe3fb98 --- /dev/null +++ b/tests/benchmark/future.cpp @@ -0,0 +1,30 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +#include "io/future.hpp" + +static void FuturePairFillWait(benchmark::State &state) { + uint64_t counter = 0; + while (state.KeepRunning()) { + auto [future, promise] = memgraph::io::FuturePromisePair(); + promise.Fill(1); + std::move(future).Wait(); + + ++counter; + } + state.SetItemsProcessed(counter); +} + +BENCHMARK(FuturePairFillWait)->Unit(benchmark::kNanosecond)->UseRealTime(); + +BENCHMARK_MAIN(); diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt new file mode 100644 index 000000000..142657401 --- /dev/null +++ b/tests/simulation/CMakeLists.txt @@ -0,0 +1,30 @@ +set(test_prefix memgraph__simulation__) + +find_package(gflags) + +add_custom_target(memgraph__simulation) + +function(add_simulation_test test_cpp san) + # get exec name (remove extension from the abs path) + get_filename_component(exec_name ${test_cpp} NAME_WE) + set(target_name ${test_prefix}${exec_name}) + add_executable(${target_name} ${test_cpp}) + + # OUTPUT_NAME sets the real name of a target when it is built and can be + # used to help create two targets of the same name even though CMake + # requires unique logical target names + set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${exec_name}) + target_link_libraries(${target_name} gtest gmock mg-utils mg-io mg-io-simulator) + + # sanitize + target_compile_options(${target_name} PRIVATE -fsanitize=${san}) + target_link_options(${target_name} PRIVATE -fsanitize=${san}) + + # register test + add_test(${target_name} ${exec_name}) + add_dependencies(memgraph__simulation ${target_name}) +endfunction(add_simulation_test) + +add_simulation_test(basic_request.cpp address) + +add_simulation_test(trial_query_storage/query_storage_test.cpp address) diff --git a/tests/simulation/basic_request.cpp b/tests/simulation/basic_request.cpp new file mode 100644 index 000000000..ac3190ad7 --- /dev/null +++ b/tests/simulation/basic_request.cpp @@ -0,0 +1,87 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +#include "io/simulator/simulator.hpp" + +using memgraph::io::Address; +using memgraph::io::Io; +using memgraph::io::ResponseFuture; +using memgraph::io::ResponseResult; +using memgraph::io::simulator::Simulator; +using memgraph::io::simulator::SimulatorConfig; +using memgraph::io::simulator::SimulatorTransport; + +struct CounterRequest { + uint64_t proposal; +}; + +struct CounterResponse { + uint64_t highest_seen; +}; + +void run_server(Io io) { + uint64_t highest_seen = 0; + + while (!io.ShouldShutDown()) { + std::cout << "[SERVER] Is receiving..." << std::endl; + auto request_result = io.Receive(); + if (request_result.HasError()) { + std::cout << "[SERVER] Error, continue" << std::endl; + continue; + } + auto request_envelope = request_result.GetValue(); + auto req = std::get(request_envelope.message); + + highest_seen = std::max(highest_seen, req.proposal); + auto srv_res = CounterResponse{highest_seen}; + + io.Send(request_envelope.from_address, request_envelope.request_id, srv_res); + } +} + +int main() { + auto config = SimulatorConfig{ + .drop_percent = 0, + .perform_timeouts = true, + .scramble_messages = true, + .rng_seed = 0, + }; + auto simulator = Simulator(config); + + auto cli_addr = Address::TestAddress(1); + auto srv_addr = Address::TestAddress(2); + + Io cli_io = simulator.Register(cli_addr); + Io srv_io = simulator.Register(srv_addr); + + auto srv_thread = std::jthread(run_server, std::move(srv_io)); + simulator.IncrementServerCountAndWaitForQuiescentState(srv_addr); + + for (int i = 1; i < 3; ++i) { + // send request + CounterRequest cli_req; + cli_req.proposal = i; + auto res_f = cli_io.Request(srv_addr, cli_req); + auto res_rez = std::move(res_f).Wait(); + if (!res_rez.HasError()) { + std::cout << "[CLIENT] Got a valid response" << std::endl; + auto env = res_rez.GetValue(); + MG_ASSERT(env.message.highest_seen == i); + } else { + std::cout << "[CLIENT] Got an error" << std::endl; + } + } + + simulator.ShutDown(); + return 0; +} diff --git a/tests/simulation/trial_query_storage/messages.hpp b/tests/simulation/trial_query_storage/messages.hpp new file mode 100644 index 000000000..8db78a54c --- /dev/null +++ b/tests/simulation/trial_query_storage/messages.hpp @@ -0,0 +1,34 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +namespace memgraph::tests::simulation { + +struct Vertex { + std::string key; +}; + +struct ScanVerticesRequest { + int64_t count; + std::optional continuation; +}; + +struct VerticesResponse { + std::vector vertices; + std::optional continuation; +}; + +} // namespace memgraph::tests::simulation diff --git a/tests/simulation/trial_query_storage/query_storage_test.cpp b/tests/simulation/trial_query_storage/query_storage_test.cpp new file mode 100644 index 000000000..9cdff4ee6 --- /dev/null +++ b/tests/simulation/trial_query_storage/query_storage_test.cpp @@ -0,0 +1,85 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include + +#include "io/address.hpp" +#include "io/simulator/simulator.hpp" +#include "io/simulator/simulator_config.hpp" +#include "io/simulator/simulator_transport.hpp" +#include "io/transport.hpp" + +#include "messages.hpp" + +namespace memgraph::tests::simulation { +using memgraph::io::Io; +using memgraph::io::simulator::SimulatorTransport; + +void run_server(Io io) { + while (!io.ShouldShutDown()) { + std::cout << "[STORAGE] Is receiving..." << std::endl; + auto request_result = io.Receive(); + if (request_result.HasError()) { + std::cout << "[STORAGE] Error, continue" << std::endl; + continue; + } + auto request_envelope = request_result.GetValue(); + auto req = std::get(request_envelope.message); + + VerticesResponse response{}; + const int64_t start_index = std::invoke([&req] { + if (req.continuation.has_value()) { + return *req.continuation; + } + return 0L; + }); + for (auto index = start_index; index < start_index + req.count; ++index) { + response.vertices.push_back({std::string("Vertex_") + std::to_string(index)}); + } + io.Send(request_envelope.from_address, request_envelope.request_id, response); + } +} + +} // namespace memgraph::tests::simulation + +int main() { + using memgraph::io::Address; + using memgraph::io::Io; + using memgraph::io::simulator::Simulator; + using memgraph::io::simulator::SimulatorConfig; + using memgraph::io::simulator::SimulatorTransport; + using memgraph::tests::simulation::run_server; + using memgraph::tests::simulation::ScanVerticesRequest; + using memgraph::tests::simulation::VerticesResponse; + auto config = SimulatorConfig{ + .drop_percent = 0, + .perform_timeouts = true, + .scramble_messages = true, + .rng_seed = 0, + }; + auto simulator = Simulator(config); + + auto cli_addr = Address::TestAddress(1); + auto srv_addr = Address::TestAddress(2); + + Io cli_io = simulator.Register(cli_addr); + Io srv_io = simulator.Register(srv_addr); + + auto srv_thread = std::jthread(run_server, std::move(srv_io)); + simulator.IncrementServerCountAndWaitForQuiescentState(srv_addr); + + auto req = ScanVerticesRequest{2, std::nullopt}; + + auto res_f = cli_io.Request(srv_addr, req); + auto res_rez = std::move(res_f).Wait(); + simulator.ShutDown(); + return 0; +} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 55fb7b01b..9de4860ef 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -396,3 +396,7 @@ find_package(Boost REQUIRED) add_unit_test(websocket.cpp) target_link_libraries(${test_prefix}websocket mg-communication Boost::headers) + +# Test future +add_unit_test(future.cpp) +target_link_libraries(${test_prefix}future mg-io) \ No newline at end of file diff --git a/tests/unit/future.cpp b/tests/unit/future.cpp new file mode 100644 index 000000000..490e19bbc --- /dev/null +++ b/tests/unit/future.cpp @@ -0,0 +1,55 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include + +#include "gtest/gtest.h" + +#include "io/future.hpp" + +using namespace memgraph::io; + +void Fill(Promise promise_1) { promise_1.Fill("success"); } + +void Wait(Future future_1, Promise promise_2) { + std::string result_1 = std::move(future_1).Wait(); + EXPECT_TRUE(result_1 == "success"); + promise_2.Fill("it worked"); +} + +TEST(Future, BasicLifecycle) { + std::atomic_bool waiting = false; + + std::function notifier = [&] { + waiting.store(true, std::memory_order_seq_cst); + return false; + }; + + auto [future_1, promise_1] = FuturePromisePairWithNotifier(notifier); + auto [future_2, promise_2] = FuturePromisePair(); + + std::jthread t1(Wait, std::move(future_1), std::move(promise_2)); + + // spin in a loop until the promise signals + // that it is waiting + while (!waiting.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + + std::jthread t2(Fill, std::move(promise_1)); + + t1.join(); + t2.join(); + + std::string result_2 = std::move(future_2).Wait(); + EXPECT_TRUE(result_2 == "it worked"); +} From 5e4733ac98da8d8c2e6da2305df018dedb2bbf23 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Fri, 12 Aug 2022 13:20:11 +0000 Subject: [PATCH 02/39] Add extra safety checks to commit_index management --- src/io/rsm/raft.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index dc1a1e9b3..02cc89604 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -250,7 +250,7 @@ class Raft { size_t new_committed_log_size = indices[(indices.size() / 2)]; - state_.committed_log_size = new_committed_log_size; + state_.committed_log_size = std::max(state_.committed_log_size, new_committed_log_size); // For each index between the old index and the new one (inclusive), // Apply that log's WriteOperation to our replicated_state_, @@ -664,6 +664,7 @@ class Raft { state_.log.insert(state_.log.end(), req.entries.begin(), req.entries.end()); + MG_ASSERT(req.leader_commit >= state_.committed_log_size); state_.committed_log_size = std::min(req.leader_commit, LastLogIndex()); for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) { From 1423da5f5147d25fe006ab84fb7696fe32ec8490 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Mon, 15 Aug 2022 11:57:19 +0000 Subject: [PATCH 03/39] Improve comments around majority index selection --- src/io/rsm/raft.hpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 02cc89604..b8eead191 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -237,9 +237,16 @@ class Raft { // When the entry has been safely replicated, the leader applies the // entry to its state machine and returns the result of that // execution to the client. + // + // "Safely replicated" is defined as being known to be present + // on at least a majority of all peers (inclusive of the Leader). void BumpCommitIndexAndReplyToClients(Leader &leader) { - // set the current committed_log_size based on the - auto indices = std::vector{state_.log.size()}; + auto indices = std::vector{}; + + // We include our own log size in the calculation of the log + // index that is present on at least a majority of all peers. + indices.push_back(state_.log.size()); + for (const auto &[addr, f] : leader.followers) { indices.push_back(f.confirmed_contiguous_index); Log("at port ", addr.last_known_port, " has confirmed contiguous index of: ", f.confirmed_contiguous_index); @@ -248,9 +255,25 @@ class Raft { // reverse sort from highest to lowest (using std::ranges::greater) std::ranges::sort(indices, std::ranges::greater()); - size_t new_committed_log_size = indices[(indices.size() / 2)]; + // This is a particularly correctness-critical calculation because it + // determines which index we will consider to be the committed index. + // + // If the following indexes are recorded for clusters of different sizes, + // these are the expected indexes that are considered to have reached + // consensus: + // state | expected value | (indices.size() / 2) + // [1] 1 (1 / 2) => 0 + // [2, 1] 1 (2 / 2) => 1 + // [3, 2, 1] 2 (3 / 2) => 1 + // [4, 3, 2, 1] 2 (4 / 2) => 2 + // [5, 4, 3, 2, 1] 3 (5 / 2) => 2 + size_t index_present_on_majority = indices.size() / 2; + LogIndex new_committed_log_size = indices[index_present_on_majority]; - state_.committed_log_size = std::max(state_.committed_log_size, new_committed_log_size); + // We never go backwards in history. + MG_ASSERT(state_.committed_log_size <= new_committed_log_size); + + state_.committed_log_size = new_committed_log_size; // For each index between the old index and the new one (inclusive), // Apply that log's WriteOperation to our replicated_state_, From 6f906c0488b6fae123bc9d8350b9a5a10ad9ad6b Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 14:44:17 +0000 Subject: [PATCH 04/39] Use spdlog instead of cout --- src/io/rsm/raft.hpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index b8eead191..072ba7f2b 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -135,7 +135,7 @@ struct Leader { std::unordered_map pending_client_requests; Time last_broadcast = Time::min(); - static void Print() { std::cout << "\tLeader \t"; } + std::string ToString() { return "\tLeader \t"; } }; struct Candidate { @@ -143,14 +143,14 @@ struct Candidate { Time election_began = Time::min(); std::set
outstanding_votes; - static void Print() { std::cout << "\tCandidate\t"; } + std::string ToString() { return "\tCandidate\t"; } }; struct Follower { Time last_received_append_entries_timestamp; Address leader_address; - static void Print() { std::cout << "\tFollower \t"; } + std::string ToString() { return "\tFollower \t"; } }; using Role = std::variant; @@ -391,11 +391,17 @@ class Raft { const Term term = state_.term; - std::cout << '\t' << micros << "\t" << term << "\t" << io_.GetAddress().last_known_port; + std::ostringstream out; - std::visit([&](auto &&role) { role.Print(); }, role_); + out << '\t' << (int)micros << "\t" << term << "\t" << io_.GetAddress().last_known_port; - (std::cout << ... << args) << std::endl; + std::string role_string = std::visit([&](auto &&role) { return role.ToString(); }, role_); + + out << role_string; + + (out << ... << args); + + spdlog::debug(out.str()); } ///////////////////////////////////////////////////////////// From 996165910310435af1153add92d4cf709323cd20 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 14:52:03 +0000 Subject: [PATCH 05/39] Use spdlog instead of cout --- tests/simulation/raft.cpp | 50 ++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index b7e6af5ee..1e5bffb21 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -162,7 +162,7 @@ void RunSimulation() { auto srv_thread_3 = std::jthread(RunRaft, std::move(srv_3)); simulator.IncrementServerCountAndWaitForQuiescentState(srv_addr_3); - std::cout << "beginning test after servers have become quiescent" << std::endl; + spdlog::debug("beginning test after servers have become quiescent"); std::mt19937 cli_rng_{0}; Address server_addrs[]{srv_addr_1, srv_addr_2, srv_addr_3}; @@ -185,7 +185,7 @@ void RunSimulation() { WriteRequest cli_req; cli_req.operation = cas_req; - std::cout << "client sending CasRequest to Leader " << leader.last_known_port << std::endl; + spdlog::debug("client sending CasRequest to Leader {} ", leader.last_known_port); ResponseFuture> cas_response_future = cli_io.Request, WriteResponse>(leader, cli_req); @@ -193,7 +193,8 @@ void RunSimulation() { ResponseResult> cas_response_result = std::move(cas_response_future).Wait(); if (cas_response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; + spdlog::debug("client timed out while trying to communicate with assumed Leader server {}", + leader.last_known_port); continue; } @@ -203,14 +204,14 @@ void RunSimulation() { if (write_cas_response.retry_leader) { MG_ASSERT(!write_cas_response.success, "retry_leader should never be set for successful responses"); leader = write_cas_response.retry_leader.value(); - std::cout << "client redirected to leader server " << leader.last_known_port << std::endl; + spdlog::debug("client redirected to leader server {}", leader.last_known_port); } else if (!write_cas_response.success) { std::uniform_int_distribution addr_distrib(0, 2); size_t addr_index = addr_distrib(cli_rng_); leader = server_addrs[addr_index]; - std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index - << " with port " << leader.last_known_port << std::endl; + spdlog::debug("client NOT redirected to leader server, trying a random one at index {} with port {}", addr_index, + leader.last_known_port); continue; } @@ -218,8 +219,8 @@ void RunSimulation() { bool cas_succeeded = cas_response.cas_success; - std::cout << "Client received CasResponse! success: " << cas_succeeded - << " last_known_value: " << (int)*last_known_value << std::endl; + spdlog::debug("Client received CasResponse! success: {} last_known_value {}", cas_succeeded, + (int)*last_known_value); if (cas_succeeded) { last_known_value = i; @@ -234,7 +235,8 @@ void RunSimulation() { ReadRequest read_req; read_req.operation = get_req; - std::cout << "client sending GetRequest to Leader " << leader.last_known_port << std::endl; + spdlog::debug("client sending GetRequest to Leader {}", leader.last_known_port); + ResponseFuture> get_response_future = cli_io.Request, ReadResponse>(leader, read_req); @@ -242,7 +244,7 @@ void RunSimulation() { ResponseResult> get_response_result = std::move(get_response_future).Wait(); if (get_response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; + spdlog::debug("client timed out while trying to communicate with Leader server {}", leader.last_known_port); continue; } @@ -257,21 +259,21 @@ void RunSimulation() { if (read_get_response.retry_leader) { MG_ASSERT(!read_get_response.success, "retry_leader should never be set for successful responses"); leader = read_get_response.retry_leader.value(); - std::cout << "client redirected to leader server " << leader.last_known_port << std::endl; + spdlog::debug("client redirected to Leader server {}", leader.last_known_port); } else if (!read_get_response.success) { std::uniform_int_distribution addr_distrib(0, 2); size_t addr_index = addr_distrib(cli_rng_); leader = server_addrs[addr_index]; - std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index - << " with port " << leader.last_known_port << std::endl; + spdlog::debug("client NOT redirected to leader server, trying a random one at index {} with port {}", addr_index, + leader.last_known_port); } GetResponse get_response = read_get_response.read_return; MG_ASSERT(get_response.value == i); - std::cout << "client successfully cas'd a value and read it back! value: " << i << std::endl; + spdlog::debug("client successfully cas'd a value and read it back! value: {}", i); success = true; } @@ -282,14 +284,14 @@ void RunSimulation() { SimulatorStats stats = simulator.Stats(); - std::cout << "total messages: " << stats.total_messages << std::endl; - std::cout << "dropped messages: " << stats.dropped_messages << std::endl; - std::cout << "timed out requests: " << stats.timed_out_requests << std::endl; - std::cout << "total requests: " << stats.total_requests << std::endl; - std::cout << "total responses: " << stats.total_responses << std::endl; - std::cout << "simulator ticks: " << stats.simulator_ticks << std::endl; + spdlog::debug("total messages: ", stats.total_messages); + spdlog::debug("dropped messages: ", stats.dropped_messages); + spdlog::debug("timed out requests: ", stats.timed_out_requests); + spdlog::debug("total requests: ", stats.total_requests); + spdlog::debug("total responses: ", stats.total_responses); + spdlog::debug("simulator ticks: ", stats.simulator_ticks); - std::cout << "========================== SUCCESS :) ==========================" << std::endl; + spdlog::debug("========================== SUCCESS :) =========================="); /* this is implicit in jthread's dtor @@ -303,12 +305,12 @@ int main() { int n_tests = 50; for (int i = 0; i < n_tests; i++) { - std::cout << "========================== NEW SIMULATION " << i << " ==========================" << std::endl; - std::cout << "\tTime\tTerm\tPort\tRole\t\tMessage\n"; + spdlog::debug("========================== NEW SIMULATION {} ==========================", i); + spdlog::debug("\tTime\tTerm\tPort\tRole\t\tMessage\n"); RunSimulation(); } - std::cout << "passed " << n_tests << " tests!" << std::endl; + spdlog::debug("passed {} tests!", n_tests); return 0; } From bb44aaa18823907aff04345ed3d90807729d965a Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:09:51 +0200 Subject: [PATCH 06/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 072ba7f2b..6a094a45f 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -186,7 +186,7 @@ concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) /// ReplicatedState the high-level data structure that is managed by the raft-backed replicated state machine /// WriteOperation the individual operation type that is applied to the ReplicatedState in identical order /// across each replica -/// WriteResponseValue the return value of calling ReplicatedState::write(WriteOperation), which is executed in +/// WriteResponseValue the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in /// identical order across all replicas after an WriteOperation reaches consensus. /// ReadOperation the type of operations that do not require consensus before executing directly /// on a const ReplicatedState & From 7ad9b629689922b48552962475a270b98413a593 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:10:37 +0200 Subject: [PATCH 07/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 6a094a45f..2057428aa 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -289,7 +289,7 @@ class Raft { WriteResponse resp; resp.success = true; - resp.write_return = write_return; + resp.write_return = std::move(write_return); io_.Send(client_request.address, client_request.request_id, std::move(resp)); leader.pending_client_requests.erase(apply_index); From c7d96ed5c5c7c61808d6ae53848b2695d0f2cb70 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:11:11 +0200 Subject: [PATCH 08/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 2057428aa..5373cf954 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -325,7 +325,7 @@ class Raft { }; // request_id not necessary to set because it's not a Future-backed Request. - const RequestId request_id = 0; + static constexpr RequestId request_id = 0; io_.Send(address, request_id, ar); } From dcc5ec920a328e60306af4f509bf39ce532f1229 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:11:41 +0200 Subject: [PATCH 09/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 5373cf954..6a2cb8127 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -334,10 +334,7 @@ class Raft { // Raft paper - 5.2 // Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly Duration RandomTimeout(Duration min, Duration max) { - auto min_micros = std::chrono::duration_cast(min).count(); - auto max_micros = std::chrono::duration_cast(max).count(); - - std::uniform_int_distribution time_distrib(min_micros, max_micros); + std::uniform_int_distribution time_distrib(min.count(), max.count()); auto rand_micros = io_.Rand(time_distrib); From 47186cab189a4bfd1779477f9be55df04c9de616 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:11:56 +0200 Subject: [PATCH 10/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 6a2cb8127..451b1dbd2 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -349,7 +349,7 @@ class Raft { return std::chrono::microseconds{rand_micros}; } - Term PreviousTermFromIndex(LogIndex index) { + Term PreviousTermFromIndex(LogIndex index) const { if (index == 0 || state_.log.size() + 1 <= index) { return 0; } From 2e9cf8f37dff80c502343f89170adad23ee11469 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:12:13 +0200 Subject: [PATCH 11/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 451b1dbd2..5fc6c0f67 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -354,7 +354,7 @@ class Raft { return 0; } - auto &[term, data] = state_.log.at(index - 1); + const auto &[term, data] = state_.log.at(index - 1); return term; } From c256dce6013635dd6a240a5f75b456ee88bb61ff Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:13:03 +0200 Subject: [PATCH 12/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index 5fc6c0f67..e4c2fc7fb 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -372,15 +372,16 @@ class Raft { LogIndex LastLogIndex() { return state_.log.size(); } - Term LastLogTerm() { + Term LastLogTerm() const { if (state_.log.empty()) { return 0; } - auto &[term, data] = state_.log.back(); + const auto &[term, data] = state_.log.back(); return term; } + template void Log(Ts &&...args) { const Time now = io_.Now(); From 342611691a4b967180658be35807bd09e2139f14 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:13:30 +0200 Subject: [PATCH 13/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index e4c2fc7fb..ba4fc7ec0 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -431,7 +431,7 @@ class Raft { std::optional Cron(Candidate &candidate) { const auto now = io_.Now(); const Duration election_timeout = RandomTimeout(100000, 200000); - auto election_timeout_us = std::chrono::duration_cast(election_timeout).count(); + const auto election_timeout_us = std::chrono::duration_cast(election_timeout).count(); if (now - candidate.election_began > election_timeout) { state_.term++; From 6169eb221b700c643e32cc1d421cb5c42082a2aa Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 17:13:41 +0200 Subject: [PATCH 14/39] Update src/io/rsm/raft.hpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: János Benjamin Antal --- src/io/rsm/raft.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index ba4fc7ec0..34b9ac080 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -448,7 +448,7 @@ class Raft { for (const auto &peer : peers_) { // request_id not necessary to set because it's not a Future-backed Request. - auto request_id = 0; + static constexpr auto request_id = 0; io_.template Send(peer, request_id, request); outstanding_votes.insert(peer); } From 79539d13c9a6c24e5da954cfbff9dd92810ba1c2 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 09:44:05 +0000 Subject: [PATCH 15/39] Use Read/Apply instead of read/apply in Rsm concept --- src/io/rsm/raft.hpp | 4 ++-- tests/simulation/raft.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index d3b9de49d..cde9016c2 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -187,8 +187,8 @@ concept Rsm = requires(T t, Write w) template concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) { - { state.read(r) } -> std::same_as; - { state.apply(w) } -> std::same_as; + { state.Read(r) } -> std::same_as; + { state.Apply(w) } -> std::same_as; }; /// Parameter Purpose diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index 7cf6144fd..9182fbc8e 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -60,7 +60,7 @@ class TestState { std::map state_; public: - GetResponse read(GetRequest request) { + GetResponse Read(GetRequest request) { GetResponse ret; if (state_.contains(request.key)) { ret.value = state_[request.key]; @@ -68,7 +68,7 @@ class TestState { return ret; } - CasResponse apply(CasRequest request) { + CasResponse Apply(CasRequest request) { CasResponse ret; // Key exist From e3dd4048654116680bb9940d1e4e15880c602d78 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 10:55:05 +0000 Subject: [PATCH 16/39] Add new test, start to fill out coordinator RSM --- src/coordinator/coordinator.hpp | 79 ++++++++++++++++-------- src/coordinator/hybrid_logical_clock.hpp | 4 ++ src/coordinator/shard_map.hpp | 3 +- tests/simulation/CMakeLists.txt | 2 + tests/simulation/sharded_map.cpp | 78 +++++++++++++++++++++++ 5 files changed, 140 insertions(+), 26 deletions(-) create mode 100644 tests/simulation/sharded_map.cpp diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 48d637d65..1010e0900 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -20,9 +20,28 @@ namespace memgraph::coordinator { using Address = memgraph::io::Address; -using Io = memgraph::io::Io; using SimT = memgraph::io::simulator::SimulatorTransport; +struct HlcRequest { + Hlc last_shard_map_version; +}; + +struct HlcResponse { + Hlc new_hlc; + std::optional fresher_shard_map; +}; + +struct AllocateHlcBatchRequest { + Hlc low; + Hlc high; +}; + +struct AllocateHlcBatchResponse { + bool success; + Hlc low; + Hlc high; +}; + struct SplitShardRequest { Hlc previous_shard_map_version; Label label; @@ -49,43 +68,53 @@ struct DeregisterStorageEngineResponse { bool success; }; -struct HlcRequest { - Hlc last_shard_map_version; -}; +using WriteRequests = std::variant; +using WriteResponses = std::variant; -struct HlcResponse { - Hlc new_hlc; - std::optional fresher_shard_map; -}; +using ReadRequests = std::variant; +using ReadResponses = std::variant; class Coordinator { ShardMap shard_map_; - Io io_; - /// This splits the shard - void Handle(SplitShardRequest &split_shard_request, Address from_addr) { + WriteResponses Apply(AllocateHlcBatchRequest &&ahr) { + AllocateHlcBatchResponse res{}; + + return res; + } + + /// This splits the shard immediately beneath the provided + /// split key, keeping the assigned peers identical for now, + /// but letting them be gradually migrated over time. + WriteResponses Apply(SplitShardRequest &&split_shard_request) { + SplitShardResponse res{}; + if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) { // TODO reply with failure } + + return res; } - void Handle(RegisterStorageEngineRequest ®ister_storage_engine_request, Address from_addr) {} + WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { + RegisterStorageEngineResponse res{}; + + return res; + } + + WriteResponses Apply(DeregisterStorageEngineRequest &®ister_storage_engine_request) { + DeregisterStorageEngineResponse res{}; + + return res; + } public: - void Run() { - while (!io_.ShouldShutDown()) { - std::cout << "[Coordinator] Is receiving..." << std::endl; - auto request_result = - io_.Receive(); - if (request_result.HasError()) { - std::cout << "[Coordinator] Error, continue" << std::endl; - continue; - } + ReadResponses Read(ReadRequests requests) { return HlcResponse{}; } - auto request_envelope = request_result.GetValue(); - // TODO std::visit to determine whether to handle shard split, registration etc... (see raft.hpp Run / Handle - // methods in T0941) - } + WriteResponses Apply(WriteRequests requests) { + return std::visit([&](auto &&requests) { return Apply(requests); }, std::move(requests)); } }; diff --git a/src/coordinator/hybrid_logical_clock.hpp b/src/coordinator/hybrid_logical_clock.hpp index 484d2a7ab..3b65e4947 100644 --- a/src/coordinator/hybrid_logical_clock.hpp +++ b/src/coordinator/hybrid_logical_clock.hpp @@ -21,6 +21,10 @@ using Time = memgraph::io::Time; struct Hlc { uint64_t logical_id; Time coordinator_wall_clock; + + bool operator==(const Hlc &other) const { + return (logical_id == other.logical_id) && (coordinator_wall_clock == other.coordinator_wall_clock); + } }; } // namespace memgraph::coordinator diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index bceca3abd..795bdc1d2 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -16,6 +16,7 @@ #include "coordinator/hybrid_logical_clock.hpp" #include "io/address.hpp" +#include "storage/v3/property_value.hpp" namespace memgraph::coordinator { @@ -32,7 +33,7 @@ struct AddressAndStatus { Status status; }; -using CompoundKey = std::vector; +using CompoundKey = std::vector; using Shard = std::vector; using Shards = std::map; diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index 67f073418..80117eb67 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -30,3 +30,5 @@ add_simulation_test(future.cpp thread) add_simulation_test(basic_request.cpp address) add_simulation_test(trial_query_storage/query_storage_test.cpp address) + +add_simulation_test(sharded_map.cpp address) diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp new file mode 100644 index 000000000..d7376ca21 --- /dev/null +++ b/tests/simulation/sharded_map.cpp @@ -0,0 +1,78 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/address.hpp" +#include "io/rsm/coordinator_rsm.hpp" +#include "io/rsm/raft.hpp" +#include "io/simulator/simulator.hpp" +#include "io/simulator/simulator_transport.hpp" + +using memgraph::coordinator::Coordinator; +using memgraph::coordinator::CoordinatorRsm; +using memgraph::io::Address; +using memgraph::io::Io; +using memgraph::io::ResponseEnvelope; +using memgraph::io::ResponseFuture; +using memgraph::io::ResponseResult; +using memgraph::io::rsm::Raft; +using memgraph::io::rsm::ReadRequest; +using memgraph::io::rsm::ReadResponse; +using memgraph::io::rsm::WriteRequest; +using memgraph::io::rsm::WriteResponse; +using memgraph::io::simulator::Simulator; +using memgraph::io::simulator::SimulatorConfig; +using memgraph::io::simulator::SimulatorStats; +using memgraph::io::simulator::SimulatorTransport; + +int main() { + SimulatorConfig config{ + /* + .drop_percent = 5, + .perform_timeouts = true, + .scramble_messages = true, + .rng_seed = 0, + .start_time = 256 * 1024, + .abort_time = std::chrono::microseconds{8 * 1024 * 1024}, + */ + }; + + auto simulator = Simulator(config); + + auto cli_addr = Address::TestAddress(1); + auto srv_addr_1 = Address::TestAddress(2); + auto srv_addr_2 = Address::TestAddress(3); + auto srv_addr_3 = Address::TestAddress(4); + + Io cli_io = simulator.Register(cli_addr); + Io srv_io_1 = simulator.Register(srv_addr_1); + Io srv_io_2 = simulator.Register(srv_addr_2); + Io srv_io_3 = simulator.Register(srv_addr_3); + + std::vector
srv_1_peers = {srv_addr_2, srv_addr_3}; + std::vector
srv_2_peers = {srv_addr_1, srv_addr_3}; + std::vector
srv_3_peers = {srv_addr_1, srv_addr_2}; + + using ConcreteCoordinatorRsm = CoordinatorRsm; + ConcreteCoordinatorRsm srv_1{std::move(srv_io_1), srv_1_peers, Coordinator{}}; + ConcreteCoordinatorRsm srv_2{std::move(srv_io_2), srv_2_peers, Coordinator{}}; + ConcreteCoordinatorRsm srv_3{std::move(srv_io_3), srv_3_peers, Coordinator{}}; + + return 0; +} From 6b9311e0b8d7b99429fd4fb97de7cc3c26bd57d2 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 12:04:27 +0000 Subject: [PATCH 17/39] Bump shard_rsm to use new method names --- src/coordinator/coordinator.hpp | 10 +++++++++- src/io/rsm/shard_rsm.hpp | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 1010e0900..5dfd7bf03 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -79,6 +79,12 @@ using ReadResponses = std::variant; class Coordinator { ShardMap shard_map_; + ReadResponses Read(HlcRequest &&hlc_request) { + HlcResponse res{}; + + return res; + } + WriteResponses Apply(AllocateHlcBatchRequest &&ahr) { AllocateHlcBatchResponse res{}; @@ -111,7 +117,9 @@ class Coordinator { } public: - ReadResponses Read(ReadRequests requests) { return HlcResponse{}; } + ReadResponses Read(ReadRequests requests) { + return std::visit([&](auto &&requests) { return Read(requests); }, std::move(requests)); + } WriteResponses Apply(WriteRequests requests) { return std::visit([&](auto &&requests) { return Apply(requests); }, std::move(requests)); diff --git a/src/io/rsm/shard_rsm.hpp b/src/io/rsm/shard_rsm.hpp index a2b82dc62..8a7f032a9 100644 --- a/src/io/rsm/shard_rsm.hpp +++ b/src/io/rsm/shard_rsm.hpp @@ -41,7 +41,7 @@ using memgraph::io::simulator::SimulatorStats; using memgraph::io::simulator::SimulatorTransport; using memgraph::storage::PropertyValue; -namespace memgraph::tests::simulation { +namespace memgraph::io::rsm { using ShardRsmKey = std::vector; @@ -69,7 +69,7 @@ class StorageRsm { std::optional maximum_key_{std::nullopt}; public: - StorageGetResponse read(StorageGetRequest request) { + StorageGetResponse Read(StorageGetRequest request) { StorageGetResponse ret; if (state_.contains(request.key)) { ret.value = state_[request.key]; @@ -77,7 +77,7 @@ class StorageRsm { return ret; } - StorageWriteResponse apply(StorageWriteRequest request) { + StorageWriteResponse Apply(StorageWriteRequest request) { StorageWriteResponse ret; // Key exist @@ -121,4 +121,4 @@ class StorageRsm { } }; -} // namespace memgraph::tests::simulation +} // namespace memgraph::io::rsm From c62b0eff938ac59343ee5854a5a267e3c53afff1 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 12:05:25 +0000 Subject: [PATCH 18/39] Add Simulator::RegisterNew helper method --- src/io/simulator/simulator.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/io/simulator/simulator.hpp b/src/io/simulator/simulator.hpp index 8e3ad85c0..6bad9239e 100644 --- a/src/io/simulator/simulator.hpp +++ b/src/io/simulator/simulator.hpp @@ -23,6 +23,7 @@ namespace memgraph::io::simulator { class Simulator { std::mt19937 rng_; std::shared_ptr simulator_handle_; + uint16_t auto_port_ = 0; public: explicit Simulator(SimulatorConfig config) @@ -30,6 +31,11 @@ class Simulator { void ShutDown() { simulator_handle_->ShutDown(); } + Io RegisterNew() { + Address address = Address::TestAddress(auto_port_++); + return Register(address); + } + Io Register(Address address) { std::uniform_int_distribution seed_distrib; uint64_t seed = seed_distrib(rng_); From 502a9b48236f988f237c1d8f712e86786e90f821 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 12:06:24 +0000 Subject: [PATCH 19/39] Check-in coordinator_rsm.hpp --- src/io/rsm/coordinator_rsm.hpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/io/rsm/coordinator_rsm.hpp diff --git a/src/io/rsm/coordinator_rsm.hpp b/src/io/rsm/coordinator_rsm.hpp new file mode 100644 index 000000000..b9f098772 --- /dev/null +++ b/src/io/rsm/coordinator_rsm.hpp @@ -0,0 +1,25 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include "coordinator/coordinator.hpp" +#include "io/rsm/raft.hpp" + +namespace memgraph::io::rsm { + +// TODO(tyler) don't +using namespace memgraph::coordinator; + +template +using CoordinatorRsm = io::rsm::Raft; + +} // namespace memgraph::io::rsm From 95e90e6c2e622b994ecacb7d55450f40686fecda Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 12:07:00 +0000 Subject: [PATCH 20/39] Check-in in-progress shard test --- tests/simulation/sharded_map.cpp | 87 +++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 17 deletions(-) diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index d7376ca21..19f6de4b9 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -21,19 +21,24 @@ #include "io/address.hpp" #include "io/rsm/coordinator_rsm.hpp" #include "io/rsm/raft.hpp" +#include "io/rsm/shard_rsm.hpp" #include "io/simulator/simulator.hpp" #include "io/simulator/simulator_transport.hpp" using memgraph::coordinator::Coordinator; -using memgraph::coordinator::CoordinatorRsm; using memgraph::io::Address; using memgraph::io::Io; using memgraph::io::ResponseEnvelope; using memgraph::io::ResponseFuture; -using memgraph::io::ResponseResult; +using memgraph::io::rsm::CoordinatorRsm; using memgraph::io::rsm::Raft; using memgraph::io::rsm::ReadRequest; using memgraph::io::rsm::ReadResponse; +using memgraph::io::rsm::StorageGetRequest; +using memgraph::io::rsm::StorageGetResponse; +using memgraph::io::rsm::StorageRsm; +using memgraph::io::rsm::StorageWriteRequest; +using memgraph::io::rsm::StorageWriteResponse; using memgraph::io::rsm::WriteRequest; using memgraph::io::rsm::WriteResponse; using memgraph::io::simulator::Simulator; @@ -41,6 +46,10 @@ using memgraph::io::simulator::SimulatorConfig; using memgraph::io::simulator::SimulatorStats; using memgraph::io::simulator::SimulatorTransport; +using ConcreteCoordinatorRsm = CoordinatorRsm; +using ConcreteStorageRsm = Raft; + int main() { SimulatorConfig config{ /* @@ -55,24 +64,68 @@ int main() { auto simulator = Simulator(config); - auto cli_addr = Address::TestAddress(1); - auto srv_addr_1 = Address::TestAddress(2); - auto srv_addr_2 = Address::TestAddress(3); - auto srv_addr_3 = Address::TestAddress(4); + Io cli_io = simulator.RegisterNew(); - Io cli_io = simulator.Register(cli_addr); - Io srv_io_1 = simulator.Register(srv_addr_1); - Io srv_io_2 = simulator.Register(srv_addr_2); - Io srv_io_3 = simulator.Register(srv_addr_3); + // spin up coordinators - std::vector
srv_1_peers = {srv_addr_2, srv_addr_3}; - std::vector
srv_2_peers = {srv_addr_1, srv_addr_3}; - std::vector
srv_3_peers = {srv_addr_1, srv_addr_2}; + Io c_io_1 = simulator.RegisterNew(); + Io c_io_2 = simulator.RegisterNew(); + Io c_io_3 = simulator.RegisterNew(); - using ConcreteCoordinatorRsm = CoordinatorRsm; - ConcreteCoordinatorRsm srv_1{std::move(srv_io_1), srv_1_peers, Coordinator{}}; - ConcreteCoordinatorRsm srv_2{std::move(srv_io_2), srv_2_peers, Coordinator{}}; - ConcreteCoordinatorRsm srv_3{std::move(srv_io_3), srv_3_peers, Coordinator{}}; + Address c_addrs[] = {c_io_1.GetAddress(), c_io_2.GetAddress(), c_io_3.GetAddress()}; + + std::vector
c_1_peers = {c_addrs[1], c_addrs[2]}; + std::vector
c_2_peers = {c_addrs[0], c_addrs[2]}; + std::vector
c_3_peers = {c_addrs[0], c_addrs[1]}; + + ConcreteCoordinatorRsm c_1{std::move(c_io_1), c_1_peers, Coordinator{}}; + ConcreteCoordinatorRsm c_2{std::move(c_io_2), c_2_peers, Coordinator{}}; + ConcreteCoordinatorRsm c_3{std::move(c_io_3), c_3_peers, Coordinator{}}; + + auto c_thread_1 = std::jthread([c_1]() mutable { c_1.Run(); }); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[0]); + + /* + auto c_thread_2 = std::jthread(RunRaft< Coordinator>, std::move(c_2)); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[1]); + + auto c_thread_3 = std::jthread(RunRaft, std::move(c_3)); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); + */ + + // spin up shard A + + Io a_io_1 = simulator.RegisterNew(); + Io a_io_2 = simulator.RegisterNew(); + Io a_io_3 = simulator.RegisterNew(); + + Address a_addrs[] = {a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress()}; + + std::vector
a_1_peers = {a_addrs[1], a_addrs[2]}; + std::vector
a_2_peers = {a_addrs[0], a_addrs[2]}; + std::vector
a_3_peers = {a_addrs[0], a_addrs[1]}; + + ConcreteStorageRsm a_1{std::move(a_io_1), a_1_peers, StorageRsm{}}; + ConcreteStorageRsm a_2{std::move(a_io_2), a_2_peers, StorageRsm{}}; + ConcreteStorageRsm a_3{std::move(a_io_3), a_3_peers, StorageRsm{}}; + + // spin up shard B + + Io b_io_1 = simulator.RegisterNew(); + Io b_io_2 = simulator.RegisterNew(); + Io b_io_3 = simulator.RegisterNew(); + + Address b_addrs[] = {b_io_1.GetAddress(), b_io_2.GetAddress(), b_io_3.GetAddress()}; + + std::vector
b_1_peers = {b_addrs[1], b_addrs[2]}; + std::vector
b_2_peers = {b_addrs[0], b_addrs[2]}; + std::vector
b_3_peers = {b_addrs[0], b_addrs[1]}; + + ConcreteStorageRsm b_1{std::move(b_io_1), b_1_peers, StorageRsm{}}; + ConcreteStorageRsm b_2{std::move(b_io_2), b_2_peers, StorageRsm{}}; + ConcreteStorageRsm b_3{std::move(b_io_3), b_3_peers, StorageRsm{}}; + + std::cout << "beginning test after servers have become quiescent" << std::endl; return 0; } From 92d69e080ce8382c0d9efdd4d6b89d5113dc096e Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Thu, 4 Aug 2022 14:28:27 +0000 Subject: [PATCH 21/39] Add todos to coordinator.hpp --- src/coordinator/coordinator.hpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 5dfd7bf03..c45dca666 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -78,7 +78,22 @@ using ReadResponses = std::variant; class Coordinator { ShardMap shard_map_; + /// The highest reserved timestamp / highest allocated timestamp + /// is a way for minimizing communication involved in query engines + /// reserving Hlc's for their transaction processing. + /// Periodically, the coordinator will allocate a batch of timestamps + /// and this will need to go over consensus. From that point forward, + /// each timestamp in that batch can be given out to "readers" who issue + /// HlcRequest without blocking on consensus first. But if + /// highest_allocated_timestamp_ approaches highest_reserved_timestamp_, + /// it is time to allocate another batch, so that we can keep guaranteeing + /// forward progress. + /// Any time a coordinator becomes a new leader, it will need to issue + /// a new AllocateHlcBatchRequest to create a pool of IDs to allocate. + uint64_t highest_allocated_timestamp_; + uint64_t highest_reserved_timestamp_; + /// Increment our ReadResponses Read(HlcRequest &&hlc_request) { HlcResponse res{}; @@ -101,18 +116,28 @@ class Coordinator { // TODO reply with failure } + // TODO apply split + return res; } + /// This adds the provided storage engine to the standby storage engine pool, + /// which can be used to rebalance storage over time. WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { RegisterStorageEngineResponse res{}; + // TODO + return res; } + /// This begins the process of draining the provided storage engine from all raft + /// clusters that it might be participating in. WriteResponses Apply(DeregisterStorageEngineRequest &®ister_storage_engine_request) { DeregisterStorageEngineResponse res{}; + // TODO + return res; } From dd46cc407fb50f5a387434222855704cebb31b27 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Fri, 5 Aug 2022 15:20:40 +0200 Subject: [PATCH 22/39] Return an error and the latest known ShardMap version if the requested key is not possibly stored in the given shard --- src/io/rsm/shard_rsm.hpp | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/io/rsm/shard_rsm.hpp b/src/io/rsm/shard_rsm.hpp index 8a7f032a9..9e72ca06f 100644 --- a/src/io/rsm/shard_rsm.hpp +++ b/src/io/rsm/shard_rsm.hpp @@ -19,12 +19,15 @@ #include #include +#include "coordinator/hybrid_logical_clock.hpp" #include "io/address.hpp" #include "io/rsm/raft.hpp" #include "io/simulator/simulator.hpp" #include "io/simulator/simulator_transport.hpp" #include "storage/v2/property_value.hpp" +#include "utils/logging.hpp" +using memgraph::coordinator::Hlc; using memgraph::io::Address; using memgraph::io::Io; using memgraph::io::ResponseEnvelope; @@ -53,6 +56,8 @@ struct StorageWriteRequest { struct StorageWriteResponse { bool shard_rsm_success; std::optional last_value; + // Only has a value if the given shard does not contain the requested key + std::optional latest_known_shard_map_version{std::nullopt}; }; struct StorageGetRequest { @@ -60,19 +65,37 @@ struct StorageGetRequest { }; struct StorageGetResponse { + bool shard_rsm_success; std::optional value; + // Only has a value if the given shard does not contain the requested key + std::optional latest_known_shard_map_version{std::nullopt}; }; class StorageRsm { std::map state_; ShardRsmKey minimum_key_; std::optional maximum_key_{std::nullopt}; + Hlc shard_map_version_; + + // The key is not located in this shard + bool IsKeyInRange(const ShardRsmKey &key) { + MG_ASSERT(maximum_key_); + return (key >= minimum_key_ && key <= maximum_key_); + } public: StorageGetResponse Read(StorageGetRequest request) { StorageGetResponse ret; - if (state_.contains(request.key)) { + + if (IsKeyInRange(request.key)) { + ret.latest_known_shard_map_version = shard_map_version_; + ret.shard_rsm_success = false; + } else if (state_.contains(request.key)) { ret.value = state_[request.key]; + ret.shard_rsm_success = true; + } else { + ret.shard_rsm_success = false; + ret.value = std::nullopt; } return ret; } @@ -80,8 +103,12 @@ class StorageRsm { StorageWriteResponse Apply(StorageWriteRequest request) { StorageWriteResponse ret; + if (IsKeyInRange(request.key)) { + ret.latest_known_shard_map_version = shard_map_version_; + ret.shard_rsm_success = false; + } // Key exist - if (state_.contains(request.key)) { + else if (state_.contains(request.key)) { auto &val = state_[request.key]; /* From edf12932744da3ff023fe947cfb8031e1776126d Mon Sep 17 00:00:00 2001 From: gvolfing Date: Fri, 5 Aug 2022 15:30:04 +0200 Subject: [PATCH 23/39] Add GetShardMap read-only request to coordinator --- src/coordinator/coordinator.hpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index c45dca666..1fcb23fa8 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -31,6 +31,14 @@ struct HlcResponse { std::optional fresher_shard_map; }; +struct GetShardMapRequest { + // No state +}; + +struct GetShardMapResponse { + ShardMap shard_map; +}; + struct AllocateHlcBatchRequest { Hlc low; Hlc high; @@ -73,8 +81,8 @@ using WriteRequests = std::variant; -using ReadRequests = std::variant; -using ReadResponses = std::variant; +using ReadRequests = std::variant; +using ReadResponses = std::variant; class Coordinator { ShardMap shard_map_; @@ -100,6 +108,13 @@ class Coordinator { return res; } + GetShardMapResponse Read(GetShardMapRequest &&get_shard_map_request) { + GetShardMapResponse res; + res.shard_map = shard_map_; + + return res; + } + WriteResponses Apply(AllocateHlcBatchRequest &&ahr) { AllocateHlcBatchResponse res{}; From 5963c83a6072486dd435664bd87e68b855e33b03 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Fri, 5 Aug 2022 16:45:44 +0200 Subject: [PATCH 24/39] Update Coordinator --- src/coordinator/coordinator.hpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 1fcb23fa8..0e3feef1e 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -11,6 +11,8 @@ #pragma once +#include + #include "coordinator/hybrid_logical_clock.hpp" #include "coordinator/shard_map.hpp" #include "io/simulator/simulator.hpp" @@ -85,7 +87,11 @@ using ReadRequests = std::variant; using ReadResponses = std::variant; class Coordinator { + using StandbySotrageEnginePool = std::unordered_set
; + ShardMap shard_map_; + StandbySotrageEnginePool storage_engine_pool_; + /// The highest reserved timestamp / highest allocated timestamp /// is a way for minimizing communication involved in query engines /// reserving Hlc's for their transaction processing. @@ -129,10 +135,11 @@ class Coordinator { if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) { // TODO reply with failure + res.success = false; + } else { + // TODO apply split } - // TODO apply split - return res; } @@ -140,8 +147,10 @@ class Coordinator { /// which can be used to rebalance storage over time. WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { RegisterStorageEngineResponse res{}; - // TODO + const Address &address = register_storage_engine_request.address; + storage_engine_pool_.insert(address); + res.success = true; return res; } @@ -150,8 +159,10 @@ class Coordinator { /// clusters that it might be participating in. WriteResponses Apply(DeregisterStorageEngineRequest &®ister_storage_engine_request) { DeregisterStorageEngineResponse res{}; - // TODO + const Address &address = register_storage_engine_request.address; + storage_engine_pool_.erase(address); + res.success = true; return res; } From 4ed580158867a36ce3d30c29e790f5526977750f Mon Sep 17 00:00:00 2001 From: gvolfing Date: Sun, 7 Aug 2022 20:31:22 +0200 Subject: [PATCH 25/39] Change min and max key related shard logic --- src/io/rsm/shard_rsm.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/io/rsm/shard_rsm.hpp b/src/io/rsm/shard_rsm.hpp index 9e72ca06f..0be4a1b12 100644 --- a/src/io/rsm/shard_rsm.hpp +++ b/src/io/rsm/shard_rsm.hpp @@ -79,8 +79,10 @@ class StorageRsm { // The key is not located in this shard bool IsKeyInRange(const ShardRsmKey &key) { - MG_ASSERT(maximum_key_); - return (key >= minimum_key_ && key <= maximum_key_); + if (maximum_key_) [[likely]] { + return (key >= minimum_key_ && key <= maximum_key_); + } + return key >= minimum_key_; } public: @@ -103,6 +105,7 @@ class StorageRsm { StorageWriteResponse Apply(StorageWriteRequest request) { StorageWriteResponse ret; + // Key is outside the prohibited range if (IsKeyInRange(request.key)) { ret.latest_known_shard_map_version = shard_map_version_; ret.shard_rsm_success = false; From c7282e89359f79178b6b50a86ff36eddf1cebdea Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 08:56:03 +0200 Subject: [PATCH 26/39] Temporarly remove the stanby server pool --- src/coordinator/coordinator.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 0e3feef1e..9f87210e8 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -90,7 +90,7 @@ class Coordinator { using StandbySotrageEnginePool = std::unordered_set
; ShardMap shard_map_; - StandbySotrageEnginePool storage_engine_pool_; + // StandbySotrageEnginePool storage_engine_pool_; /// The highest reserved timestamp / highest allocated timestamp /// is a way for minimizing communication involved in query engines @@ -148,9 +148,9 @@ class Coordinator { WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { RegisterStorageEngineResponse res{}; // TODO - const Address &address = register_storage_engine_request.address; - storage_engine_pool_.insert(address); - res.success = true; + // const Address &address = register_storage_engine_request.address; + // storage_engine_pool_.insert(address); + // res.success = true; return res; } @@ -160,9 +160,9 @@ class Coordinator { WriteResponses Apply(DeregisterStorageEngineRequest &®ister_storage_engine_request) { DeregisterStorageEngineResponse res{}; // TODO - const Address &address = register_storage_engine_request.address; - storage_engine_pool_.erase(address); - res.success = true; + // const Address &address = register_storage_engine_request.address; + // storage_engine_pool_.erase(address); + // res.success = true; return res; } From 7af917e408362befe0a219550e728c22d4d46c0a Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 13:31:28 +0200 Subject: [PATCH 27/39] Add abstraction for RsmClient --- tests/simulation/CMakeLists.txt | 2 +- tests/simulation/raft.cpp | 166 +++++++++++++++++++------------- 2 files changed, 102 insertions(+), 66 deletions(-) diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index b44dacb36..15c426e8c 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -31,4 +31,4 @@ add_simulation_test(raft.cpp address) add_simulation_test(trial_query_storage/query_storage_test.cpp address) -add_simulation_test(sharded_map.cpp address) +#add_simulation_test(sharded_map.cpp address) diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index b7e6af5ee..29f0a97dd 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -116,6 +116,91 @@ class TestState { } }; +template +class RsmClient { + using ServerPool = std::vector
; + + IoImpl io_; + Address leader_; + + std::mt19937 cli_rng_{0}; + ServerPool server_addrs_; + + template + std::optional CheckForCorrectLeader(ResponseT response) { + if (response.retry_leader) { + MG_ASSERT(!response.success, "retry_leader should never be set for successful responses"); + leader_ = response.retry_leader.value(); + std::cout << "client redirected to leader server " << leader_.last_known_port << std::endl; + } else if (!response.success) { + std::uniform_int_distribution addr_distrib(0, (server_addrs_.size() - 1)); + size_t addr_index = addr_distrib(cli_rng_); + leader_ = server_addrs_[addr_index]; + + std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index + << " with port " << leader_.last_known_port << std::endl; + return {}; + } + + return response; + } + + public: + RsmClient(IoImpl &&io, Address &&leader, ServerPool &&server_addrs) + : io_{io}, leader_{leader}, server_addrs_{server_addrs} {} + + RsmClient() = delete; + + std::optional> SendWriteRequest(WriteRequestT req) { + WriteRequest client_req; + client_req.operation = req; + + std::cout << "client sending CasRequest to Leader " << leader_.last_known_port << std::endl; + ResponseFuture> response_future = + io_.template Request, WriteResponse>(leader_, client_req); + ResponseResult> response_result = std::move(response_future).Wait(); + + if (response_result.HasError()) { + std::cout << "client timed out while trying to communicate with leader server " << std::endl; + // continue; + return std::nullopt; + } + + ResponseEnvelope> response_envelope = response_result.GetValue(); + WriteResponse write_response = response_envelope.message; + + return CheckForCorrectLeader(write_response); + } + + std::optional> SendReadRequest(ReadRequestT req) { + ReadRequest read_req; + read_req.operation = req; + + std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; + ResponseFuture> get_response_future = + io_.template Request, ReadResponse>(leader_, read_req); + + // receive response + ResponseResult> get_response_result = std::move(get_response_future).Wait(); + + if (get_response_result.HasError()) { + std::cout << "client timed out while trying to communicate with leader server " << std::endl; + return {}; + } + + ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); + ReadResponse read_get_response = get_response_envelope.message; + + if (!read_get_response.success) { + // sent to a non-leader + return {}; + } + + return CheckForCorrectLeader(read_get_response); + } +}; + template void RunRaft(Raft server) { server.Run(); @@ -147,7 +232,6 @@ void RunSimulation() { std::vector
srv_2_peers = {srv_addr_1, srv_addr_3}; std::vector
srv_3_peers = {srv_addr_1, srv_addr_2}; - // TODO(tyler / gabor) supply default TestState to Raft constructor using RaftClass = Raft; RaftClass srv_1{std::move(srv_io_1), srv_1_peers, TestState{}}; RaftClass srv_2{std::move(srv_io_2), srv_2_peers, TestState{}}; @@ -165,16 +249,21 @@ void RunSimulation() { std::cout << "beginning test after servers have become quiescent" << std::endl; std::mt19937 cli_rng_{0}; - Address server_addrs[]{srv_addr_1, srv_addr_2, srv_addr_3}; + std::vector
server_addrs{srv_addr_1, srv_addr_2, srv_addr_3}; Address leader = server_addrs[0]; + RsmClient, CasRequest, CasResponse, GetRequest, GetResponse> client( + std::move(cli_io), std::move(leader), std::move(server_addrs)); + const int key = 0; std::optional last_known_value; bool success = false; for (int i = 0; !success; i++) { - // send request + /* + * Write Request + */ CasRequest cas_req; cas_req.key = key; @@ -182,37 +271,11 @@ void RunSimulation() { cas_req.new_value = i; - WriteRequest cli_req; - cli_req.operation = cas_req; - - std::cout << "client sending CasRequest to Leader " << leader.last_known_port << std::endl; - ResponseFuture> cas_response_future = - cli_io.Request, WriteResponse>(leader, cli_req); - - // receive cas_response - ResponseResult> cas_response_result = std::move(cas_response_future).Wait(); - - if (cas_response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; - continue; - } - - ResponseEnvelope> cas_response_envelope = cas_response_result.GetValue(); - WriteResponse write_cas_response = cas_response_envelope.message; - - if (write_cas_response.retry_leader) { - MG_ASSERT(!write_cas_response.success, "retry_leader should never be set for successful responses"); - leader = write_cas_response.retry_leader.value(); - std::cout << "client redirected to leader server " << leader.last_known_port << std::endl; - } else if (!write_cas_response.success) { - std::uniform_int_distribution addr_distrib(0, 2); - size_t addr_index = addr_distrib(cli_rng_); - leader = server_addrs[addr_index]; - - std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index - << " with port " << leader.last_known_port << std::endl; + auto write_cas_response_opt = client.SendWriteRequest(cas_req); + if (!write_cas_response_opt) { continue; } + auto write_cas_response = write_cas_response_opt.value(); CasResponse cas_response = write_cas_response.write_return; @@ -228,44 +291,17 @@ void RunSimulation() { continue; } + /* + * Get Request + */ GetRequest get_req; get_req.key = key; - ReadRequest read_req; - read_req.operation = get_req; - - std::cout << "client sending GetRequest to Leader " << leader.last_known_port << std::endl; - ResponseFuture> get_response_future = - cli_io.Request, ReadResponse>(leader, read_req); - - // receive response - ResponseResult> get_response_result = std::move(get_response_future).Wait(); - - if (get_response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; + auto read_get_response_opt = client.SendReadRequest(get_req); + if (!read_get_response_opt) { continue; } - - ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); - ReadResponse read_get_response = get_response_envelope.message; - - if (!read_get_response.success) { - // sent to a non-leader - continue; - } - - if (read_get_response.retry_leader) { - MG_ASSERT(!read_get_response.success, "retry_leader should never be set for successful responses"); - leader = read_get_response.retry_leader.value(); - std::cout << "client redirected to leader server " << leader.last_known_port << std::endl; - } else if (!read_get_response.success) { - std::uniform_int_distribution addr_distrib(0, 2); - size_t addr_index = addr_distrib(cli_rng_); - leader = server_addrs[addr_index]; - - std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index - << " with port " << leader.last_known_port << std::endl; - } + auto read_get_response = read_get_response_opt.value(); GetResponse get_response = read_get_response.read_return; From 797c76cdfd6c9be9df390b5766fa7f8ee8a3e67a Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 16:16:36 +0200 Subject: [PATCH 28/39] Add logic to split shards --- src/coordinator/coordinator.hpp | 12 +++++++++- src/coordinator/shard_map.hpp | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 9f87210e8..5078b2196 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -110,6 +110,17 @@ class Coordinator { /// Increment our ReadResponses Read(HlcRequest &&hlc_request) { HlcResponse res{}; + shard_map_.UpdateShardMapVersion(); + + res.new_hlc = shard_map_.GetHlc(); + + // TODO(gabor) Once the walclock update is implemented, this + // comparison should also be updated + if (hlc_request.last_shard_map_version.logical_id == res.new_hlc.logical_id) { + res.fresher_shard_map = shard_map_; + } else { + res.fresher_shard_map = {}; + } return res; } @@ -117,7 +128,6 @@ class Coordinator { GetShardMapResponse Read(GetShardMapRequest &&get_shard_map_request) { GetShardMapResponse res; res.shard_map = shard_map_; - return res; } diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index 795bdc1d2..68c387eec 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -44,7 +44,46 @@ struct ShardMap { Hlc shard_map_version; std::map shards; + // TODO(gabor) later we will want to update the wallclock time with + // the given Io's time as well. This function should just be + // replaced with operator== since it is already overloaded for Hlc + // objects. + bool CompareShardMapVersions(Hlc one, Hlc two) { return one.logical_id == two.logical_id; } + public: + // TODO(gabor) later we will want to update the wallclock time with + // the given Io's time as well + void UpdateShardMapVersion() noexcept { ++shard_map_version.logical_id; } + + Hlc GetHlc() const noexcept { return shard_map_version; } + + bool SplitShard(Hlc previous_shard_map_version, Label label, CompoundKey key) { + if (CompareShardMapVersions(previous_shard_map_version, shard_map_version)) { + MG_ASSERT(shards.contains(label)); + auto &shards_in_map = shards[label]; + MG_ASSERT(!shards_in_map.contains(key)); + + // Finding the Shard that the new CompoundKey should map to. + Shard shard_to_map_to; + auto &prev_key = (*shards_in_map.begin()).first; + + for (auto iter = std::next(shards_in_map.begin()); iter != shards_in_map.end(); ++iter) { + const auto ¤t_key = (*iter).first; + if (key > prev_key && key < current_key) { + shard_to_map_to = shards_in_map[prev_key]; + } + + prev_key = (*iter).first; + } + + // Apply the split + shards_in_map[key] = shard_to_map_to; + return true; + } + + return false; + } + Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end); Shard GetShardForKey(Label label, CompoundKey key); From 66e791d04294aefa83bdb8f7d8fe263ec1cc4c5b Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 16:25:11 +0200 Subject: [PATCH 29/39] Make Coordinator apply splitting on request. --- src/coordinator/coordinator.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 5078b2196..76bd7027f 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -144,10 +144,10 @@ class Coordinator { SplitShardResponse res{}; if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) { - // TODO reply with failure res.success = false; } else { - // TODO apply split + res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label, + split_shard_request.split_key); } return res; From a74a556fcca150b889d1f18a8844a2bd5225271a Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 17:48:11 +0200 Subject: [PATCH 30/39] Remove unnecessary storage pool, general cleanup --- src/coordinator/coordinator.hpp | 5 ----- src/coordinator/shard_map.hpp | 8 +++++++- tests/simulation/CMakeLists.txt | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 76bd7027f..787248f5d 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -90,8 +90,6 @@ class Coordinator { using StandbySotrageEnginePool = std::unordered_set
; ShardMap shard_map_; - // StandbySotrageEnginePool storage_engine_pool_; - /// The highest reserved timestamp / highest allocated timestamp /// is a way for minimizing communication involved in query engines /// reserving Hlc's for their transaction processing. @@ -158,9 +156,6 @@ class Coordinator { WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { RegisterStorageEngineResponse res{}; // TODO - // const Address &address = register_storage_engine_request.address; - // storage_engine_pool_.insert(address); - // res.success = true; return res; } diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index 68c387eec..7e20da565 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -33,6 +33,8 @@ struct AddressAndStatus { Status status; }; +using memgraph::io::Address; + using CompoundKey = std::vector; using Shard = std::vector; using Shards = std::map; @@ -65,7 +67,7 @@ struct ShardMap { // Finding the Shard that the new CompoundKey should map to. Shard shard_to_map_to; - auto &prev_key = (*shards_in_map.begin()).first; + CompoundKey prev_key = ((*shards_in_map.begin()).first); for (auto iter = std::next(shards_in_map.begin()); iter != shards_in_map.end(); ++iter) { const auto ¤t_key = (*iter).first; @@ -84,6 +86,10 @@ struct ShardMap { return false; } + void AddServer(Address server_address) { + // Find a random place for the server to plug in + } + Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end); Shard GetShardForKey(Label label, CompoundKey key); diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index 15c426e8c..b44dacb36 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -31,4 +31,4 @@ add_simulation_test(raft.cpp address) add_simulation_test(trial_query_storage/query_storage_test.cpp address) -#add_simulation_test(sharded_map.cpp address) +add_simulation_test(sharded_map.cpp address) From 1b2c8f6b2980e3ca3e752c4fa6cde816c40f8374 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Mon, 8 Aug 2022 17:59:39 +0200 Subject: [PATCH 31/39] Move RsmClient into a separate folder and header --- tests/simulation/CMakeLists.txt | 2 +- tests/simulation/raft.cpp | 131 ++++++++++++++++---------------- 2 files changed, 67 insertions(+), 66 deletions(-) diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index b44dacb36..15c426e8c 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -31,4 +31,4 @@ add_simulation_test(raft.cpp address) add_simulation_test(trial_query_storage/query_storage_test.cpp address) -add_simulation_test(sharded_map.cpp address) +#add_simulation_test(sharded_map.cpp address) diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index 29f0a97dd..6f6d8fb08 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -22,6 +22,7 @@ #include "io/rsm/raft.hpp" #include "io/simulator/simulator.hpp" #include "io/simulator/simulator_transport.hpp" +#include "utils/rsm_client.hpp" using memgraph::io::Address; using memgraph::io::Duration; @@ -116,90 +117,90 @@ class TestState { } }; -template -class RsmClient { - using ServerPool = std::vector
; +// template +// class RsmClient { +// using ServerPool = std::vector
; - IoImpl io_; - Address leader_; +// IoImpl io_; +// Address leader_; - std::mt19937 cli_rng_{0}; - ServerPool server_addrs_; +// std::mt19937 cli_rng_{0}; +// ServerPool server_addrs_; - template - std::optional CheckForCorrectLeader(ResponseT response) { - if (response.retry_leader) { - MG_ASSERT(!response.success, "retry_leader should never be set for successful responses"); - leader_ = response.retry_leader.value(); - std::cout << "client redirected to leader server " << leader_.last_known_port << std::endl; - } else if (!response.success) { - std::uniform_int_distribution addr_distrib(0, (server_addrs_.size() - 1)); - size_t addr_index = addr_distrib(cli_rng_); - leader_ = server_addrs_[addr_index]; +// template +// std::optional CheckForCorrectLeader(ResponseT response) { +// if (response.retry_leader) { +// MG_ASSERT(!response.success, "retry_leader should never be set for successful responses"); +// leader_ = response.retry_leader.value(); +// std::cout << "client redirected to leader server " << leader_.last_known_port << std::endl; +// } else if (!response.success) { +// std::uniform_int_distribution addr_distrib(0, (server_addrs_.size() - 1)); +// size_t addr_index = addr_distrib(cli_rng_); +// leader_ = server_addrs_[addr_index]; - std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index - << " with port " << leader_.last_known_port << std::endl; - return {}; - } +// std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index +// << " with port " << leader_.last_known_port << std::endl; +// return {}; +// } - return response; - } +// return response; +// } - public: - RsmClient(IoImpl &&io, Address &&leader, ServerPool &&server_addrs) - : io_{io}, leader_{leader}, server_addrs_{server_addrs} {} +// public: +// RsmClient(IoImpl &&io, Address &&leader, ServerPool &&server_addrs) +// : io_{io}, leader_{leader}, server_addrs_{server_addrs} {} - RsmClient() = delete; +// RsmClient() = delete; - std::optional> SendWriteRequest(WriteRequestT req) { - WriteRequest client_req; - client_req.operation = req; +// std::optional> SendWriteRequest(WriteRequestT req) { +// WriteRequest client_req; +// client_req.operation = req; - std::cout << "client sending CasRequest to Leader " << leader_.last_known_port << std::endl; - ResponseFuture> response_future = - io_.template Request, WriteResponse>(leader_, client_req); - ResponseResult> response_result = std::move(response_future).Wait(); +// std::cout << "client sending CasRequest to Leader " << leader_.last_known_port << std::endl; +// ResponseFuture> response_future = +// io_.template Request, WriteResponse>(leader_, client_req); +// ResponseResult> response_result = std::move(response_future).Wait(); - if (response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; - // continue; - return std::nullopt; - } +// if (response_result.HasError()) { +// std::cout << "client timed out while trying to communicate with leader server " << std::endl; +// // continue; +// return std::nullopt; +// } - ResponseEnvelope> response_envelope = response_result.GetValue(); - WriteResponse write_response = response_envelope.message; +// ResponseEnvelope> response_envelope = response_result.GetValue(); +// WriteResponse write_response = response_envelope.message; - return CheckForCorrectLeader(write_response); - } +// return CheckForCorrectLeader(write_response); +// } - std::optional> SendReadRequest(ReadRequestT req) { - ReadRequest read_req; - read_req.operation = req; +// std::optional> SendReadRequest(ReadRequestT req) { +// ReadRequest read_req; +// read_req.operation = req; - std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; - ResponseFuture> get_response_future = - io_.template Request, ReadResponse>(leader_, read_req); +// std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; +// ResponseFuture> get_response_future = +// io_.template Request, ReadResponse>(leader_, read_req); - // receive response - ResponseResult> get_response_result = std::move(get_response_future).Wait(); +// // receive response +// ResponseResult> get_response_result = std::move(get_response_future).Wait(); - if (get_response_result.HasError()) { - std::cout << "client timed out while trying to communicate with leader server " << std::endl; - return {}; - } +// if (get_response_result.HasError()) { +// std::cout << "client timed out while trying to communicate with leader server " << std::endl; +// return {}; +// } - ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); - ReadResponse read_get_response = get_response_envelope.message; +// ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); +// ReadResponse read_get_response = get_response_envelope.message; - if (!read_get_response.success) { - // sent to a non-leader - return {}; - } +// if (!read_get_response.success) { +// // sent to a non-leader +// return {}; +// } - return CheckForCorrectLeader(read_get_response); - } -}; +// return CheckForCorrectLeader(read_get_response); +// } +// }; template void RunRaft(Raft server) { From 9aeca7a4b39e2a927da72a3983e154b95ca16e93 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 9 Aug 2022 16:04:14 +0200 Subject: [PATCH 32/39] Add test for sharded_map --- src/coordinator/coordinator.hpp | 21 ++- src/coordinator/shard_map.hpp | 7 +- src/io/rsm/shard_rsm.hpp | 15 +- tests/simulation/CMakeLists.txt | 2 +- tests/simulation/raft.cpp | 4 +- tests/simulation/sharded_map.cpp | 244 +++++++++++++++++++++----- tests/simulation/utils/rsm_client.hpp | 111 ++++++++++++ 7 files changed, 347 insertions(+), 57 deletions(-) create mode 100644 tests/simulation/utils/rsm_client.hpp diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 787248f5d..aa36af6a2 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -87,8 +87,6 @@ using ReadRequests = std::variant; using ReadResponses = std::variant; class Coordinator { - using StandbySotrageEnginePool = std::unordered_set
; - ShardMap shard_map_; /// The highest reserved timestamp / highest allocated timestamp /// is a way for minimizing communication involved in query engines @@ -108,17 +106,16 @@ class Coordinator { /// Increment our ReadResponses Read(HlcRequest &&hlc_request) { HlcResponse res{}; - shard_map_.UpdateShardMapVersion(); - res.new_hlc = shard_map_.GetHlc(); + auto hlc_shard_map = shard_map_.GetHlc(); - // TODO(gabor) Once the walclock update is implemented, this - // comparison should also be updated - if (hlc_request.last_shard_map_version.logical_id == res.new_hlc.logical_id) { - res.fresher_shard_map = shard_map_; - } else { - res.fresher_shard_map = {}; - } + MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id)); + + res.new_hlc = shard_map_.UpdateShardMapVersion(); + + res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id + ? std::make_optional(shard_map_) + : std::nullopt; return res; } @@ -173,6 +170,8 @@ class Coordinator { } public: + explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {} + ReadResponses Read(ReadRequests requests) { return std::visit([&](auto &&requests) { return Read(requests); }, std::move(requests)); } diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index 7e20da565..e78699b1b 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -55,7 +55,10 @@ struct ShardMap { public: // TODO(gabor) later we will want to update the wallclock time with // the given Io's time as well - void UpdateShardMapVersion() noexcept { ++shard_map_version.logical_id; } + Hlc UpdateShardMapVersion() noexcept { + ++shard_map_version.logical_id; + return shard_map_version; + } Hlc GetHlc() const noexcept { return shard_map_version; } @@ -90,6 +93,8 @@ struct ShardMap { // Find a random place for the server to plug in } + std::map &GetShards() noexcept { return shards; } + Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end); Shard GetShardForKey(Label label, CompoundKey key); diff --git a/src/io/rsm/shard_rsm.hpp b/src/io/rsm/shard_rsm.hpp index 0be4a1b12..ea45c33fc 100644 --- a/src/io/rsm/shard_rsm.hpp +++ b/src/io/rsm/shard_rsm.hpp @@ -89,13 +89,16 @@ class StorageRsm { StorageGetResponse Read(StorageGetRequest request) { StorageGetResponse ret; - if (IsKeyInRange(request.key)) { + if (!IsKeyInRange(request.key)) { + std::cout << "ONE" << std::endl; ret.latest_known_shard_map_version = shard_map_version_; ret.shard_rsm_success = false; } else if (state_.contains(request.key)) { + std::cout << "TWO" << std::endl; ret.value = state_[request.key]; ret.shard_rsm_success = true; } else { + std::cout << "THREE" << std::endl; ret.shard_rsm_success = false; ret.value = std::nullopt; } @@ -106,13 +109,15 @@ class StorageRsm { StorageWriteResponse ret; // Key is outside the prohibited range - if (IsKeyInRange(request.key)) { + if (!IsKeyInRange(request.key)) { ret.latest_known_shard_map_version = shard_map_version_; ret.shard_rsm_success = false; + std::cout << "WRITE 0" << std::endl; } // Key exist else if (state_.contains(request.key)) { auto &val = state_[request.key]; + std::cout << "WRITE 1" << std::endl; /* * Delete @@ -121,6 +126,7 @@ class StorageRsm { ret.shard_rsm_success = true; ret.last_value = val; state_.erase(state_.find(request.key)); + std::cout << "WRITE 2" << std::endl; } /* @@ -132,9 +138,12 @@ class StorageRsm { ret.shard_rsm_success = true; val = request.value.value(); + std::cout << "WRITE 3" << std::endl; + } else { ret.last_value = val; ret.shard_rsm_success = false; + std::cout << "WRITE 4" << std::endl; } } /* @@ -145,8 +154,10 @@ class StorageRsm { ret.shard_rsm_success = true; state_.emplace(request.key, std::move(request.value).value()); + std::cout << "WRITE 5" << std::endl; } + std::cout << "WRITE ret" << std::endl; return ret; } }; diff --git a/tests/simulation/CMakeLists.txt b/tests/simulation/CMakeLists.txt index 15c426e8c..b44dacb36 100644 --- a/tests/simulation/CMakeLists.txt +++ b/tests/simulation/CMakeLists.txt @@ -31,4 +31,4 @@ add_simulation_test(raft.cpp address) add_simulation_test(trial_query_storage/query_storage_test.cpp address) -#add_simulation_test(sharded_map.cpp address) +add_simulation_test(sharded_map.cpp address) diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index 6f6d8fb08..c0cb84476 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -253,8 +253,8 @@ void RunSimulation() { std::vector
server_addrs{srv_addr_1, srv_addr_2, srv_addr_3}; Address leader = server_addrs[0]; - RsmClient, CasRequest, CasResponse, GetRequest, GetResponse> client( - std::move(cli_io), std::move(leader), std::move(server_addrs)); + RsmClient, CasRequest, CasResponse, GetRequest, GetResponse> client(cli_io, leader, + server_addrs); const int key = 0; std::optional last_known_value; diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index 19f6de4b9..785db46ff 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -24,12 +24,21 @@ #include "io/rsm/shard_rsm.hpp" #include "io/simulator/simulator.hpp" #include "io/simulator/simulator_transport.hpp" +#include "utils/rsm_client.hpp" +using memgraph::coordinator::Address; +using memgraph::coordinator::AddressAndStatus; +using memgraph::coordinator::CompoundKey; using memgraph::coordinator::Coordinator; +using memgraph::coordinator::Shard; +using memgraph::coordinator::ShardMap; +using memgraph::coordinator::Shards; +using memgraph::coordinator::Status; using memgraph::io::Address; using memgraph::io::Io; using memgraph::io::ResponseEnvelope; using memgraph::io::ResponseFuture; +using memgraph::io::Time; using memgraph::io::rsm::CoordinatorRsm; using memgraph::io::rsm::Raft; using memgraph::io::rsm::ReadRequest; @@ -46,60 +55,96 @@ using memgraph::io::simulator::SimulatorConfig; using memgraph::io::simulator::SimulatorStats; using memgraph::io::simulator::SimulatorTransport; +namespace { +ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Address b_io_1, Address b_io_2, + Address b_io_3) { + ShardMap sm1; + auto &shards = sm1.GetShards(); + + // 1 + std::string label1 = std::string("label1"); + auto key1 = memgraph::storage::v3::PropertyValue(3); + auto key2 = memgraph::storage::v3::PropertyValue(4); + CompoundKey cm1 = {key1, key2}; + AddressAndStatus aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT}; + AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT}; + AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT}; + + Shard shard1 = {aas1_1, aas1_2, aas1_3}; + Shards shards1; + shards1[cm1] = shard1; + + // 2 + std::string label2 = std::string("label2"); + auto key3 = memgraph::storage::v3::PropertyValue(12); + auto key4 = memgraph::storage::v3::PropertyValue(13); + CompoundKey cm2 = {key3, key4}; + AddressAndStatus aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT}; + AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT}; + AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT}; + + Shard shard2 = {aas2_1, aas2_2, aas2_3}; + Shards shards2; + shards2[cm2] = shard2; + + shards[label2] = shards2; + + return sm1; +} +} // namespace + using ConcreteCoordinatorRsm = CoordinatorRsm; using ConcreteStorageRsm = Raft; +template +void RunStorageRaft( + Raft server) { + server.Run(); +} + int main() { SimulatorConfig config{ - /* - .drop_percent = 5, - .perform_timeouts = true, - .scramble_messages = true, - .rng_seed = 0, - .start_time = 256 * 1024, - .abort_time = std::chrono::microseconds{8 * 1024 * 1024}, - */ + .drop_percent = 5, + .perform_timeouts = true, + .scramble_messages = true, + .rng_seed = 0, + .start_time = Time::min() + std::chrono::microseconds{256 * 1024}, + .abort_time = Time::min() + std::chrono::microseconds{8 * 1024 * 1024}, }; auto simulator = Simulator(config); Io cli_io = simulator.RegisterNew(); - // spin up coordinators + // auto c_thread_1 = std::jthread(RunRaft< Coordinator>, std::move(c_1)); + // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[0]); - Io c_io_1 = simulator.RegisterNew(); - Io c_io_2 = simulator.RegisterNew(); - Io c_io_3 = simulator.RegisterNew(); + // auto c_thread_2 = std::jthread(RunRaft< Coordinator>, std::move(c_2)); + // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[1]); - Address c_addrs[] = {c_io_1.GetAddress(), c_io_2.GetAddress(), c_io_3.GetAddress()}; - - std::vector
c_1_peers = {c_addrs[1], c_addrs[2]}; - std::vector
c_2_peers = {c_addrs[0], c_addrs[2]}; - std::vector
c_3_peers = {c_addrs[0], c_addrs[1]}; - - ConcreteCoordinatorRsm c_1{std::move(c_io_1), c_1_peers, Coordinator{}}; - ConcreteCoordinatorRsm c_2{std::move(c_io_2), c_2_peers, Coordinator{}}; - ConcreteCoordinatorRsm c_3{std::move(c_io_3), c_3_peers, Coordinator{}}; - - auto c_thread_1 = std::jthread([c_1]() mutable { c_1.Run(); }); - simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[0]); - - /* - auto c_thread_2 = std::jthread(RunRaft< Coordinator>, std::move(c_2)); - simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[1]); - - auto c_thread_3 = std::jthread(RunRaft, std::move(c_3)); - simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); - */ - - // spin up shard A + // auto c_thread_3 = std::jthread(RunRaft, std::move(c_3)); + // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); + // Register Io a_io_1 = simulator.RegisterNew(); Io a_io_2 = simulator.RegisterNew(); Io a_io_3 = simulator.RegisterNew(); - Address a_addrs[] = {a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress()}; + Io b_io_1 = simulator.RegisterNew(); + Io b_io_2 = simulator.RegisterNew(); + Io b_io_3 = simulator.RegisterNew(); + + // Preconfigure coordinator with kv shard 'A' and 'B' + auto sm1 = CreateDummyShardmap(a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress(), b_io_1.GetAddress(), + b_io_2.GetAddress(), b_io_3.GetAddress()); + auto sm2 = CreateDummyShardmap(a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress(), b_io_1.GetAddress(), + b_io_2.GetAddress(), b_io_3.GetAddress()); + auto sm3 = CreateDummyShardmap(a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress(), b_io_1.GetAddress(), + b_io_2.GetAddress(), b_io_3.GetAddress()); + + // Spin up shard A + std::vector
a_addrs = {a_io_1.GetAddress(), a_io_2.GetAddress(), a_io_3.GetAddress()}; std::vector
a_1_peers = {a_addrs[1], a_addrs[2]}; std::vector
a_2_peers = {a_addrs[0], a_addrs[2]}; @@ -109,13 +154,17 @@ int main() { ConcreteStorageRsm a_2{std::move(a_io_2), a_2_peers, StorageRsm{}}; ConcreteStorageRsm a_3{std::move(a_io_3), a_3_peers, StorageRsm{}}; - // spin up shard B + auto a_thread_1 = std::jthread(RunStorageRaft, std::move(a_1)); + simulator.IncrementServerCountAndWaitForQuiescentState(a_addrs[0]); - Io b_io_1 = simulator.RegisterNew(); - Io b_io_2 = simulator.RegisterNew(); - Io b_io_3 = simulator.RegisterNew(); + auto a_thread_2 = std::jthread(RunStorageRaft, std::move(a_2)); + simulator.IncrementServerCountAndWaitForQuiescentState(a_addrs[1]); - Address b_addrs[] = {b_io_1.GetAddress(), b_io_2.GetAddress(), b_io_3.GetAddress()}; + auto a_thread_3 = std::jthread(RunStorageRaft, std::move(a_3)); + simulator.IncrementServerCountAndWaitForQuiescentState(a_addrs[2]); + + // Spin up shard B + std::vector
b_addrs = {b_io_1.GetAddress(), b_io_2.GetAddress(), b_io_3.GetAddress()}; std::vector
b_1_peers = {b_addrs[1], b_addrs[2]}; std::vector
b_2_peers = {b_addrs[0], b_addrs[2]}; @@ -125,7 +174,122 @@ int main() { ConcreteStorageRsm b_2{std::move(b_io_2), b_2_peers, StorageRsm{}}; ConcreteStorageRsm b_3{std::move(b_io_3), b_3_peers, StorageRsm{}}; + auto b_thread_1 = std::jthread(RunStorageRaft, std::move(b_1)); + simulator.IncrementServerCountAndWaitForQuiescentState(b_addrs[0]); + + auto b_thread_2 = std::jthread(RunStorageRaft, std::move(b_2)); + simulator.IncrementServerCountAndWaitForQuiescentState(b_addrs[1]); + + auto b_thread_3 = std::jthread(RunStorageRaft, std::move(b_3)); + simulator.IncrementServerCountAndWaitForQuiescentState(b_addrs[2]); + std::cout << "beginning test after servers have become quiescent" << std::endl; + // Spin up coordinators + + Io c_io_1 = simulator.RegisterNew(); + Io c_io_2 = simulator.RegisterNew(); + Io c_io_3 = simulator.RegisterNew(); + + std::vector
c_addrs = {c_io_1.GetAddress(), c_io_2.GetAddress(), c_io_3.GetAddress()}; + + std::vector
c_1_peers = {c_addrs[1], c_addrs[2]}; + std::vector
c_2_peers = {c_addrs[0], c_addrs[2]}; + std::vector
c_3_peers = {c_addrs[0], c_addrs[1]}; + + ConcreteCoordinatorRsm c_1{std::move(c_io_1), c_1_peers, Coordinator{(sm1)}}; + ConcreteCoordinatorRsm c_2{std::move(c_io_2), c_2_peers, Coordinator{(sm2)}}; + ConcreteCoordinatorRsm c_3{std::move(c_io_3), c_3_peers, Coordinator{(sm3)}}; + + auto c_thread_1 = std::jthread([c_1]() mutable { c_1.Run(); }); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[0]); + + auto c_thread_2 = std::jthread([c_2]() mutable { c_2.Run(); }); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[1]); + + auto c_thread_3 = std::jthread([c_3]() mutable { c_3.Run(); }); + simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); + + // Have client contact coordinator RSM for a new transaction ID and + // also get the current shard map + using CoordinatorClient = + RsmClient, memgraph::coordinator::WriteRequests, memgraph::coordinator::WriteResponses, + memgraph::coordinator::ReadRequests, memgraph::coordinator::ReadResponses>; + CoordinatorClient coordinator_client(cli_io, c_addrs[2], c_addrs); + + using StorageClient = RsmClient, StorageWriteRequest, StorageWriteResponse, StorageGetRequest, + StorageGetResponse>; + StorageClient shard_a_client(cli_io, a_addrs[0], a_addrs); + StorageClient shard_b_client(cli_io, b_addrs[0], b_addrs); + + memgraph::coordinator::HlcRequest req; + + // Last ShardMap Version The query engine knows about. + ShardMap client_shard_map; + req.last_shard_map_version = client_shard_map.GetHlc(); + + while (true) { + // auto read_res_opt = coordinator_client.SendReadRequest(req); + // if(!read_res_opt) + // { + // std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!0" << std::endl; + // continue; + // } + // auto read_res = read_res_opt.value(); + + // auto res = std::get(read_res.read_return); + + // auto transaction_id = res.new_hlc; + + // client_shard_map = res.fresher_shard_map.value(); + + // // Have client use shard map to decide which shard to communicate + // // with in order to write a new value + + // //client_shard_map. + StorageWriteRequest storage_req; + auto write_key_1 = memgraph::storage::PropertyValue(3); + auto write_key_2 = memgraph::storage::PropertyValue(4); + storage_req.key = {write_key_1, write_key_2}; + storage_req.value = 1000; + auto write_res_opt = shard_a_client.SendWriteRequest(storage_req); + if (!write_res_opt) { + std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1" << std::endl; + continue; + } + auto write_res = write_res_opt.value().write_return; + + bool cas_succeeded = write_res.shard_rsm_success; + + if (cas_succeeded) { + last_known_value = i; + } else { + last_known_value = cas_response.last_value; + continue; + } + + // ... write_res. + + // Have client use shard map to decide which shard to communicate + // with to read that same value back + + StorageGetRequest storage_get_req; + storage_get_req.key = {write_key_1, write_key_2}; + auto get_res_opt = shard_a_client.SendReadRequest(storage_get_req); + if (!get_res_opt) { + std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!2" << std::endl; + continue; + } + auto get_res = get_res_opt.value(); + auto val = get_res.read_return.value.value(); + + std::cout << "val -> " << val << std::endl; + + MG_ASSERT(get_res.read_return.value == 1000); + break; + } + + simulator.ShutDown(); + return 0; } diff --git a/tests/simulation/utils/rsm_client.hpp b/tests/simulation/utils/rsm_client.hpp new file mode 100644 index 000000000..0b4b47cae --- /dev/null +++ b/tests/simulation/utils/rsm_client.hpp @@ -0,0 +1,111 @@ +// Copyright 2022 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include + +#include "io/address.hpp" +#include "io/rsm/raft.hpp" + +using memgraph::io::Address; +using memgraph::io::ResponseEnvelope; +using memgraph::io::ResponseFuture; +using memgraph::io::ResponseResult; +using memgraph::io::rsm::ReadRequest; +using memgraph::io::rsm::ReadResponse; +using memgraph::io::rsm::WriteRequest; +using memgraph::io::rsm::WriteResponse; + +template +class RsmClient { + using ServerPool = std::vector
; + + IoImpl io_; + Address leader_; + + std::mt19937 cli_rng_{0}; + ServerPool server_addrs_; + + template + std::optional CheckForCorrectLeader(ResponseT response) { + if (response.retry_leader) { + MG_ASSERT(!response.success, "retry_leader should never be set for successful responses"); + leader_ = response.retry_leader.value(); + std::cout << "client redirected to leader server " << leader_.last_known_port << std::endl; + } else if (!response.success) { + std::uniform_int_distribution addr_distrib(0, (server_addrs_.size() - 1)); + size_t addr_index = addr_distrib(cli_rng_); + leader_ = server_addrs_[addr_index]; + + std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index + << " with port " << leader_.last_known_port << std::endl; + return std::nullopt; + } + + return response; + } + + public: + RsmClient(IoImpl io, Address leader, ServerPool server_addrs) + : io_{io}, leader_{leader}, server_addrs_{server_addrs} {} + + RsmClient() = delete; + + std::optional> SendWriteRequest(WriteRequestT req) { + WriteRequest client_req; + client_req.operation = req; + + std::cout << "client sending CasRequest to Leader " << leader_.last_known_port << std::endl; + ResponseFuture> response_future = + io_.template Request, WriteResponse>(leader_, client_req); + ResponseResult> response_result = std::move(response_future).Wait(); + + if (response_result.HasError()) { + std::cout << "client timed out while trying to communicate with leader server " << std::endl; + // continue; + return std::nullopt; + } + + ResponseEnvelope> response_envelope = response_result.GetValue(); + WriteResponse write_response = response_envelope.message; + + return CheckForCorrectLeader(write_response); + } + + std::optional> SendReadRequest(ReadRequestT req) { + ReadRequest read_req; + read_req.operation = req; + + std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; + ResponseFuture> get_response_future = + io_.template Request, ReadResponse>(leader_, read_req); + + // receive response + ResponseResult> get_response_result = std::move(get_response_future).Wait(); + + if (get_response_result.HasError()) { + std::cout << "client timed out while trying to communicate with leader server " << std::endl; + return std::nullopt; + } + + ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); + ReadResponse read_get_response = get_response_envelope.message; + + // if (!read_get_response.success) { + // // sent to a non-leader + // return {}; + // } + + return CheckForCorrectLeader(read_get_response); + } +}; From 2553a8fdc377db5859ee141a1e628b7d26a54c30 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 9 Aug 2022 16:06:17 +0200 Subject: [PATCH 33/39] Remove unfinished statement --- tests/simulation/sharded_map.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index 785db46ff..ee41d833b 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -261,12 +261,12 @@ int main() { bool cas_succeeded = write_res.shard_rsm_success; - if (cas_succeeded) { - last_known_value = i; - } else { - last_known_value = cas_response.last_value; - continue; - } + // if (cas_succeeded) { + // last_known_value = i; + // } else { + // last_known_value = cas_response.last_value; + // continue; + // } // ... write_res. From 5d66a4e828cfbd38940463b6b9382b05df97b0c2 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 9 Aug 2022 16:25:00 +0200 Subject: [PATCH 34/39] Check against cas_succeeded --- tests/simulation/sharded_map.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index ee41d833b..f547dc9c6 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -261,12 +261,9 @@ int main() { bool cas_succeeded = write_res.shard_rsm_success; - // if (cas_succeeded) { - // last_known_value = i; - // } else { - // last_known_value = cas_response.last_value; - // continue; - // } + if (!cas_succeeded) { + continue; + } // ... write_res. From 9c453779f5daa6982c872c1e24ea2c5142de7448 Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 9 Aug 2022 17:57:13 +0200 Subject: [PATCH 35/39] Add the coordinator part to the sharded_map test --- src/coordinator/coordinator.hpp | 11 +++-- src/coordinator/shard_map.hpp | 2 +- tests/simulation/sharded_map.cpp | 84 +++++++++++++++++++++++--------- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index aa36af6a2..d37e93f53 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -107,15 +107,20 @@ class Coordinator { ReadResponses Read(HlcRequest &&hlc_request) { HlcResponse res{}; + std::cout << "HlcRequest->HlcResponse" << std::endl; + auto hlc_shard_map = shard_map_.GetHlc(); MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id)); res.new_hlc = shard_map_.UpdateShardMapVersion(); - res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id - ? std::make_optional(shard_map_) - : std::nullopt; + // res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id + // ? std::make_optional(shard_map_) + // : std::nullopt; + + // Allways return fresher shard_map for now. + res.fresher_shard_map = std::make_optional(shard_map_); return res; } diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index e78699b1b..cf7cec541 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -97,7 +97,7 @@ struct ShardMap { Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end); - Shard GetShardForKey(Label label, CompoundKey key); + Shard GetShardForKey(Label label, CompoundKey key) { return shards.at(label).at(key); } }; } // namespace memgraph::coordinator diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index f547dc9c6..9b38ac123 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -55,9 +55,12 @@ using memgraph::io::simulator::SimulatorConfig; using memgraph::io::simulator::SimulatorStats; using memgraph::io::simulator::SimulatorTransport; +using StorageClient = + RsmClient, StorageWriteRequest, StorageWriteResponse, StorageGetRequest, StorageGetResponse>; namespace { -ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Address b_io_1, Address b_io_2, - Address b_io_3) { +ShardMap CreateDummyShardmap(memgraph::coordinator::Address a_io_1, memgraph::coordinator::Address a_io_2, + memgraph::coordinator::Address a_io_3, memgraph::coordinator::Address b_io_1, + memgraph::coordinator::Address b_io_2, memgraph::coordinator::Address b_io_3) { ShardMap sm1; auto &shards = sm1.GetShards(); @@ -91,6 +94,21 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add return sm1; } + +std::optional DetermineShardLocation(Shard target_shard, const std::vector
&a_addrs, + StorageClient a_client, const std::vector
&b_addrs, + StorageClient b_client) { + for (const auto &addr : target_shard) { + if (addr.address == b_addrs[0]) { + return b_client; + } + if (addr.address == a_addrs[0]) { + return a_client; + } + } + return {}; +} + } // namespace using ConcreteCoordinatorRsm = CoordinatorRsm; @@ -215,10 +233,8 @@ int main() { using CoordinatorClient = RsmClient, memgraph::coordinator::WriteRequests, memgraph::coordinator::WriteResponses, memgraph::coordinator::ReadRequests, memgraph::coordinator::ReadResponses>; - CoordinatorClient coordinator_client(cli_io, c_addrs[2], c_addrs); + CoordinatorClient coordinator_client(cli_io, c_addrs[0], c_addrs); - using StorageClient = RsmClient, StorageWriteRequest, StorageWriteResponse, StorageGetRequest, - StorageGetResponse>; StorageClient shard_a_client(cli_io, a_addrs[0], a_addrs); StorageClient shard_b_client(cli_io, b_addrs[0], b_addrs); @@ -229,30 +245,56 @@ int main() { req.last_shard_map_version = client_shard_map.GetHlc(); while (true) { - // auto read_res_opt = coordinator_client.SendReadRequest(req); - // if(!read_res_opt) - // { - // std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!0" << std::endl; - // continue; - // } - // auto read_res = read_res_opt.value(); + // Create CompoundKey + auto cm_key_1 = memgraph::storage::v3::PropertyValue(3); + auto cm_key_2 = memgraph::storage::v3::PropertyValue(4); - // auto res = std::get(read_res.read_return); + CompoundKey cm_k = {cm_key_1, cm_key_2}; - // auto transaction_id = res.new_hlc; + // Look for Shard + auto read_res_opt = coordinator_client.SendReadRequest(req); + if (!read_res_opt) { + std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!0" << std::endl; + continue; + } - // client_shard_map = res.fresher_shard_map.value(); + std::cout << "Before" << std::endl; + auto read_res = read_res_opt.value(); + std::cout << "After" << std::endl; - // // Have client use shard map to decide which shard to communicate - // // with in order to write a new value + auto res = std::get(read_res.read_return); + auto transaction_id = res.new_hlc; - // //client_shard_map. + std::cout << "transaction_id: " << transaction_id.logical_id << std::endl; + + if (!res.fresher_shard_map) { + // continue; + std::cout << "Something is really not OK..." << std::endl; + } + + std::cout << "Before2" << std::endl; + client_shard_map = res.fresher_shard_map.value(); + std::cout << "After2" << std::endl; + + auto target_shard = client_shard_map.GetShardForKey("label1", cm_k); + + // Determine which shard to send the requests to + auto storage_client_opt = DetermineShardLocation(target_shard, a_addrs, shard_a_client, b_addrs, shard_b_client); + MG_ASSERT(storage_client_opt); + + std::cout << "Before3" << std::endl; + auto storage_client = storage_client_opt.value(); + std::cout << "After3" << std::endl; + + // Have client use shard map to decide which shard to communicate + // with in order to write a new value + // client_shard_map. StorageWriteRequest storage_req; auto write_key_1 = memgraph::storage::PropertyValue(3); auto write_key_2 = memgraph::storage::PropertyValue(4); storage_req.key = {write_key_1, write_key_2}; storage_req.value = 1000; - auto write_res_opt = shard_a_client.SendWriteRequest(storage_req); + auto write_res_opt = storage_client.SendWriteRequest(storage_req); if (!write_res_opt) { std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1" << std::endl; continue; @@ -265,14 +307,12 @@ int main() { continue; } - // ... write_res. - // Have client use shard map to decide which shard to communicate // with to read that same value back StorageGetRequest storage_get_req; storage_get_req.key = {write_key_1, write_key_2}; - auto get_res_opt = shard_a_client.SendReadRequest(storage_get_req); + auto get_res_opt = storage_client.SendReadRequest(storage_get_req); if (!get_res_opt) { std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!2" << std::endl; continue; From 518e8df04cff6104e5f731dbf731a3d2a10e0dea Mon Sep 17 00:00:00 2001 From: gvolfing Date: Tue, 9 Aug 2022 20:51:53 +0200 Subject: [PATCH 36/39] Resolve issues with the sharded_map test --- src/coordinator/coordinator.hpp | 25 +++++++++++++++++++++---- src/coordinator/shard_map.hpp | 10 +++++++++- tests/simulation/sharded_map.cpp | 13 ++++++++++++- tests/simulation/utils/rsm_client.hpp | 1 + 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index d37e93f53..69a2e5cd7 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -104,10 +104,9 @@ class Coordinator { uint64_t highest_reserved_timestamp_; /// Increment our - ReadResponses Read(HlcRequest &&hlc_request) { - HlcResponse res{}; - + ReadResponses Read(HlcRequest hlc_request) { std::cout << "HlcRequest->HlcResponse" << std::endl; + HlcResponse res{}; auto hlc_shard_map = shard_map_.GetHlc(); @@ -126,6 +125,8 @@ class Coordinator { } GetShardMapResponse Read(GetShardMapRequest &&get_shard_map_request) { + std::cout << "GetShardMapRequest" << std::endl; + GetShardMapResponse res; res.shard_map = shard_map_; return res; @@ -178,7 +179,23 @@ class Coordinator { explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {} ReadResponses Read(ReadRequests requests) { - return std::visit([&](auto &&requests) { return Read(requests); }, std::move(requests)); + if (std::get_if(&requests)) { + std::cout << "HlcRequest" << std::endl; + } else if (std::get_if(&requests)) { + std::cout << "GetShardMapRequest" << std::endl; + } else { + std::cout << "idk requests" << std::endl; + } + std::cout << "Coordinator Read()" << std::endl; + auto ret = std::visit([&](auto requests) { return Read(requests); }, (requests)); + if (std::get_if(&ret)) { + std::cout << "HlcResponse" << std::endl; + } else if (std::get_if(&ret)) { + std::cout << "GetShardMapResponse" << std::endl; + } else { + std::cout << "idk response" << std::endl; + } + return ret; } WriteResponses Apply(WriteRequests requests) { diff --git a/src/coordinator/shard_map.hpp b/src/coordinator/shard_map.hpp index cf7cec541..bc891db5e 100644 --- a/src/coordinator/shard_map.hpp +++ b/src/coordinator/shard_map.hpp @@ -97,7 +97,15 @@ struct ShardMap { Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end); - Shard GetShardForKey(Label label, CompoundKey key) { return shards.at(label).at(key); } + Shard GetShardForKey(Label label, CompoundKey key) { + // return shards.at(label).at(key); + std::cout << "label" << std::endl; + auto asd1 = shards.at(label); + std::cout << "key" << std::endl; + auto asd2 = asd1[key]; + + return asd2; + } }; } // namespace memgraph::coordinator diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index 9b38ac123..ba24c4295 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -90,6 +90,7 @@ ShardMap CreateDummyShardmap(memgraph::coordinator::Address a_io_1, memgraph::co Shards shards2; shards2[cm2] = shard2; + shards[label1] = shards1; shards[label2] = shards2; return sm1; @@ -258,6 +259,11 @@ int main() { continue; } + if (!read_res_opt.value().success) { + std::cout << "Not successful." << std::endl; + continue; + } + std::cout << "Before" << std::endl; auto read_res = read_res_opt.value(); std::cout << "After" << std::endl; @@ -276,7 +282,12 @@ int main() { client_shard_map = res.fresher_shard_map.value(); std::cout << "After2" << std::endl; - auto target_shard = client_shard_map.GetShardForKey("label1", cm_k); + // TODO(gabor) check somewhere in the call chain if the entries are actually valid + for (auto &[key, val] : client_shard_map.GetShards()) { + std::cout << "key: " << key << std::endl; + } + + auto target_shard = client_shard_map.GetShardForKey(std::string("label1"), cm_k); // Determine which shard to send the requests to auto storage_client_opt = DetermineShardLocation(target_shard, a_addrs, shard_a_client, b_addrs, shard_b_client); diff --git a/tests/simulation/utils/rsm_client.hpp b/tests/simulation/utils/rsm_client.hpp index 0b4b47cae..e1614af8d 100644 --- a/tests/simulation/utils/rsm_client.hpp +++ b/tests/simulation/utils/rsm_client.hpp @@ -87,6 +87,7 @@ class RsmClient { read_req.operation = req; std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; + ResponseFuture> get_response_future = io_.template Request, ReadResponse>(leader_, read_req); From 16d1fa2c228964dad6e4f25da068a3166b085e8e Mon Sep 17 00:00:00 2001 From: gvolfing Date: Wed, 10 Aug 2022 08:33:40 +0200 Subject: [PATCH 37/39] Gerneral clean-up --- src/coordinator/coordinator.hpp | 33 ++++++------- src/io/rsm/shard_rsm.hpp | 10 ---- tests/simulation/raft.cpp | 85 -------------------------------- tests/simulation/sharded_map.cpp | 50 +++++++------------ 4 files changed, 32 insertions(+), 146 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 69a2e5cd7..259d10f9c 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -105,7 +105,6 @@ class Coordinator { /// Increment our ReadResponses Read(HlcRequest hlc_request) { - std::cout << "HlcRequest->HlcResponse" << std::endl; HlcResponse res{}; auto hlc_shard_map = shard_map_.GetHlc(); @@ -125,8 +124,6 @@ class Coordinator { } GetShardMapResponse Read(GetShardMapRequest &&get_shard_map_request) { - std::cout << "GetShardMapRequest" << std::endl; - GetShardMapResponse res; res.shard_map = shard_map_; return res; @@ -179,22 +176,22 @@ class Coordinator { explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {} ReadResponses Read(ReadRequests requests) { - if (std::get_if(&requests)) { - std::cout << "HlcRequest" << std::endl; - } else if (std::get_if(&requests)) { - std::cout << "GetShardMapRequest" << std::endl; - } else { - std::cout << "idk requests" << std::endl; - } - std::cout << "Coordinator Read()" << std::endl; + // if (std::get_if(&requests)) { + // std::cout << "HlcRequest" << std::endl; + // } else if (std::get_if(&requests)) { + // std::cout << "GetShardMapRequest" << std::endl; + // } else { + // std::cout << "idk requests" << std::endl; + // } + // std::cout << "Coordinator Read()" << std::endl; auto ret = std::visit([&](auto requests) { return Read(requests); }, (requests)); - if (std::get_if(&ret)) { - std::cout << "HlcResponse" << std::endl; - } else if (std::get_if(&ret)) { - std::cout << "GetShardMapResponse" << std::endl; - } else { - std::cout << "idk response" << std::endl; - } + // if (std::get_if(&ret)) { + // std::cout << "HlcResponse" << std::endl; + // } else if (std::get_if(&ret)) { + // std::cout << "GetShardMapResponse" << std::endl; + // } else { + // std::cout << "idk response" << std::endl; + // } return ret; } diff --git a/src/io/rsm/shard_rsm.hpp b/src/io/rsm/shard_rsm.hpp index ea45c33fc..8cce21664 100644 --- a/src/io/rsm/shard_rsm.hpp +++ b/src/io/rsm/shard_rsm.hpp @@ -90,15 +90,12 @@ class StorageRsm { StorageGetResponse ret; if (!IsKeyInRange(request.key)) { - std::cout << "ONE" << std::endl; ret.latest_known_shard_map_version = shard_map_version_; ret.shard_rsm_success = false; } else if (state_.contains(request.key)) { - std::cout << "TWO" << std::endl; ret.value = state_[request.key]; ret.shard_rsm_success = true; } else { - std::cout << "THREE" << std::endl; ret.shard_rsm_success = false; ret.value = std::nullopt; } @@ -112,12 +109,10 @@ class StorageRsm { if (!IsKeyInRange(request.key)) { ret.latest_known_shard_map_version = shard_map_version_; ret.shard_rsm_success = false; - std::cout << "WRITE 0" << std::endl; } // Key exist else if (state_.contains(request.key)) { auto &val = state_[request.key]; - std::cout << "WRITE 1" << std::endl; /* * Delete @@ -126,7 +121,6 @@ class StorageRsm { ret.shard_rsm_success = true; ret.last_value = val; state_.erase(state_.find(request.key)); - std::cout << "WRITE 2" << std::endl; } /* @@ -138,12 +132,10 @@ class StorageRsm { ret.shard_rsm_success = true; val = request.value.value(); - std::cout << "WRITE 3" << std::endl; } else { ret.last_value = val; ret.shard_rsm_success = false; - std::cout << "WRITE 4" << std::endl; } } /* @@ -154,10 +146,8 @@ class StorageRsm { ret.shard_rsm_success = true; state_.emplace(request.key, std::move(request.value).value()); - std::cout << "WRITE 5" << std::endl; } - std::cout << "WRITE ret" << std::endl; return ret; } }; diff --git a/tests/simulation/raft.cpp b/tests/simulation/raft.cpp index c0cb84476..199cce5ce 100644 --- a/tests/simulation/raft.cpp +++ b/tests/simulation/raft.cpp @@ -117,91 +117,6 @@ class TestState { } }; -// template -// class RsmClient { -// using ServerPool = std::vector
; - -// IoImpl io_; -// Address leader_; - -// std::mt19937 cli_rng_{0}; -// ServerPool server_addrs_; - -// template -// std::optional CheckForCorrectLeader(ResponseT response) { -// if (response.retry_leader) { -// MG_ASSERT(!response.success, "retry_leader should never be set for successful responses"); -// leader_ = response.retry_leader.value(); -// std::cout << "client redirected to leader server " << leader_.last_known_port << std::endl; -// } else if (!response.success) { -// std::uniform_int_distribution addr_distrib(0, (server_addrs_.size() - 1)); -// size_t addr_index = addr_distrib(cli_rng_); -// leader_ = server_addrs_[addr_index]; - -// std::cout << "client NOT redirected to leader server, trying a random one at index " << addr_index -// << " with port " << leader_.last_known_port << std::endl; -// return {}; -// } - -// return response; -// } - -// public: -// RsmClient(IoImpl &&io, Address &&leader, ServerPool &&server_addrs) -// : io_{io}, leader_{leader}, server_addrs_{server_addrs} {} - -// RsmClient() = delete; - -// std::optional> SendWriteRequest(WriteRequestT req) { -// WriteRequest client_req; -// client_req.operation = req; - -// std::cout << "client sending CasRequest to Leader " << leader_.last_known_port << std::endl; -// ResponseFuture> response_future = -// io_.template Request, WriteResponse>(leader_, client_req); -// ResponseResult> response_result = std::move(response_future).Wait(); - -// if (response_result.HasError()) { -// std::cout << "client timed out while trying to communicate with leader server " << std::endl; -// // continue; -// return std::nullopt; -// } - -// ResponseEnvelope> response_envelope = response_result.GetValue(); -// WriteResponse write_response = response_envelope.message; - -// return CheckForCorrectLeader(write_response); -// } - -// std::optional> SendReadRequest(ReadRequestT req) { -// ReadRequest read_req; -// read_req.operation = req; - -// std::cout << "client sending GetRequest to Leader " << leader_.last_known_port << std::endl; -// ResponseFuture> get_response_future = -// io_.template Request, ReadResponse>(leader_, read_req); - -// // receive response -// ResponseResult> get_response_result = std::move(get_response_future).Wait(); - -// if (get_response_result.HasError()) { -// std::cout << "client timed out while trying to communicate with leader server " << std::endl; -// return {}; -// } - -// ResponseEnvelope> get_response_envelope = get_response_result.GetValue(); -// ReadResponse read_get_response = get_response_envelope.message; - -// if (!read_get_response.success) { -// // sent to a non-leader -// return {}; -// } - -// return CheckForCorrectLeader(read_get_response); -// } -// }; - template void RunRaft(Raft server) { server.Run(); diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index ba24c4295..8b0928ecc 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -129,22 +129,13 @@ int main() { .scramble_messages = true, .rng_seed = 0, .start_time = Time::min() + std::chrono::microseconds{256 * 1024}, - .abort_time = Time::min() + std::chrono::microseconds{8 * 1024 * 1024}, + .abort_time = Time::min() + std::chrono::microseconds{2 * 8 * 1024 * 1024}, }; auto simulator = Simulator(config); Io cli_io = simulator.RegisterNew(); - // auto c_thread_1 = std::jthread(RunRaft< Coordinator>, std::move(c_1)); - // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[0]); - - // auto c_thread_2 = std::jthread(RunRaft< Coordinator>, std::move(c_2)); - // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[1]); - - // auto c_thread_3 = std::jthread(RunRaft, std::move(c_3)); - // simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); - // Register Io a_io_1 = simulator.RegisterNew(); Io a_io_2 = simulator.RegisterNew(); @@ -255,37 +246,25 @@ int main() { // Look for Shard auto read_res_opt = coordinator_client.SendReadRequest(req); if (!read_res_opt) { - std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!0" << std::endl; continue; } if (!read_res_opt.value().success) { - std::cout << "Not successful." << std::endl; continue; } - std::cout << "Before" << std::endl; auto read_res = read_res_opt.value(); - std::cout << "After" << std::endl; auto res = std::get(read_res.read_return); + // Transaction ID to be used later... auto transaction_id = res.new_hlc; - std::cout << "transaction_id: " << transaction_id.logical_id << std::endl; - - if (!res.fresher_shard_map) { - // continue; - std::cout << "Something is really not OK..." << std::endl; - } - - std::cout << "Before2" << std::endl; client_shard_map = res.fresher_shard_map.value(); - std::cout << "After2" << std::endl; // TODO(gabor) check somewhere in the call chain if the entries are actually valid - for (auto &[key, val] : client_shard_map.GetShards()) { - std::cout << "key: " << key << std::endl; - } + // for (auto &[key, val] : client_shard_map.GetShards()) { + // std::cout << "key: " << key << std::endl; + // } auto target_shard = client_shard_map.GetShardForKey(std::string("label1"), cm_k); @@ -293,9 +272,7 @@ int main() { auto storage_client_opt = DetermineShardLocation(target_shard, a_addrs, shard_a_client, b_addrs, shard_b_client); MG_ASSERT(storage_client_opt); - std::cout << "Before3" << std::endl; auto storage_client = storage_client_opt.value(); - std::cout << "After3" << std::endl; // Have client use shard map to decide which shard to communicate // with in order to write a new value @@ -307,7 +284,6 @@ int main() { storage_req.value = 1000; auto write_res_opt = storage_client.SendWriteRequest(storage_req); if (!write_res_opt) { - std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1" << std::endl; continue; } auto write_res = write_res_opt.value().write_return; @@ -325,19 +301,27 @@ int main() { storage_get_req.key = {write_key_1, write_key_2}; auto get_res_opt = storage_client.SendReadRequest(storage_get_req); if (!get_res_opt) { - std::cout << "ERROR!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!2" << std::endl; continue; } auto get_res = get_res_opt.value(); auto val = get_res.read_return.value.value(); - std::cout << "val -> " << val << std::endl; - - MG_ASSERT(get_res.read_return.value == 1000); + MG_ASSERT(val == 1000); break; } simulator.ShutDown(); + SimulatorStats stats = simulator.Stats(); + + std::cout << "total messages: " << stats.total_messages << std::endl; + std::cout << "dropped messages: " << stats.dropped_messages << std::endl; + std::cout << "timed out requests: " << stats.timed_out_requests << std::endl; + std::cout << "total requests: " << stats.total_requests << std::endl; + std::cout << "total responses: " << stats.total_responses << std::endl; + std::cout << "simulator ticks: " << stats.simulator_ticks << std::endl; + + std::cout << "========================== SUCCESS :) ==========================" << std::endl; + return 0; } From 866f1feeeb1a3eca7293a1799dc6a53404f37d50 Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Wed, 10 Aug 2022 11:29:20 +0000 Subject: [PATCH 38/39] Added request for edge id batch request to coordinator rsm --- src/coordinator/coordinator.hpp | 45 ++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/coordinator/coordinator.hpp b/src/coordinator/coordinator.hpp index 259d10f9c..fcd073d15 100644 --- a/src/coordinator/coordinator.hpp +++ b/src/coordinator/coordinator.hpp @@ -52,6 +52,15 @@ struct AllocateHlcBatchResponse { Hlc high; }; +struct AllocateEdgeIdBatchRequest { + size_t batch_size; +}; + +struct AllocateEdgeIdBatchResponse { + uint64_t low; + uint64_t high; +}; + struct SplitShardRequest { Hlc previous_shard_map_version; Label label; @@ -78,10 +87,10 @@ struct DeregisterStorageEngineResponse { bool success; }; -using WriteRequests = std::variant; -using WriteResponses = std::variant; +using WriteRequests = std::variant; +using WriteResponses = std::variant; using ReadRequests = std::variant; using ReadResponses = std::variant; @@ -103,6 +112,9 @@ class Coordinator { uint64_t highest_allocated_timestamp_; uint64_t highest_reserved_timestamp_; + /// Query engines need to periodically request batches of unique edge IDs. + uint64_t highest_allocated_edge_id_; + /// Increment our ReadResponses Read(HlcRequest hlc_request) { HlcResponse res{}; @@ -129,16 +141,31 @@ class Coordinator { return res; } - WriteResponses Apply(AllocateHlcBatchRequest &&ahr) { + WriteResponses ApplyWrite(AllocateHlcBatchRequest &&ahr) { AllocateHlcBatchResponse res{}; return res; } + WriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) { + AllocateEdgeIdBatchResponse res{}; + + uint64_t low = highest_allocated_edge_id_; + + highest_allocated_edge_id_ += ahr.batch_size; + + uint64_t high = highest_allocated_edge_id_; + + res.low = low; + res.high = high; + + return res; + } + /// This splits the shard immediately beneath the provided /// split key, keeping the assigned peers identical for now, /// but letting them be gradually migrated over time. - WriteResponses Apply(SplitShardRequest &&split_shard_request) { + WriteResponses ApplyWrite(SplitShardRequest &&split_shard_request) { SplitShardResponse res{}; if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) { @@ -153,7 +180,7 @@ class Coordinator { /// This adds the provided storage engine to the standby storage engine pool, /// which can be used to rebalance storage over time. - WriteResponses Apply(RegisterStorageEngineRequest &®ister_storage_engine_request) { + WriteResponses ApplyWrite(RegisterStorageEngineRequest &®ister_storage_engine_request) { RegisterStorageEngineResponse res{}; // TODO @@ -162,7 +189,7 @@ class Coordinator { /// This begins the process of draining the provided storage engine from all raft /// clusters that it might be participating in. - WriteResponses Apply(DeregisterStorageEngineRequest &®ister_storage_engine_request) { + WriteResponses ApplyWrite(DeregisterStorageEngineRequest &®ister_storage_engine_request) { DeregisterStorageEngineResponse res{}; // TODO // const Address &address = register_storage_engine_request.address; @@ -196,7 +223,7 @@ class Coordinator { } WriteResponses Apply(WriteRequests requests) { - return std::visit([&](auto &&requests) { return Apply(requests); }, std::move(requests)); + return std::visit([&](auto &&requests) { return ApplyWrite(std::move(requests)); }, std::move(requests)); } }; From 3c50b68954677f93269e230296f1bbb5cf059aee Mon Sep 17 00:00:00 2001 From: Tyler Neely Date: Tue, 16 Aug 2022 16:40:47 +0000 Subject: [PATCH 39/39] Small comment fix --- src/io/rsm/raft.hpp | 9 ++++++--- tests/simulation/sharded_map.cpp | 9 ++++++--- tests/unit/CMakeLists.txt | 8 +------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/io/rsm/raft.hpp b/src/io/rsm/raft.hpp index dc1a1e9b3..07b6b8a4b 100644 --- a/src/io/rsm/raft.hpp +++ b/src/io/rsm/raft.hpp @@ -156,14 +156,17 @@ struct Follower { using Role = std::variant; /* + all ReplicatedState classes should have an Apply method -that returns our WriteResponseValue: +that returns our WriteResponseValue after consensus, and +a Read method that returns our ReadResponseValue without +requiring consensus. ReadResponse Read(ReadOperation); WriteResponseValue ReplicatedState::Apply(WriteRequest); -for examples: -if the state is uint64_t, and WriteRequest is `struct PlusOne {};`, +For example: +If the state is uint64_t, and WriteRequest is `struct PlusOne {};`, and WriteResponseValue is also uint64_t (the new value), then each call to state.Apply(PlusOne{}) will return the new value after incrementing it. 0, 1, 2, 3... and this will be sent back diff --git a/tests/simulation/sharded_map.cpp b/tests/simulation/sharded_map.cpp index 8b0928ecc..e990e2045 100644 --- a/tests/simulation/sharded_map.cpp +++ b/tests/simulation/sharded_map.cpp @@ -58,6 +58,7 @@ using memgraph::io::simulator::SimulatorTransport; using StorageClient = RsmClient, StorageWriteRequest, StorageWriteResponse, StorageGetRequest, StorageGetResponse>; namespace { + ShardMap CreateDummyShardmap(memgraph::coordinator::Address a_io_1, memgraph::coordinator::Address a_io_2, memgraph::coordinator::Address a_io_3, memgraph::coordinator::Address b_io_1, memgraph::coordinator::Address b_io_2, memgraph::coordinator::Address b_io_3) { @@ -193,8 +194,6 @@ int main() { auto b_thread_3 = std::jthread(RunStorageRaft, std::move(b_3)); simulator.IncrementServerCountAndWaitForQuiescentState(b_addrs[2]); - std::cout << "beginning test after servers have become quiescent" << std::endl; - // Spin up coordinators Io c_io_1 = simulator.RegisterNew(); @@ -220,6 +219,8 @@ int main() { auto c_thread_3 = std::jthread([c_3]() mutable { c_3.Run(); }); simulator.IncrementServerCountAndWaitForQuiescentState(c_addrs[2]); + std::cout << "beginning test after servers have become quiescent" << std::endl; + // Have client contact coordinator RSM for a new transaction ID and // also get the current shard map using CoordinatorClient = @@ -259,7 +260,9 @@ int main() { // Transaction ID to be used later... auto transaction_id = res.new_hlc; - client_shard_map = res.fresher_shard_map.value(); + if (res.fresher_shard_map) { + client_shard_map = res.fresher_shard_map.value(); + } // TODO(gabor) check somewhere in the call chain if the entries are actually valid // for (auto &[key, val] : client_shard_map.GetShards()) { diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 9de4860ef..3050ed24b 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -391,12 +391,6 @@ add_custom_target(test_lcp ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/test_lcp) add_test(test_lcp ${CMAKE_CURRENT_BINARY_DIR}/test_lcp) add_dependencies(memgraph__unit test_lcp) -# Test websocket -find_package(Boost REQUIRED) - -add_unit_test(websocket.cpp) -target_link_libraries(${test_prefix}websocket mg-communication Boost::headers) - # Test future add_unit_test(future.cpp) -target_link_libraries(${test_prefix}future mg-io) \ No newline at end of file +target_link_libraries(${test_prefix}future mg-io)