add register instance and set instance to main queries and logic

.github/workflows/diff.yaml

@@ -337,6 +337,52 @@ jobs:
             # multiple paths could be defined
             build/logs
 
+  experimental_build:
+    name: "High availability build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          source /opt/toolchain-v4/activate
+          ./init
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
+          make -j$THREADS
+      - name: Run unit tests
+        run: |
+          source /opt/toolchain-v4/activate
+          cd build
+          ctest -R memgraph__unit --output-on-failure -j$THREADS
+      - name: Run e2e tests
+        run: |
+          cd tests
+          ./setup.sh /opt/toolchain-v4/activate
+          source ve3/bin/activate_e2e
+          cd e2e
+          ./run.sh "Coordinator"
+          ./run.sh "Client initiated failover"
+          ./run.sh "Uninitialized cluster"
+      - name: Save test data
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: "Test data"
+          path: |
+            # multiple paths could be defined
+            build/logs
+
   release_jepsen_test:
     name: "Release Jepsen Test"
     runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]

CMakeLists.txt

@@ -271,6 +271,17 @@ endif()
 set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
 add_subdirectory(libs EXCLUDE_FROM_ALL)
 
+option(MG_EXPERIMENTAL_HIGH_AVAILABILITY "Feature flag for experimental high availability" OFF)
+
+if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_HIGH_AVAILABILITY)
+    set(MG_EXPERIMENTAL_HIGH_AVAILABILITY OFF)
+    message(FATAL_ERROR "MG_EXPERIMENTAL_HIGH_AVAILABILITY must be used with enterpise version of the code.")
+endif ()
+
+if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
+    add_compile_definitions(MG_EXPERIMENTAL_HIGH_AVAILABILITY)
+endif ()
+
 # Optional subproject configuration -------------------------------------------
 option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
 option(TOOLS "Build tools binaries" ON)

src/CMakeLists.txt

@@ -22,6 +22,8 @@ add_subdirectory(dbms)
 add_subdirectory(flags)
 add_subdirectory(distributed)
 add_subdirectory(replication)
+add_subdirectory(coordination)
+add_subdirectory(replication_coordination_glue)
 
 string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
 

src/auth/models.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
 // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -62,7 +62,8 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
                                                  Permission::TRANSACTION_MANAGEMENT,
                                                  Permission::STORAGE_MODE,
                                                  Permission::MULTI_DATABASE_EDIT,
-                                                 Permission::MULTI_DATABASE_USE};
+                                                 Permission::MULTI_DATABASE_USE,
+                                                 Permission::COORDINATOR};
 
 }  // namespace
 
@@ -118,6 +119,8 @@ std::string PermissionToString(Permission permission) {
       return "MULTI_DATABASE_EDIT";
     case Permission::MULTI_DATABASE_USE:
       return "MULTI_DATABASE_USE";
+    case Permission::COORDINATOR:
+      return "COORDINATOR";
   }
 }
 

src/auth/models.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
 // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -48,6 +48,7 @@ enum class Permission : uint64_t {
   STORAGE_MODE = 1U << 22U,
   MULTI_DATABASE_EDIT = 1U << 23U,
   MULTI_DATABASE_USE  = 1U << 24U,
+  COORDINATOR  = 1U << 25U,
 };
 // clang-format on
 

src/coordination/CMakeLists.txt

@@ -0,0 +1,28 @@
+add_library(mg-coordination STATIC)
+add_library(mg::coordination ALIAS mg-coordination)
+target_sources(mg-coordination
+        PUBLIC
+        include/coordination/coordinator_client.hpp
+        include/coordination/coordinator_state.hpp
+        include/coordination/coordinator_rpc.hpp
+        include/coordination/coordinator_server.hpp
+        include/coordination/coordinator_config.hpp
+        include/coordination/coordinator_exceptions.hpp
+        include/coordination/coordinator_slk.hpp
+        include/coordination/coordinator_data.hpp
+        include/coordination/constants.hpp
+        include/coordination/failover_status.hpp
+        include/coordination/coordinator_client_info.hpp
+        include/coordination/coordinator_cluster_config.hpp
+
+        PRIVATE
+        coordinator_client.cpp
+        coordinator_state.cpp
+        coordinator_rpc.cpp
+        coordinator_server.cpp
+)
+target_include_directories(mg-coordination PUBLIC include)
+
+target_link_libraries(mg-coordination
+    PUBLIC mg::utils mg::rpc mg::slk mg::io mg::repl_coord_glue
+)

src/coordination/coordinator_client.cpp

@@ -0,0 +1,126 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_client.hpp"
+
+#include "coordination/coordinator_config.hpp"
+#include "coordination/coordinator_rpc.hpp"
+#include "replication_coordination_glue/messages.hpp"
+
+namespace memgraph::coordination {
+
+namespace {
+auto CreateClientContext(const memgraph::coordination::CoordinatorClientConfig &config)
+    -> communication::ClientContext {
+  return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file}
+                      : communication::ClientContext{};
+}
+}  // namespace
+
+CoordinatorClient::CoordinatorClient(CoordinatorState *coord_state, CoordinatorClientConfig config,
+                                     HealthCheckCallback succ_cb, HealthCheckCallback fail_cb)
+    : rpc_context_{CreateClientContext(config)},
+      rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port),
+                  &rpc_context_},
+      config_{std::move(config)},
+      coord_state_{coord_state},
+      succ_cb_{std::move(succ_cb)},
+      fail_cb_{std::move(fail_cb)} {}
+
+CoordinatorClient::~CoordinatorClient() {
+  auto exit_job = utils::OnScopeExit([&] {
+    replica_checker_.Stop();
+    thread_pool_.Shutdown();
+  });
+  const auto endpoint = rpc_client_.Endpoint();
+  // Logging can throw
+  spdlog::trace("Closing coordinator client on {}:{}", endpoint.address, endpoint.port);
+}
+
+void CoordinatorClient::StartFrequentCheck() {
+  MG_ASSERT(config_.health_check_frequency_sec > std::chrono::seconds(0),
+            "Health check frequency must be greater than 0");
+
+  std::string_view instance_name = config_.instance_name;
+  replica_checker_.Run("Coord checker", config_.health_check_frequency_sec, [this, instance_name] {
+    try {
+      spdlog::trace("Sending frequent heartbeat to machine {} on {}:{}", instance_name, rpc_client_.Endpoint().address,
+                    rpc_client_.Endpoint().port);
+      auto stream{rpc_client_.Stream<memgraph::replication_coordination_glue::FrequentHeartbeatRpc>()};
+      if (stream.AwaitResponse().success) {
+        succ_cb_(coord_state_, instance_name);
+      } else {
+        fail_cb_(coord_state_, instance_name);
+      }
+    } catch (const rpc::RpcFailedException &) {
+      fail_cb_(coord_state_, instance_name);
+    }
+  });
+}
+
+void CoordinatorClient::StopFrequentCheck() { replica_checker_.Stop(); }
+
+void CoordinatorClient::PauseFrequentCheck() { replica_checker_.Pause(); }
+void CoordinatorClient::ResumeFrequentCheck() { replica_checker_.Resume(); }
+
+auto CoordinatorClient::InstanceName() const -> std::string_view { return config_.instance_name; }
+auto CoordinatorClient::SocketAddress() const -> std::string { return rpc_client_.Endpoint().SocketAddress(); }
+auto CoordinatorClient::Config() const -> CoordinatorClientConfig const & { return config_; }
+auto CoordinatorClient::SuccCallback() const -> HealthCheckCallback const & { return succ_cb_; }
+auto CoordinatorClient::FailCallback() const -> HealthCheckCallback const & { return fail_cb_; }
+
+////// AF design choice
+auto CoordinatorClient::ReplicationClientInfo() const -> CoordinatorClientConfig::ReplicationClientInfo const & {
+  MG_ASSERT(config_.replication_client_info.has_value(), "No ReplicationClientInfo for MAIN instance!");
+  return *config_.replication_client_info;
+}
+
+////// AF design choice
+auto CoordinatorClient::ReplicationClientInfo() -> std::optional<CoordinatorClientConfig::ReplicationClientInfo> & {
+  MG_ASSERT(config_.replication_client_info.has_value(), "No ReplicationClientInfo for MAIN instance!");
+  return config_.replication_client_info;
+}
+
+auto CoordinatorClient::SendPromoteReplicaToMainRpc(
+    std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info) const -> bool {
+  try {
+    auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(std::move(replication_clients_info))};
+    if (!stream.AwaitResponse().success) {
+      spdlog::error("Failed to perform failover!");
+      return false;
+    }
+    spdlog::info("Sent failover RPC from coordinator to new main!");
+    return true;
+  } catch (const rpc::RpcFailedException &) {
+    spdlog::error("Failed to send failover RPC from coordinator to new main!");
+  }
+  return false;
+}
+
+auto CoordinatorClient::SendSetToReplicaRpc(CoordinatorClient::ReplClientInfo replication_client_info) const -> bool {
+  try {
+    auto stream{rpc_client_.Stream<SetMainToReplicaRpc>(std::move(replication_client_info))};
+    if (!stream.AwaitResponse().success) {
+      spdlog::error("Failed to set main to replica!");
+      return false;
+    }
+    spdlog::info("Sent request RPC from coordinator to instance to set it as replica!");
+    return true;
+  } catch (const rpc::RpcFailedException &) {
+    spdlog::error("Failed to send failover RPC from coordinator to new main!");
+  }
+  return false;
+}
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/coordinator_rpc.cpp

@@ -0,0 +1,107 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_rpc.hpp"
+
+#include "coordination/coordinator_slk.hpp"
+#include "slk/serialization.hpp"
+
+namespace memgraph {
+
+namespace coordination {
+
+void PromoteReplicaToMainReq::Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self, builder);
+}
+
+void PromoteReplicaToMainReq::Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(self, reader);
+}
+
+void PromoteReplicaToMainRes::Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self, builder);
+}
+
+void PromoteReplicaToMainRes::Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(self, reader);
+}
+
+void SetMainToReplicaReq::Save(const SetMainToReplicaReq &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self, builder);
+}
+
+void SetMainToReplicaReq::Load(SetMainToReplicaReq *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(self, reader);
+}
+
+void SetMainToReplicaRes::Save(const SetMainToReplicaRes &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self, builder);
+}
+
+void SetMainToReplicaRes::Load(SetMainToReplicaRes *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(self, reader);
+}
+
+}  // namespace coordination
+
+constexpr utils::TypeInfo coordination::PromoteReplicaToMainReq::kType{utils::TypeId::COORD_FAILOVER_REQ,
+                                                                       "CoordPromoteReplicaToMainReq", nullptr};
+
+constexpr utils::TypeInfo coordination::PromoteReplicaToMainRes::kType{utils::TypeId::COORD_FAILOVER_RES,
+                                                                       "CoordPromoteReplicaToMainRes", nullptr};
+
+constexpr utils::TypeInfo coordination::SetMainToReplicaReq::kType{utils::TypeId::COORD_SET_REPL_MAIN_REQ,
+                                                                   "CoordSetReplMainReq", nullptr};
+
+constexpr utils::TypeInfo coordination::SetMainToReplicaRes::kType{utils::TypeId::COORD_SET_REPL_MAIN_RES,
+                                                                   "CoordSetReplMainRes", nullptr};
+
+namespace slk {
+
+void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self.success, builder);
+}
+
+void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(&self->success, reader);
+}
+
+void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self.replication_clients_info, builder);
+}
+
+void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(&self->replication_clients_info, reader);
+}
+
+void Save(const memgraph::coordination::SetMainToReplicaReq &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self.replication_client_info, builder);
+}
+
+void Load(memgraph::coordination::SetMainToReplicaReq *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(&self->replication_client_info, reader);
+}
+
+void Save(const memgraph::coordination::SetMainToReplicaRes &self, memgraph::slk::Builder *builder) {
+  memgraph::slk::Save(self.success, builder);
+}
+
+void Load(memgraph::coordination::SetMainToReplicaRes *self, memgraph::slk::Reader *reader) {
+  memgraph::slk::Load(&self->success, reader);
+}
+
+}  // namespace slk
+
+}  // namespace memgraph
+
+#endif

src/coordination/coordinator_server.cpp

@@ -0,0 +1,57 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_server.hpp"
+#include "replication_coordination_glue/messages.hpp"
+
+namespace memgraph::coordination {
+
+namespace {
+
+auto CreateServerContext(const memgraph::coordination::CoordinatorServerConfig &config)
+    -> communication::ServerContext {
+  return (config.ssl) ? communication::ServerContext{config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file,
+                                                     config.ssl->verify_peer}
+                      : communication::ServerContext{};
+}
+
+// NOTE: The coordinator server doesn't more than 1 processing thread - each replica can
+// have only a single coordinator server. Also, the single-threaded guarantee
+// simplifies the rest of the implementation.
+constexpr auto kCoordinatorServerThreads = 1;
+
+}  // namespace
+
+CoordinatorServer::CoordinatorServer(const CoordinatorServerConfig &config)
+    : rpc_server_context_{CreateServerContext(config)},
+      rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_,
+                  kCoordinatorServerThreads} {
+  rpc_server_.Register<replication_coordination_glue::FrequentHeartbeatRpc>([](auto *req_reader, auto *res_builder) {
+    spdlog::debug("Received FrequentHeartbeatRpc on coordinator server");
+    replication_coordination_glue::FrequentHeartbeatHandler(req_reader, res_builder);
+  });
+}
+
+CoordinatorServer::~CoordinatorServer() {
+  if (rpc_server_.IsRunning()) {
+    auto const &endpoint = rpc_server_.endpoint();
+    spdlog::trace("Closing coordinator server on {}:{}", endpoint.address, endpoint.port);
+    rpc_server_.Shutdown();
+  }
+  rpc_server_.AwaitShutdown();
+}
+
+bool CoordinatorServer::Start() { return rpc_server_.Start(); }
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/coordinator_state.cpp

@@ -0,0 +1,474 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "coordination/coordinator_state.hpp"
+#include <algorithm>
+#include "coordination/coordinator_client_info.hpp"
+#include "coordination/register_main_replica_coordinator_status.hpp"
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_client.hpp"
+#include "coordination/coordinator_cluster_config.hpp"
+#include "coordination/coordinator_config.hpp"
+#include "flags/replication.hpp"
+#include "spdlog/spdlog.h"
+#include "utils/logging.hpp"
+#include "utils/variant_helpers.hpp"
+
+#include <atomic>
+#include <exception>
+#include <optional>
+
+namespace memgraph::coordination {
+
+namespace {
+
+bool CheckName(const std::list<CoordinatorClient> &replicas, const CoordinatorClientConfig &config) {
+  auto name_matches = [&instance_name = config.instance_name](auto const &replica) {
+    return replica.InstanceName() == instance_name;
+  };
+  return std::any_of(replicas.begin(), replicas.end(), name_matches);
+};
+
+}  // namespace
+
+CoordinatorState::CoordinatorState() {
+  MG_ASSERT(!(FLAGS_coordinator && FLAGS_coordinator_server_port),
+            "Instance cannot be a coordinator and have registered coordinator server.");
+
+  spdlog::info("Executing coordinator constructor");
+  if (FLAGS_coordinator_server_port) {
+    spdlog::info("Coordinator server port set");
+    auto const config = CoordinatorServerConfig{
+        .ip_address = kDefaultReplicationServerIp,
+        .port = static_cast<uint16_t>(FLAGS_coordinator_server_port),
+    };
+    spdlog::info("Executing coordinator constructor main replica");
+
+    data_ = CoordinatorMainReplicaData{.coordinator_server_ = std::make_unique<CoordinatorServer>(config)};
+  }
+}
+
+auto CoordinatorState::RegisterInstanceOnCoordinator(CoordinatorClientConfig config)
+    -> RegisterInstanceCoordinatorStatus {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Coordinator cannot register replica since variant holds wrong alternative");
+
+  const auto name_endpoint_status =
+      std::visit(memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
+                                               return RegisterInstanceCoordinatorStatus::NOT_COORDINATOR;
+                                             },
+                                             [&config](const CoordinatorData &coordinator_data) {
+                                               if (memgraph::coordination::CheckName(
+                                                       coordinator_data.registered_replicas_, config)) {
+                                                 return RegisterInstanceCoordinatorStatus::NAME_EXISTS;
+                                               }
+                                               return RegisterInstanceCoordinatorStatus::SUCCESS;
+                                             }},
+                 data_);
+
+  if (name_endpoint_status != RegisterInstanceCoordinatorStatus::SUCCESS) {
+    return name_endpoint_status;
+  }
+
+  auto find_client_info = [](CoordinatorState *coord_state, std::string_view instance_name) -> CoordinatorClientInfo & {
+    MG_ASSERT(std::holds_alternative<CoordinatorData>(coord_state->data_),
+              "Can't execute CoordinatorClient's callback since variant holds wrong alternative");
+    auto &coord_data = std::get<CoordinatorData>(coord_state->data_);
+    std::shared_lock<utils::RWLock> lock{coord_data.coord_data_lock_};
+
+    auto replica_client_info = std::ranges::find_if(
+        coord_data.registered_replicas_info_,
+        [instance_name](const CoordinatorClientInfo &replica) { return replica.InstanceName() == instance_name; });
+
+    if (replica_client_info != coord_data.registered_replicas_info_.end()) {
+      return *replica_client_info;
+    }
+
+    MG_ASSERT(coord_data.registered_main_info_->InstanceName() == instance_name,
+              "Instance is neither a replica nor main...");
+    return *coord_data.registered_main_info_;
+  };
+
+  // TODO MERGE WITH ANDI's WORK
+  auto repl_succ_cb = [find_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &client_info = find_client_info(coord_state, instance_name);
+    client_info.UpdateLastResponseTime();
+  };
+
+  auto repl_fail_cb = [find_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &client_info = find_client_info(coord_state, instance_name);
+    client_info.UpdateInstanceStatus();
+  };
+  CoordinatorClientConfig::ReplicationClientInfo replication_client_info = *config.replication_client_info;
+  auto *coord_client = &std::get<CoordinatorData>(data_).registered_replicas_.emplace_back(
+      this, std::move(config), std::move(repl_succ_cb), std::move(repl_fail_cb));
+
+  coord_client->SendSetToReplicaRpc(replication_client_info);
+
+  std::get<CoordinatorData>(data_).registered_replicas_info_.emplace_back(coord_client->InstanceName(),
+                                                                          coord_client->SocketAddress());
+  coord_client->StartFrequentCheck();
+
+  return RegisterInstanceCoordinatorStatus::SUCCESS;
+}
+
+auto CoordinatorState::SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Coordinator cannot register replica since variant holds wrong alternative");
+
+  // TODO: (andi) How does the situation change when restoration of main is implemented regarding callbacks?
+  // We should probably at that point search for main also in instances as for replicas...
+  auto get_client_info = [](CoordinatorState *coord_state, std::string_view instance_name) -> CoordinatorClientInfo & {
+    MG_ASSERT(std::holds_alternative<CoordinatorData>(coord_state->data_),
+              "Can't execute CoordinatorClient's callback since variant holds wrong alternative");
+    MG_ASSERT(std::get<CoordinatorData>(coord_state->data_).registered_main_info_.has_value(),
+              "Main info is not set, but callback is called");
+    auto &coord_data = std::get<CoordinatorData>(coord_state->data_);
+    std::shared_lock<utils::RWLock> lock{coord_data.coord_data_lock_};
+
+    // TODO When we will support restoration of main, we have to assert that the instance is main or replica, not at
+    // this point....
+    auto &registered_main_info = coord_data.registered_main_info_;
+    MG_ASSERT(registered_main_info->InstanceName() == instance_name,
+              "Callback called for wrong instance name: {}, expected: {}", instance_name,
+              registered_main_info->InstanceName());
+    return *registered_main_info;
+  };
+
+  auto succ_cb = [get_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &registered_main_info = get_client_info(coord_state, instance_name);
+    registered_main_info.UpdateLastResponseTime();
+  };
+
+  auto fail_cb = [this, get_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &registered_main_info = get_client_info(coord_state, instance_name);
+    if (bool main_alive = registered_main_info.UpdateInstanceStatus(); !main_alive) {
+      spdlog::warn("Main is not alive, starting failover");
+      switch (auto failover_status = DoFailover(); failover_status) {
+        using enum DoFailoverStatus;
+        case ALL_REPLICAS_DOWN:
+          spdlog::warn("Failover aborted since all replicas are down!");
+        case MAIN_ALIVE:
+          spdlog::warn("Failover aborted since main is alive!");
+        case CLUSTER_UNINITIALIZED:
+          spdlog::warn("Failover aborted since cluster is uninitialized!");
+        case SUCCESS:
+          break;
+      }
+    }
+  };
+
+  auto &registered_replicas = std::get<CoordinatorData>(data_).registered_replicas_;
+  // Find replica we already registered
+  auto registered_replica =
+      std::find_if(registered_replicas.begin(), registered_replicas.end(), [instance_name](const auto &replica_client) {
+        std::cout << "replica name: " << replica_client.InstanceName() << ", instance name: " << instance_name
+                  << std::endl;
+        return replica_client.InstanceName() == instance_name;
+      });
+
+  std::for_each(registered_replicas.begin(), registered_replicas.end(),
+                [](const auto &client) { std::cout << "replica names: " << client.InstanceName() << std::endl; });
+  // if replica not found...
+  if (registered_replica == registered_replicas.end()) {
+    spdlog::error("You didn't register instance with given name {}", instance_name);
+    return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
+  }
+  registered_replica->StopFrequentCheck();
+  // Set instance as MAIN
+  // THIS WILL SHUT DOWN CLIENT
+  auto &registered_main = std::get<CoordinatorData>(data_).registered_main_;
+  registered_main =
+      std::make_unique<CoordinatorClient>(this, registered_replica->Config(), std::move(succ_cb), std::move(fail_cb));
+
+  std::get<CoordinatorData>(data_).registered_main_info_.emplace(registered_main->InstanceName(),
+                                                                 registered_main->SocketAddress());
+  std::vector<CoordinatorClientConfig::ReplicationClientInfo> repl_clients_info;
+  repl_clients_info.reserve(registered_replicas.size() - 1);
+  std::ranges::for_each(registered_replicas,
+                        [registered_replica, &repl_clients_info](const CoordinatorClient &replica) {
+                          if (replica != *registered_replica) {
+                            repl_clients_info.emplace_back(replica.ReplicationClientInfo());
+                          }
+                        });
+
+  // PROMOTE REPLICA TO MAIN
+  // THIS SHOULD FAIL HERE IF IT IS DOWN
+  if (auto result = registered_main->SendPromoteReplicaToMainRpc(std::move(repl_clients_info)); !result) {
+    registered_replica->StartFrequentCheck();
+    registered_main.reset();
+    return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
+  }
+
+  registered_main->StartFrequentCheck();
+  registered_replicas.erase(registered_replica);
+  return SetInstanceToMainCoordinatorStatus::SUCCESS;
+}
+
+auto CoordinatorState::RegisterReplica(CoordinatorClientConfig config) -> RegisterMainReplicaCoordinatorStatus {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Coordinator cannot register replica since variant holds wrong alternative");
+
+  const auto name_endpoint_status =
+      std::visit(memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
+                                               return RegisterMainReplicaCoordinatorStatus::NOT_COORDINATOR;
+                                             },
+                                             [&config](const CoordinatorData &coordinator_data) {
+                                               if (memgraph::coordination::CheckName(
+                                                       coordinator_data.registered_replicas_, config)) {
+                                                 return RegisterMainReplicaCoordinatorStatus::NAME_EXISTS;
+                                               }
+                                               return RegisterMainReplicaCoordinatorStatus::SUCCESS;
+                                             }},
+                 data_);
+
+  if (name_endpoint_status != RegisterMainReplicaCoordinatorStatus::SUCCESS) {
+    return name_endpoint_status;
+  }
+
+  // TODO: Refactor so that we just know about instances, nothing more. Design struct Instance which will know about
+  // role... From CoordinatorState perspective there should just be instances, no specific handling of main vs. replica.
+  // Here after failover it can happen that replica
+  // has become main MG_ASSERT(replica_client_info != registered_replicas_info.end(), "Replica {} not found in
+  // registered replicas info",
+  //          instance_name);
+  auto find_client_info = [](CoordinatorState *coord_state, std::string_view instance_name) -> CoordinatorClientInfo & {
+    MG_ASSERT(std::holds_alternative<CoordinatorData>(coord_state->data_),
+              "Can't execute CoordinatorClient's callback since variant holds wrong alternative");
+    auto &coord_data = std::get<CoordinatorData>(coord_state->data_);
+    std::shared_lock<utils::RWLock> lock{coord_data.coord_data_lock_};
+
+    auto replica_client_info = std::ranges::find_if(
+        coord_data.registered_replicas_info_,
+        [instance_name](const CoordinatorClientInfo &replica) { return replica.InstanceName() == instance_name; });
+
+    if (replica_client_info != coord_data.registered_replicas_info_.end()) {
+      return *replica_client_info;
+    }
+
+    MG_ASSERT(coord_data.registered_main_info_->InstanceName() == instance_name,
+              "Instance is neither a replica nor main...");
+    return *coord_data.registered_main_info_;
+  };
+
+  auto repl_succ_cb = [find_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &client_info = find_client_info(coord_state, instance_name);
+    client_info.UpdateLastResponseTime();
+  };
+
+  auto repl_fail_cb = [find_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &client_info = find_client_info(coord_state, instance_name);
+    client_info.UpdateInstanceStatus();
+  };
+
+  auto *coord_client = &std::get<CoordinatorData>(data_).registered_replicas_.emplace_back(
+      this, std::move(config), std::move(repl_succ_cb), std::move(repl_fail_cb));
+
+  std::get<CoordinatorData>(data_).registered_replicas_info_.emplace_back(coord_client->InstanceName(),
+                                                                          coord_client->SocketAddress());
+  coord_client->StartFrequentCheck();
+
+  return RegisterMainReplicaCoordinatorStatus::SUCCESS;
+}
+
+auto CoordinatorState::RegisterMain(CoordinatorClientConfig config) -> RegisterMainReplicaCoordinatorStatus {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Coordinator cannot register main since variant holds wrong alternative");
+
+  const auto endpoint_status = std::visit(
+      memgraph::utils::Overloaded{
+          [](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
+            return RegisterMainReplicaCoordinatorStatus::NOT_COORDINATOR;
+          },
+          [](const CoordinatorData & /*coordinator_data*/) { return RegisterMainReplicaCoordinatorStatus::SUCCESS; }},
+      data_);
+
+  if (endpoint_status != RegisterMainReplicaCoordinatorStatus::SUCCESS) {
+    return endpoint_status;
+  }
+
+  // TODO: (andi) How does the situation change when restoration of main is implemented regarding callbacks?
+  // We should probably at that point search for main also in instances as for replicas...
+  auto get_client_info = [](CoordinatorState *coord_state, std::string_view instance_name) -> CoordinatorClientInfo & {
+    MG_ASSERT(std::holds_alternative<CoordinatorData>(coord_state->data_),
+              "Can't execute CoordinatorClient's callback since variant holds wrong alternative");
+    MG_ASSERT(std::get<CoordinatorData>(coord_state->data_).registered_main_info_.has_value(),
+              "Main info is not set, but callback is called");
+    auto &coord_data = std::get<CoordinatorData>(coord_state->data_);
+    std::shared_lock<utils::RWLock> lock{coord_data.coord_data_lock_};
+
+    // TODO When we will support restoration of main, we have to assert that the instance is main or replica, not at
+    // this point....
+    auto &registered_main_info = coord_data.registered_main_info_;
+    MG_ASSERT(registered_main_info->InstanceName() == instance_name,
+              "Callback called for wrong instance name: {}, expected: {}", instance_name,
+              registered_main_info->InstanceName());
+    return *registered_main_info;
+  };
+
+  auto succ_cb = [get_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &registered_main_info = get_client_info(coord_state, instance_name);
+    registered_main_info.UpdateLastResponseTime();
+  };
+
+  auto fail_cb = [this, get_client_info](CoordinatorState *coord_state, std::string_view instance_name) -> void {
+    auto &registered_main_info = get_client_info(coord_state, instance_name);
+    if (bool main_alive = registered_main_info.UpdateInstanceStatus(); !main_alive) {
+      spdlog::warn("Main is not alive, starting failover");
+      switch (auto failover_status = DoFailover(); failover_status) {
+        using enum DoFailoverStatus;
+        case ALL_REPLICAS_DOWN:
+          spdlog::warn("Failover aborted since all replicas are down!");
+        case MAIN_ALIVE:
+          spdlog::warn("Failover aborted since main is alive!");
+        case CLUSTER_UNINITIALIZED:
+          spdlog::warn("Failover aborted since cluster is uninitialized!");
+        case SUCCESS:
+          break;
+      }
+    }
+  };
+
+  auto &registered_main = std::get<CoordinatorData>(data_).registered_main_;
+  registered_main =
+      std::make_unique<CoordinatorClient>(this, std::move(config), std::move(succ_cb), std::move(fail_cb));
+
+  std::get<CoordinatorData>(data_).registered_main_info_.emplace(registered_main->InstanceName(),
+                                                                 registered_main->SocketAddress());
+  registered_main->StartFrequentCheck();
+
+  return RegisterMainReplicaCoordinatorStatus::SUCCESS;
+}
+
+auto CoordinatorState::ShowReplicas() const -> std::vector<CoordinatorInstanceStatus> {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Can't call show replicas on data_, as variant holds wrong alternative");
+  const auto &registered_replicas_info = std::get<CoordinatorData>(data_).registered_replicas_info_;
+  std::vector<CoordinatorInstanceStatus> instances_status;
+  instances_status.reserve(registered_replicas_info.size());
+
+  std::ranges::transform(registered_replicas_info, std::back_inserter(instances_status),
+                         [](const CoordinatorClientInfo &coord_client_info) {
+                           return CoordinatorInstanceStatus{.instance_name = coord_client_info.InstanceName(),
+                                                            .socket_address = coord_client_info.SocketAddress(),
+                                                            .is_alive = coord_client_info.IsAlive()};
+                         });
+  return instances_status;
+}
+
+auto CoordinatorState::ShowMain() const -> std::optional<CoordinatorInstanceStatus> {
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
+            "Can't call show main on data_, as variant holds wrong alternative");
+  const auto &main = std::get<CoordinatorData>(data_).registered_main_info_;
+  if (!main.has_value()) {
+    return std::nullopt;
+  }
+
+  return CoordinatorInstanceStatus{
+      .instance_name = main->InstanceName(), .socket_address = main->SocketAddress(), .is_alive = main->IsAlive()};
+};
+
+[[nodiscard]] auto CoordinatorState::DoFailover() -> DoFailoverStatus {
+  // 1. MAIN is already down, stop sending frequent checks
+  // 2. find new replica (coordinator)
+  // 3. make copy replica's client as potential new main client (coordinator)
+  // 4. send failover RPC to new main (coordinator and new main)
+  // 5. exchange old main to new main (coordinator)
+  // 6. remove replica which was promoted to main from all replicas -> this will shut down RPC frequent check client
+  // (coordinator)
+  // 7. for new main start frequent checks (coordinator)
+  using ReplicationClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
+  MG_ASSERT(std::holds_alternative<CoordinatorData>(data_), "Cannot do failover since variant holds wrong alternative");
+  auto &coord_state = std::get<CoordinatorData>(data_);
+
+  // std::lock_guard<utils::RWLock> lock{coord_state.coord_data_lock_};
+
+  // 1.
+  auto &current_main_info = coord_state.registered_main_info_;
+
+  if (!current_main_info.has_value()) {
+    return DoFailoverStatus::CLUSTER_UNINITIALIZED;
+  }
+
+  if (current_main_info->IsAlive()) {
+    return DoFailoverStatus::MAIN_ALIVE;
+  }
+
+  auto &current_main = coord_state.registered_main_;
+  // TODO: stop pinging as soon as you figure out that failover is needed
+  current_main->PauseFrequentCheck();
+
+  // 2.
+  // Get all replicas and find new main
+  auto &registered_replicas_info = coord_state.registered_replicas_info_;
+
+  const auto chosen_replica_info = std::ranges::find_if(
+      registered_replicas_info, [](const CoordinatorClientInfo &client_info) { return client_info.IsAlive(); });
+  if (chosen_replica_info == registered_replicas_info.end()) {
+    return DoFailoverStatus::ALL_REPLICAS_DOWN;
+  }
+
+  auto &registered_replicas = coord_state.registered_replicas_;
+  auto chosen_replica =
+      std::ranges::find_if(registered_replicas, [&chosen_replica_info](const CoordinatorClient &replica) {
+        return replica.InstanceName() == chosen_replica_info->InstanceName();
+      });
+  MG_ASSERT(chosen_replica != registered_replicas.end(), "Chosen replica {} not found in registered replicas",
+            chosen_replica_info->InstanceName());
+
+  std::vector<ReplicationClientInfo> repl_clients_info;
+  repl_clients_info.reserve(registered_replicas.size() - 1);
+  std::ranges::for_each(registered_replicas, [&chosen_replica, &repl_clients_info](const CoordinatorClient &replica) {
+    if (replica != *chosen_replica) {
+      repl_clients_info.emplace_back(replica.ReplicationClientInfo());
+    }
+  });
+
+  // 3.
+  // Set on coordinator data of new main
+  // allocate resources for new main, clear replication info on this replica as main
+  // set last response time
+  auto potential_new_main = std::make_unique<CoordinatorClient>(
+      this, chosen_replica->Config(), chosen_replica->SuccCallback(), chosen_replica->FailCallback());
+  potential_new_main->ReplicationClientInfo().reset();
+  auto potential_new_main_info = *chosen_replica_info;
+
+  // 4.
+  if (!chosen_replica->SendPromoteReplicaToMainRpc(std::move(repl_clients_info))) {
+    spdlog::error("Sent RPC message, but exception was caught, aborting Failover");
+    // TODO: new status and rollback all changes that were done...
+    MG_ASSERT(false, "RPC message failed");
+  }
+
+  // 5.
+  current_main = std::move(potential_new_main);
+  current_main_info.emplace(potential_new_main_info);
+
+  // 6. remove old replica
+  // TODO: Stop pinging chosen_replica before failover.
+  // Check that it doesn't fail when you call StopFrequentCheck if it is already stopped
+  registered_replicas.erase(chosen_replica);
+  registered_replicas_info.erase(chosen_replica_info);
+
+  current_main->ResumeFrequentCheck();
+
+  return DoFailoverStatus::SUCCESS;
+}
+
+auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
+  MG_ASSERT(std::holds_alternative<CoordinatorMainReplicaData>(data_),
+            "Cannot get coordinator server since variant holds wrong alternative");
+  return *std::get<CoordinatorMainReplicaData>(data_).coordinator_server_;
+}
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/constants.hpp

@@ -0,0 +1,22 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+namespace memgraph::coordination {
+
+#ifdef MG_EXPERIMENTAL_HIGH_AVAILABILITY
+constexpr bool allow_ha = true;
+#else
+constexpr bool allow_ha = false;
+#endif
+
+}  // namespace memgraph::coordination

src/coordination/include/coordination/coordinator_client.hpp

@@ -0,0 +1,82 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_config.hpp"
+#include "rpc/client.hpp"
+#include "utils/scheduler.hpp"
+#include "utils/thread_pool.hpp"
+
+#include <string_view>
+
+namespace memgraph::coordination {
+
+class CoordinatorState;
+
+class CoordinatorClient {
+ public:
+  using ReplClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
+  using ReplicationClientsInfo = std::vector<ReplClientInfo>;
+
+  using HealthCheckCallback = std::function<void(CoordinatorState *, std::string_view)>;
+
+  explicit CoordinatorClient(CoordinatorState *coord_state_, CoordinatorClientConfig config,
+                             HealthCheckCallback succ_cb, HealthCheckCallback fail_cb);
+
+  ~CoordinatorClient();
+
+  CoordinatorClient(CoordinatorClient &) = delete;
+  CoordinatorClient &operator=(CoordinatorClient const &) = delete;
+
+  CoordinatorClient(CoordinatorClient &&) noexcept = delete;
+  CoordinatorClient &operator=(CoordinatorClient &&) noexcept = delete;
+
+  void StartFrequentCheck();
+  void StopFrequentCheck();
+  void PauseFrequentCheck();
+  void ResumeFrequentCheck();
+
+  auto SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool;
+
+  auto SendSetToReplicaRpc(ReplClientInfo replication_client_info) const -> bool;
+
+  auto InstanceName() const -> std::string_view;
+  auto SocketAddress() const -> std::string;
+  auto Config() const -> CoordinatorClientConfig const &;
+
+  auto ReplicationClientInfo() const -> ReplClientInfo const &;
+  auto ReplicationClientInfo() -> std::optional<ReplClientInfo> &;
+
+  auto SuccCallback() const -> HealthCheckCallback const &;
+  auto FailCallback() const -> HealthCheckCallback const &;
+
+  friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
+    return first.config_ == second.config_;
+  }
+
+ private:
+  utils::ThreadPool thread_pool_{1};
+  utils::Scheduler replica_checker_;
+
+  communication::ClientContext rpc_context_;
+  mutable rpc::Client rpc_client_;
+
+  CoordinatorClientConfig config_;
+  CoordinatorState *coord_state_;
+  HealthCheckCallback succ_cb_;
+  HealthCheckCallback fail_cb_;
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_client_info.hpp

@@ -0,0 +1,87 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_cluster_config.hpp"
+#include "io/network/endpoint.hpp"
+
+#include <atomic>
+#include <chrono>
+
+namespace memgraph::coordination {
+
+class CoordinatorClientInfo {
+ public:
+  CoordinatorClientInfo(std::string_view instance_name, std::string_view socket_address)
+      : last_response_time_(std::chrono::system_clock::now()),
+        is_alive_(true),  // TODO: (andi) Maybe it should be false until the first ping
+        instance_name_(instance_name),
+        socket_address_(socket_address) {}
+
+  ~CoordinatorClientInfo() = default;
+
+  CoordinatorClientInfo(const CoordinatorClientInfo &other)
+      : last_response_time_(other.last_response_time_.load()),
+        is_alive_(other.is_alive_.load()),
+        instance_name_(other.instance_name_),
+        socket_address_(other.socket_address_) {}
+
+  CoordinatorClientInfo &operator=(const CoordinatorClientInfo &other) {
+    if (this != &other) {
+      last_response_time_ = other.last_response_time_.load();
+      is_alive_ = other.is_alive_.load();
+      instance_name_ = other.instance_name_;
+      socket_address_ = other.socket_address_;
+    }
+    return *this;
+  }
+
+  CoordinatorClientInfo(CoordinatorClientInfo &&other) noexcept
+      : last_response_time_(other.last_response_time_.load()),
+        is_alive_(other.is_alive_.load()),
+        instance_name_(other.instance_name_),
+        socket_address_(other.socket_address_) {}
+
+  CoordinatorClientInfo &operator=(CoordinatorClientInfo &&other) noexcept {
+    if (this != &other) {
+      last_response_time_.store(other.last_response_time_.load());
+      is_alive_ = other.is_alive_.load();
+      instance_name_ = other.instance_name_;
+      socket_address_ = other.socket_address_;
+    }
+    return *this;
+  }
+
+  auto UpdateInstanceStatus() -> bool {
+    is_alive_ = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() -
+                                                                 last_response_time_.load(std::memory_order_acquire))
+                    .count() < CoordinatorClusterConfig::alive_response_time_difference_sec_;
+    return is_alive_;
+  }
+
+  auto UpdateLastResponseTime() -> void { last_response_time_ = std::chrono::system_clock::now(); }
+  auto InstanceName() const -> std::string_view { return instance_name_; }
+  auto IsAlive() const -> bool { return is_alive_; }
+  auto SocketAddress() const -> std::string_view { return socket_address_; }
+
+ private:
+  std::atomic<std::chrono::system_clock::time_point> last_response_time_{};
+  std::atomic<bool> is_alive_{false};
+  std::string_view instance_name_;
+  std::string socket_address_;
+};
+
+}  // namespace memgraph::coordination
+
+#endif

src/coordination/include/coordination/coordinator_cluster_config.hpp

@@ -0,0 +1,22 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+namespace memgraph::coordination {
+
+struct CoordinatorClusterConfig {
+  static constexpr int alive_response_time_difference_sec_{5};
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_config.hpp

@@ -0,0 +1,75 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "replication_coordination_glue/mode.hpp"
+
+#include <chrono>
+#include <cstdint>
+#include <optional>
+#include <string>
+
+namespace memgraph::coordination {
+
+inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
+
+struct CoordinatorClientConfig {
+  std::string instance_name;
+  std::string ip_address;
+  uint16_t port{};
+  std::chrono::seconds health_check_frequency_sec{1};
+
+  // Info which coordinator will send to new main when performing failover
+  struct ReplicationClientInfo {
+    // Must be the same as CoordinatorClientConfig's instance_name
+    std::string instance_name;
+    replication_coordination_glue::ReplicationMode replication_mode{};
+    std::string replication_ip_address;
+    uint16_t replication_port{};
+
+    friend bool operator==(ReplicationClientInfo const &, ReplicationClientInfo const &) = default;
+  };
+
+  std::optional<ReplicationClientInfo> replication_client_info;
+
+  struct SSL {
+    std::string key_file;
+    std::string cert_file;
+
+    friend bool operator==(const SSL &, const SSL &) = default;
+  };
+
+  std::optional<SSL> ssl;
+
+  friend bool operator==(CoordinatorClientConfig const &, CoordinatorClientConfig const &) = default;
+};
+
+struct CoordinatorServerConfig {
+  std::string ip_address;
+  uint16_t port{};
+  struct SSL {
+    std::string key_file;
+    std::string cert_file;
+    std::string ca_file;
+    bool verify_peer{};
+    friend bool operator==(SSL const &, SSL const &) = default;
+  };
+
+  std::optional<SSL> ssl;
+
+  friend bool operator==(CoordinatorServerConfig const &, CoordinatorServerConfig const &) = default;
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_data.hpp

@@ -0,0 +1,41 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_client.hpp"
+#include "coordination/coordinator_client_info.hpp"
+#include "coordination/coordinator_server.hpp"
+#include "utils/rw_lock.hpp"
+
+#include <list>
+#include <memory>
+#include <optional>
+
+namespace memgraph::coordination {
+
+struct CoordinatorData {
+  std::list<CoordinatorClient> registered_replicas_;
+  std::list<CoordinatorClientInfo> registered_replicas_info_;
+  std::unique_ptr<CoordinatorClient> registered_main_;
+  std::optional<CoordinatorClientInfo> registered_main_info_;
+
+  mutable utils::RWLock coord_data_lock_{utils::RWLock::Priority::READ};
+};
+
+struct CoordinatorMainReplicaData {
+  std::unique_ptr<CoordinatorServer> coordinator_server_;
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_exceptions.hpp

@@ -0,0 +1,32 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "utils/exceptions.hpp"
+
+namespace memgraph::coordination {
+class CoordinatorFailoverException final : public utils::BasicException {
+ public:
+  explicit CoordinatorFailoverException(const std::string_view what) noexcept
+      : BasicException("Failover didn't complete successfully: " + std::string(what)) {}
+
+  template <class... Args>
+  explicit CoordinatorFailoverException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
+      : CoordinatorFailoverException(fmt::format(fmt, std::forward<Args>(args)...)) {}
+
+  SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorFailoverException)
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_instance_status.hpp

@@ -0,0 +1,30 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "io/network/endpoint.hpp"
+
+#include <string_view>
+
+namespace memgraph::coordination {
+
+struct CoordinatorInstanceStatus {
+  std::string_view instance_name;
+  std::string_view socket_address;
+  bool is_alive;
+};
+
+}  // namespace memgraph::coordination
+
+#endif

src/coordination/include/coordination/coordinator_rpc.hpp

@@ -0,0 +1,104 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_config.hpp"
+#include "rpc/messages.hpp"
+#include "slk/serialization.hpp"
+
+namespace memgraph::coordination {
+
+struct PromoteReplicaToMainReq {
+  static const utils::TypeInfo kType;
+  static const utils::TypeInfo &GetTypeInfo() { return kType; }
+
+  static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
+  static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
+
+  explicit PromoteReplicaToMainReq(std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
+      : replication_clients_info(std::move(replication_clients_info)) {}
+  PromoteReplicaToMainReq() = default;
+
+  std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
+};
+
+struct PromoteReplicaToMainRes {
+  static const utils::TypeInfo kType;
+  static const utils::TypeInfo &GetTypeInfo() { return kType; }
+
+  static void Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
+  static void Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
+
+  explicit PromoteReplicaToMainRes(bool success) : success(success) {}
+  PromoteReplicaToMainRes() = default;
+
+  bool success;
+};
+
+using PromoteReplicaToMainRpc = rpc::RequestResponse<PromoteReplicaToMainReq, PromoteReplicaToMainRes>;
+
+struct SetMainToReplicaReq {
+  static const utils::TypeInfo kType;
+  static const utils::TypeInfo &GetTypeInfo() { return kType; }
+
+  static void Load(SetMainToReplicaReq *self, memgraph::slk::Reader *reader);
+  static void Save(const SetMainToReplicaReq &self, memgraph::slk::Builder *builder);
+
+  explicit SetMainToReplicaReq(CoordinatorClientConfig::ReplicationClientInfo replication_client_info)
+      : replication_client_info(std::move(replication_client_info)) {}
+
+  SetMainToReplicaReq() = default;
+
+  CoordinatorClientConfig::ReplicationClientInfo replication_client_info;
+};
+
+struct SetMainToReplicaRes {
+  static const utils::TypeInfo kType;
+  static const utils::TypeInfo &GetTypeInfo() { return kType; }
+
+  static void Load(SetMainToReplicaRes *self, memgraph::slk::Reader *reader);
+  static void Save(const SetMainToReplicaRes &self, memgraph::slk::Builder *builder);
+
+  explicit SetMainToReplicaRes(bool success) : success(success) {}
+  SetMainToReplicaRes() = default;
+
+  bool success;
+};
+
+using SetMainToReplicaRpc = rpc::RequestResponse<SetMainToReplicaReq, SetMainToReplicaRes>;
+
+}  // namespace memgraph::coordination
+
+// SLK serialization declarations
+namespace memgraph::slk {
+
+void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
+
+void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
+
+void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
+
+void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
+
+void Save(const memgraph::coordination::SetMainToReplicaRes &self, memgraph::slk::Builder *builder);
+
+void Load(memgraph::coordination::SetMainToReplicaRes *self, memgraph::slk::Reader *reader);
+
+void Save(const memgraph::coordination::SetMainToReplicaReq &self, memgraph::slk::Builder *builder);
+
+void Load(memgraph::coordination::SetMainToReplicaReq *self, memgraph::slk::Reader *reader);
+
+}  // namespace memgraph::slk
+
+#endif

src/coordination/include/coordination/coordinator_server.hpp

@@ -0,0 +1,44 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_config.hpp"
+#include "rpc/server.hpp"
+
+namespace memgraph::coordination {
+
+class CoordinatorServer {
+ public:
+  explicit CoordinatorServer(const CoordinatorServerConfig &config);
+  CoordinatorServer(const CoordinatorServer &) = delete;
+  CoordinatorServer(CoordinatorServer &&) = delete;
+  CoordinatorServer &operator=(const CoordinatorServer &) = delete;
+  CoordinatorServer &operator=(CoordinatorServer &&) = delete;
+
+  virtual ~CoordinatorServer();
+
+  bool Start();
+
+  template <typename TRequestResponse, typename F>
+  void Register(F &&callback) {
+    rpc_server_.Register<TRequestResponse>(std::forward<F>(callback));
+  }
+
+ private:
+  communication::ServerContext rpc_server_context_;
+  rpc::Server rpc_server_;
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/coordinator_slk.hpp

@@ -0,0 +1,38 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_config.hpp"
+#include "slk/serialization.hpp"
+#include "slk/streams.hpp"
+
+namespace memgraph::slk {
+
+using ReplicationClientInfo = coordination::CoordinatorClientConfig::ReplicationClientInfo;
+
+inline void Save(const ReplicationClientInfo &obj, Builder *builder) {
+  Save(obj.instance_name, builder);
+  Save(obj.replication_mode, builder);
+  Save(obj.replication_ip_address, builder);
+  Save(obj.replication_port, builder);
+}
+
+inline void Load(ReplicationClientInfo *obj, Reader *reader) {
+  Load(&obj->instance_name, reader);
+  Load(&obj->replication_mode, reader);
+  Load(&obj->replication_ip_address, reader);
+  Load(&obj->replication_port, reader);
+}
+}  // namespace memgraph::slk
+#endif

src/coordination/include/coordination/coordinator_state.hpp

@@ -0,0 +1,66 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "coordination/coordinator_client.hpp"
+#include "coordination/coordinator_client_info.hpp"
+#include "coordination/coordinator_data.hpp"
+#include "coordination/coordinator_instance_status.hpp"
+#include "coordination/coordinator_server.hpp"
+#include "coordination/failover_status.hpp"
+#include "coordination/register_main_replica_coordinator_status.hpp"
+#include "rpc/server.hpp"
+#include "utils/result.hpp"
+#include "utils/rw_spin_lock.hpp"
+#include "utils/synchronized.hpp"
+
+#include <list>
+#include <variant>
+
+namespace memgraph::coordination {
+
+class CoordinatorState {
+ public:
+  CoordinatorState();
+  ~CoordinatorState() = default;
+
+  CoordinatorState(const CoordinatorState &) = delete;
+  CoordinatorState &operator=(const CoordinatorState &) = delete;
+
+  CoordinatorState(CoordinatorState &&) noexcept = delete;
+  CoordinatorState &operator=(CoordinatorState &&) noexcept = delete;
+
+  [[nodiscard]] auto RegisterReplica(CoordinatorClientConfig config) -> RegisterMainReplicaCoordinatorStatus;
+
+  [[nodiscard]] auto RegisterMain(CoordinatorClientConfig config) -> RegisterMainReplicaCoordinatorStatus;
+
+  [[nodiscard]] auto RegisterInstanceOnCoordinator(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
+
+  [[nodiscard]] auto SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
+
+  auto ShowReplicas() const -> std::vector<CoordinatorInstanceStatus>;
+
+  auto ShowMain() const -> std::optional<CoordinatorInstanceStatus>;
+
+  // The client code must check that the server exists before calling this method.
+  auto GetCoordinatorServer() const -> CoordinatorServer &;
+
+  [[nodiscard]] auto DoFailover() -> DoFailoverStatus;
+
+ private:
+  std::variant<CoordinatorData, CoordinatorMainReplicaData> data_;
+};
+
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/failover_status.hpp

@@ -0,0 +1,21 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include <cstdint>
+
+namespace memgraph::coordination {
+enum class DoFailoverStatus : uint8_t { SUCCESS, ALL_REPLICAS_DOWN, MAIN_ALIVE, CLUSTER_UNINITIALIZED };
+}  // namespace memgraph::coordination
+#endif

src/coordination/include/coordination/register_main_replica_coordinator_status.hpp

@@ -0,0 +1,44 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include <cstdint>
+
+namespace memgraph::coordination {
+
+enum class RegisterMainReplicaCoordinatorStatus : uint8_t {
+  NAME_EXISTS,
+  END_POINT_EXISTS,
+  COULD_NOT_BE_PERSISTED,
+  NOT_COORDINATOR,
+  SUCCESS
+};
+
+enum class RegisterInstanceCoordinatorStatus : uint8_t {
+  NAME_EXISTS,
+  END_POINT_EXISTS,
+  COULD_NOT_BE_PERSISTED,
+  NOT_COORDINATOR,
+  SUCCESS
+};
+
+enum class SetInstanceToMainCoordinatorStatus : uint8_t {
+  NO_INSTANCE_WITH_NAME,
+  NOT_COORDINATOR,
+  SUCCESS,
+  COULD_NOT_PROMOTE_TO_MAIN,
+};
+
+}  // namespace memgraph::coordination
+#endif

src/dbms/CMakeLists.txt

@@ -1,3 +1,2 @@
-
-add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp)
-target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query)
+add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp coordinator_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp coordinator_handlers.cpp)
+target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query mg-replication mg-coordination)

src/dbms/coordinator_handler.cpp

@@ -0,0 +1,57 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "coordination/register_main_replica_coordinator_status.hpp"
+#ifdef MG_ENTERPRISE
+
+#include "dbms/coordinator_handler.hpp"
+
+#include "dbms/dbms_handler.hpp"
+
+namespace memgraph::dbms {
+
+CoordinatorHandler::CoordinatorHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
+
+auto CoordinatorHandler::RegisterReplicaOnCoordinator(coordination::CoordinatorClientConfig config)
+    -> coordination::RegisterMainReplicaCoordinatorStatus {
+  return dbms_handler_.CoordinatorState().RegisterReplica(std::move(config));
+}
+
+auto CoordinatorHandler::RegisterMainOnCoordinator(memgraph::coordination::CoordinatorClientConfig config)
+    -> coordination::RegisterMainReplicaCoordinatorStatus {
+  return dbms_handler_.CoordinatorState().RegisterMain(std::move(config));
+}
+
+auto CoordinatorHandler::RegisterInstanceOnCoordinator(memgraph::coordination::CoordinatorClientConfig config)
+    -> coordination::RegisterInstanceCoordinatorStatus {
+  return dbms_handler_.CoordinatorState().RegisterInstanceOnCoordinator(std::move(config));
+}
+
+auto CoordinatorHandler::SetInstanceToMain(std::string instance_name)
+    -> coordination::SetInstanceToMainCoordinatorStatus {
+  return dbms_handler_.CoordinatorState().SetInstanceToMain(std::move(instance_name));
+}
+
+auto CoordinatorHandler::ShowReplicasOnCoordinator() const -> std::vector<coordination::CoordinatorInstanceStatus> {
+  return dbms_handler_.CoordinatorState().ShowReplicas();
+}
+
+auto CoordinatorHandler::ShowMainOnCoordinator() const -> std::optional<coordination::CoordinatorInstanceStatus> {
+  return dbms_handler_.CoordinatorState().ShowMain();
+}
+
+auto CoordinatorHandler::DoFailover() const -> coordination::DoFailoverStatus {
+  return dbms_handler_.CoordinatorState().DoFailover();
+}
+
+}  // namespace memgraph::dbms
+
+#endif

src/dbms/coordinator_handler.hpp

@@ -0,0 +1,57 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "utils/result.hpp"
+
+#include "coordination/coordinator_config.hpp"
+#include "coordination/coordinator_instance_status.hpp"
+#include "coordination/failover_status.hpp"
+#include "coordination/register_main_replica_coordinator_status.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <vector>
+
+namespace memgraph::dbms {
+
+class DbmsHandler;
+
+class CoordinatorHandler {
+ public:
+  explicit CoordinatorHandler(DbmsHandler &dbms_handler);
+
+  auto RegisterReplicaOnCoordinator(coordination::CoordinatorClientConfig config)
+      -> coordination::RegisterMainReplicaCoordinatorStatus;
+
+  auto RegisterMainOnCoordinator(coordination::CoordinatorClientConfig config)
+      -> coordination::RegisterMainReplicaCoordinatorStatus;
+
+  auto RegisterInstanceOnCoordinator(coordination::CoordinatorClientConfig config)
+      -> coordination::RegisterInstanceCoordinatorStatus;
+
+  auto SetInstanceToMain(std::string instance_name) -> coordination::SetInstanceToMainCoordinatorStatus;
+
+  auto ShowReplicasOnCoordinator() const -> std::vector<coordination::CoordinatorInstanceStatus>;
+
+  auto ShowMainOnCoordinator() const -> std::optional<coordination::CoordinatorInstanceStatus>;
+
+  auto DoFailover() const -> coordination::DoFailoverStatus;
+
+ private:
+  DbmsHandler &dbms_handler_;
+};
+
+}  // namespace memgraph::dbms
+#endif

src/dbms/coordinator_handlers.cpp

@@ -0,0 +1,136 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#ifdef MG_ENTERPRISE
+
+#include "dbms/coordinator_handlers.hpp"
+#include "dbms/utils.hpp"
+
+#include "coordination/coordinator_exceptions.hpp"
+#include "coordination/coordinator_rpc.hpp"
+#include "dbms/dbms_handler.hpp"
+
+namespace memgraph::dbms {
+
+void CoordinatorHandlers::Register(DbmsHandler &dbms_handler) {
+  auto &server = dbms_handler.CoordinatorState().GetCoordinatorServer();
+
+  server.Register<coordination::PromoteReplicaToMainRpc>(
+      [&dbms_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
+        spdlog::info("Received PromoteReplicaToMainRpc from coordinator server");
+        CoordinatorHandlers::PromoteReplicaToMainHandler(dbms_handler, req_reader, res_builder);
+      });
+
+  server.Register<coordination::SetMainToReplicaRpc>(
+      [&dbms_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
+        spdlog::info("Received PromoteReplicaToMainRpc from coordinator server");
+        CoordinatorHandlers::SetMainToReplicaHandler(dbms_handler, req_reader, res_builder);
+      });
+}
+
+void CoordinatorHandlers::SetMainToReplicaHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
+                                                  slk::Builder *res_builder) {
+  auto &repl_state = dbms_handler.ReplicationState();
+
+  if (!repl_state.IsMain()) {
+    spdlog::error("Setting to replica must be performed on main.");
+    slk::Save(coordination::SetMainToReplicaRes{false}, res_builder);
+    return;
+  }
+
+  coordination::SetMainToReplicaReq req;
+  slk::Load(&req, req_reader);
+
+  replication::ReplicationServerConfig clients_config{.ip_address = req.replication_client_info.replication_ip_address,
+                                                      .port = req.replication_client_info.replication_port};
+
+  if (bool success = memgraph::dbms::SetReplicationRoleReplica(dbms_handler, clients_config); !success) {
+    spdlog::error("Setting main to replica failed!");
+    slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+    return;
+  }
+
+  slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
+}
+
+void CoordinatorHandlers::PromoteReplicaToMainHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
+                                                      slk::Builder *res_builder) {
+  auto &repl_state = dbms_handler.ReplicationState();
+
+  if (!repl_state.IsReplica()) {
+    spdlog::error("Failover must be performed on replica!");
+    slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+    return;
+  }
+
+  if (bool success = memgraph::dbms::DoReplicaToMainPromotion(dbms_handler); !success) {
+    spdlog::error("Promoting replica to main failed!");
+    slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+    return;
+  }
+
+  coordination::PromoteReplicaToMainReq req;
+  slk::Load(&req, req_reader);
+
+  std::vector<replication::ReplicationClientConfig> clients_config;
+  clients_config.reserve(req.replication_clients_info.size());
+  std::ranges::transform(req.replication_clients_info, std::back_inserter(clients_config),
+                         [](const auto &repl_info_config) {
+                           return replication::ReplicationClientConfig{
+                               .name = repl_info_config.instance_name,
+                               .mode = repl_info_config.replication_mode,
+                               .ip_address = repl_info_config.replication_ip_address,
+                               .port = repl_info_config.replication_port,
+                           };
+                         });
+
+  std::ranges::for_each(clients_config, [&dbms_handler, &repl_state, &res_builder](const auto &config) {
+    auto instance_client = repl_state.RegisterReplica(config);
+    if (instance_client.HasError()) {
+      switch (instance_client.GetError()) {
+        case memgraph::replication::RegisterReplicaError::NOT_MAIN:
+          spdlog::error("Failover must be performed to main!");
+          slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+          return;
+        case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
+          spdlog::error("Replica with the same name already exists!");
+          slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+          return;
+        case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
+          spdlog::error("Replica with the same endpoint already exists!");
+          slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+          return;
+        case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
+          spdlog::error("Registered replica could not be persisted!");
+          slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+          return;
+        case memgraph::replication::RegisterReplicaError::SUCCESS:
+          break;
+      }
+    }
+
+    auto &instance_client_ptr = instance_client.GetValue();
+    const bool all_clients_good = memgraph::dbms::RegisterAllDatabasesClients(dbms_handler, *instance_client_ptr);
+
+    if (!all_clients_good) {
+      spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name);
+      slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
+      return;
+    }
+
+    StartReplicaClient(dbms_handler, *instance_client.GetValue());
+  });
+
+  slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
+}
+
+}  // namespace memgraph::dbms
+#endif

src/dbms/coordinator_handlers.hpp

@@ -0,0 +1,34 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#ifdef MG_ENTERPRISE
+
+#include "slk/serialization.hpp"
+
+namespace memgraph::dbms {
+
+class DbmsHandler;
+
+class CoordinatorHandlers {
+ public:
+  static void Register(DbmsHandler &dbms_handler);
+
+ private:
+  static void PromoteReplicaToMainHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
+                                          slk::Builder *res_builder);
+  static void SetMainToReplicaHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
+};
+
+}  // namespace memgraph::dbms
+
+#endif

src/dbms/dbms_handler.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -9,10 +9,15 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
+#ifdef MG_ENTERPRISE
+
 #include "dbms/dbms_handler.hpp"
 
+#include "dbms/coordinator_handlers.hpp"
+#include "flags/replication.hpp"
+
 namespace memgraph::dbms {
-#ifdef MG_ENTERPRISE
+
 DbmsHandler::DbmsHandler(
     storage::Config config,
     memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
@@ -69,7 +74,13 @@ DbmsHandler::DbmsHandler(
   // Startup proccess for main/replica
   MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()),
             "Replica recovery failure!");
+
+  // MAIN or REPLICA instance
+  if (FLAGS_coordinator_server_port) {
+    CoordinatorHandlers::Register(*this);
+    MG_ASSERT(coordinator_state_.GetCoordinatorServer().Start(), "Failed to start coordinator server!");
+  }
 }
-#endif
 
 }  // namespace memgraph::dbms
+#endif

src/dbms/dbms_handler.hpp

@@ -28,6 +28,7 @@
 #include "dbms/database.hpp"
 #include "dbms/inmemory/replication_handlers.hpp"
 #ifdef MG_ENTERPRISE
+#include "coordination/coordinator_state.hpp"
 #include "dbms/database_handler.hpp"
 #endif
 #include "dbms/replication_client.hpp"
@@ -226,6 +227,10 @@ class DbmsHandler {
   bool IsMain() const { return repl_state_.IsMain(); }
   bool IsReplica() const { return repl_state_.IsReplica(); }
 
+#ifdef MG_ENTERPRISE
+  coordination::CoordinatorState &CoordinatorState() { return coordinator_state_; }
+#endif
+
   /**
    * @brief Return the statistics all databases.
    *
@@ -522,6 +527,7 @@ class DbmsHandler {
   std::unique_ptr<kvstore::KVStore> durability_;       //!< list of active dbs (pointer so we can postpone its creation)
   bool delete_on_drop_;                                //!< Flag defining if dropping storage also deletes its directory
   std::set<std::string> defunct_dbs_;  //!< Databases that are in an unknown state due to various failures
+  coordination::CoordinatorState coordinator_state_;  //!< Replication coordinator
 #endif
   replication::ReplicationState repl_state_;  //!< Global replication state
 #ifndef MG_ENTERPRISE

src/dbms/replication_handler.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -16,10 +16,9 @@
 #include "dbms/inmemory/replication_handlers.hpp"
 #include "dbms/inmemory/storage_helper.hpp"
 #include "dbms/replication_client.hpp"
+#include "dbms/utils.hpp"
 #include "replication/state.hpp"
 
-using memgraph::replication::ReplicationClientConfig;
-using memgraph::replication::ReplicationState;
 using memgraph::replication::RoleMainData;
 using memgraph::replication::RoleReplicaData;
 
@@ -45,34 +44,13 @@ std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
 ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
 
 bool ReplicationHandler::SetReplicationRoleMain() {
-  auto const main_handler = [](RoleMainData const &) {
+  auto const main_handler = [](RoleMainData &) {
     // If we are already MAIN, we don't want to change anything
     return false;
   };
+
   auto const replica_handler = [this](RoleReplicaData const &) {
-    // STEP 1) bring down all REPLICA servers
-    dbms_handler_.ForEach([](Database *db) {
-      auto *storage = db->storage();
-      // Remember old epoch + storage timestamp association
-      storage->PrepareForNewEpoch();
-    });
-
-    // STEP 2) Change to MAIN
-    // TODO: restore replication servers if false?
-    if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) {
-      // TODO: Handle recovery on failure???
-      return false;
-    }
-
-    // STEP 3) We are now MAIN, update storage local epoch
-    const auto &epoch =
-        std::get<RoleMainData>(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_;
-    dbms_handler_.ForEach([&](Database *db) {
-      auto *storage = db->storage();
-      storage->repl_storage_state_.epoch_ = epoch;
-    });
-
-    return true;
+    return memgraph::dbms::DoReplicaToMainPromotion(dbms_handler_);
   };
 
   // TODO: under lock
@@ -124,49 +102,13 @@ auto ReplicationHandler::RegisterReplica(const memgraph::replication::Replicatio
   MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!");
 
   auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config);
-  if (instance_client.HasError()) switch (instance_client.GetError()) {
-      case memgraph::replication::RegisterReplicaError::NOT_MAIN:
-        MG_ASSERT(false, "Only main instance can register a replica!");
-        return {};
-      case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
-        return memgraph::dbms::RegisterReplicaError::NAME_EXISTS;
-      case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
-        return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS;
-      case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
-        return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
-      case memgraph::replication::RegisterReplicaError::SUCCESS:
-        break;
-    }
 
-  if (!allow_mt_repl && dbms_handler_.All().size() > 1) {
-    spdlog::warn("Multi-tenant replication is currently not supported!");
+  const auto dbms_error = memgraph::dbms::HandleErrorOnReplicaClient(instance_client);
+  if (dbms_error.has_value()) {
+    return *dbms_error;
   }
-
-  bool all_clients_good = true;
-
-  // Add database specific clients (NOTE Currently all databases are connected to each replica)
-  dbms_handler_.ForEach([&](Database *db) {
-    auto *storage = db->storage();
-    if (!allow_mt_repl && storage->id() != kDefaultDB) {
-      return;
-    }
-    // TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
-    if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
-
-    all_clients_good &=
-        storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) {
-          auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client.GetValue());
-          client->Start(storage);
-          // After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
-          // MAYBE_BEHIND isn't a statement of the current state, this is the default value
-          // Failed to start due to branching of MAIN and REPLICA
-          if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
-            return false;
-          }
-          storage_clients.push_back(std::move(client));
-          return true;
-        });
-  });
+  auto &instance_client_ptr = instance_client.GetValue();
+  const bool all_clients_good = memgraph::dbms::RegisterAllDatabasesClients(dbms_handler_, *instance_client_ptr);
 
   // NOTE Currently if any databases fails, we revert back
   if (!all_clients_good) {

src/dbms/replication_handler.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,6 @@
 #include "storage/v2/storage.hpp"
 #include "utils/result.hpp"
 
-// BEGIN fwd declares
 namespace memgraph::replication {
 struct ReplicationState;
 struct ReplicationServerConfig;
@@ -23,9 +22,11 @@ struct ReplicationClientConfig;
 }  // namespace memgraph::replication
 
 namespace memgraph::dbms {
+
 class DbmsHandler;
 
 enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
+
 enum class UnregisterReplicaResult : uint8_t {
   NOT_MAIN,
   COULD_NOT_BE_PERSISTED,

src/dbms/utils.hpp

@@ -0,0 +1,136 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+#pragma once
+
+#include "dbms/dbms_handler.hpp"
+#include "dbms/replication_handler.hpp"
+#include "replication/include/replication/state.hpp"
+#include "utils/result.hpp"
+
+namespace memgraph::dbms {
+
+inline bool DoReplicaToMainPromotion(dbms::DbmsHandler &dbms_handler) {
+  auto &repl_state = dbms_handler.ReplicationState();
+  // STEP 1) Prepare for new epoch
+  dbms_handler.ForEach([](Database *database) {
+    auto *storage = database->storage();
+    // Remember old epoch + storage timestamp association
+    storage->PrepareForNewEpoch();
+  });
+
+  // STEP 2) Change to MAIN = Kill replication server
+  if (!repl_state.SetReplicationRoleMain()) {
+    return false;
+  }
+
+  // STEP 3) We are now MAIN, update storage local epoch
+  const auto &epoch = std::get<replication::RoleMainData>(std::as_const(repl_state).ReplicationData()).epoch_;
+  dbms_handler.ForEach([&epoch](Database *database) {
+    auto *storage = database->storage();
+    storage->repl_storage_state_.epoch_ = epoch;
+  });
+  return true;
+};
+
+inline bool SetReplicationRoleReplica(dbms::DbmsHandler &dbms_handler,
+                                      const memgraph::replication::ReplicationServerConfig &config) {
+  // We don't want to restart the server if we're already a REPLICA
+  if (dbms_handler.ReplicationState().IsReplica()) {
+    return false;
+  }
+
+  // TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are
+  //      deleting the replica.
+  // Remove database specific clients
+  dbms_handler.ForEach([&](Database *db) {
+    auto *storage = db->storage();
+    storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
+  });
+  // Remove instance level clients
+  std::get<replication::RoleMainData>(dbms_handler.ReplicationState().ReplicationData()).registered_replicas_.clear();
+
+  // Creates the server
+  dbms_handler.ReplicationState().SetReplicationRoleReplica(config);
+
+  // Start
+  const auto success =
+      std::visit(utils::Overloaded{[](replication::RoleMainData const &) {
+                                     // ASSERT
+                                     return false;
+                                   },
+                                   [&dbms_handler](replication::RoleReplicaData const &data) {
+                                     // Register handlers
+                                     InMemoryReplicationHandlers::Register(&dbms_handler, *data.server);
+                                     if (!data.server->Start()) {
+                                       spdlog::error("Unable to start the replication server.");
+                                       return false;
+                                     }
+                                     return true;
+                                   }},
+                 dbms_handler.ReplicationState().ReplicationData());
+  // TODO Handle error (restore to main?)
+  return success;
+}
+
+inline bool RegisterAllDatabasesClients(dbms::DbmsHandler &dbms_handler,
+                                        replication::ReplicationClient &instance_client) {
+  if (!allow_mt_repl && dbms_handler.All().size() > 1) {
+    spdlog::warn("Multi-tenant replication is currently not supported!");
+  }
+
+  bool all_clients_good = true;
+
+  // Add database specific clients (NOTE Currently all databases are connected to each replica)
+  dbms_handler.ForEach([&](Database *db) {
+    auto *storage = db->storage();
+    if (!allow_mt_repl && storage->id() != kDefaultDB) {
+      return;
+    }
+    // TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
+    if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
+
+    all_clients_good &=
+        storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) {
+          auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
+          client->Start(storage);
+          // After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
+          // MAYBE_BEHIND isn't a statement of the current state, this is the default value
+          // Failed to start due to branching of MAIN and REPLICA
+          if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
+            return false;
+          }
+          storage_clients.push_back(std::move(client));
+          return true;
+        });
+  });
+
+  return all_clients_good;
+}
+
+inline std::optional<RegisterReplicaError> HandleErrorOnReplicaClient(
+    utils::BasicResult<replication::RegisterReplicaError, replication::ReplicationClient *> &instance_client) {
+  if (instance_client.HasError()) switch (instance_client.GetError()) {
+      case replication::RegisterReplicaError::NOT_MAIN:
+        MG_ASSERT(false, "Only main instance can register a replica!");
+        return {};
+      case replication::RegisterReplicaError::NAME_EXISTS:
+        return dbms::RegisterReplicaError::NAME_EXISTS;
+      case replication::RegisterReplicaError::END_POINT_EXISTS:
+        return dbms::RegisterReplicaError::END_POINT_EXISTS;
+      case replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
+        return dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
+      case replication::RegisterReplicaError::SUCCESS:
+        break;
+    }
+  return {};
+}
+
+}  // namespace memgraph::dbms

src/flags/CMakeLists.txt

@@ -6,6 +6,7 @@ add_library(mg-flags STATIC audit.cpp
                             memory_limit.cpp
                             run_time_configurable.cpp
                             storage_mode.cpp
-                            query.cpp)
+                            query.cpp
+                            replication.cpp)
 target_include_directories(mg-flags PUBLIC ${CMAKE_SOURCE_DIR}/include)
 target_link_libraries(mg-flags PUBLIC spdlog::spdlog mg-settings mg-utils)

src/flags/all.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,5 +17,6 @@
 #include "flags/log_level.hpp"
 #include "flags/memory_limit.hpp"
 #include "flags/query.hpp"
+#include "flags/replication.hpp"
 #include "flags/run_time_configurable.hpp"
 #include "flags/storage_mode.hpp"

src/flags/general.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -162,13 +162,6 @@ DEFINE_string(pulsar_service_url, "", "Default URL used while connecting to Puls
 
 // Query flags.
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_uint64(replication_replica_check_frequency_sec, 1,
-              "The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
-              "The MAIN instance allocates a new thread for each REPLICA.");
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");
-
 DEFINE_VALIDATED_string(query_modules_directory, "",
                         "Directory where modules with custom query procedures are stored. "
                         "NOTE: Multiple comma-separated directories can be defined.",

src/flags/general.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -116,11 +116,6 @@ namespace memgraph::flags {
 auto ParseQueryModulesDirectory() -> std::vector<std::filesystem::path>;
 }  // namespace memgraph::flags
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_uint64(replication_replica_check_frequency_sec);
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_bool(replication_restore_state_on_startup);
-
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
 DECLARE_string(license_key);
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)

src/flags/replication.cpp

@@ -0,0 +1,26 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include "replication.hpp"
+
+#ifdef MG_ENTERPRISE
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_bool(coordinator, false, "Controls whether the instance is a replication coordinator.");
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_uint32(coordinator_server_port, 0, "Port on which coordinator servers will be started.");
+#endif
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_uint64(replication_replica_check_frequency_sec, 1,
+              "The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
+              "The MAIN instance allocates a new thread for each REPLICA.");
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");

src/flags/replication.hpp

@@ -0,0 +1,26 @@
+// Copyright 2024 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#pragma once
+
+#include "gflags/gflags.h"
+
+#ifdef MG_ENTERPRISE
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_bool(coordinator);
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint32(coordinator_server_port);
+#endif
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_uint64(replication_replica_check_frequency_sec);
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+DECLARE_bool(replication_restore_state_on_startup);

src/glue/SessionHL.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -193,12 +193,13 @@ std::pair<std::vector<std::string>, std::optional<int>> SessionHL::Interpret(
   for (const auto &[key, bolt_param] : params) {
     params_pv.emplace(key, ToPropertyValue(bolt_param));
   }
+
+#ifdef MG_ENTERPRISE
   const std::string *username{nullptr};
   if (user_) {
     username = &user_->username();
   }
 
-#ifdef MG_ENTERPRISE
   // TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt)
   auto *db = interpreter_.current_db_.db_acc_->get();
   if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {

src/glue/auth.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -66,6 +66,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
       return auth::Permission::MULTI_DATABASE_EDIT;
     case query::AuthQuery::Privilege::MULTI_DATABASE_USE:
       return auth::Permission::MULTI_DATABASE_USE;
+    case query::AuthQuery::Privilege::COORDINATOR:
+      return auth::Permission::COORDINATOR;
   }
 }
 

src/io/CMakeLists.txt

@@ -8,4 +8,5 @@ find_package(fmt REQUIRED)
 find_package(Threads REQUIRED)
 
 add_library(mg-io STATIC ${io_src_files})
+add_library(mg::io ALIAS mg-io)
 target_link_libraries(mg-io stdc++fs Threads::Threads fmt::fmt mg-utils)

src/io/network/endpoint.cpp

@@ -166,7 +166,7 @@ bool Endpoint::IsResolvableAddress(const std::string &address, uint16_t port) {
 }
 
 std::optional<std::pair<std::string, uint16_t>> Endpoint::ParseSocketOrAddress(
-    const std::string &address, const std::optional<uint16_t> default_port = {}) {
+    const std::string &address, const std::optional<uint16_t> default_port) {
   const std::string delimiter = ":";
   std::vector<std::string> parts = utils::Split(address, delimiter);
   if (parts.size() == 1) {

src/memgraph.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,7 +17,6 @@
 #include "dbms/constants.hpp"
 #include "dbms/inmemory/replication_handlers.hpp"
 #include "flags/all.hpp"
-#include "flags/run_time_configurable.hpp"
 #include "glue/MonitoringServerT.hpp"
 #include "glue/ServerT.hpp"
 #include "glue/auth_checker.hpp"

src/query/config.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

src/query/exceptions.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -253,6 +253,13 @@ class ReplicationModificationInMulticommandTxException : public QueryException {
   SPECIALIZE_GET_EXCEPTION_NAME(ReplicationModificationInMulticommandTxException)
 };
 
+class CoordinatorModificationInMulticommandTxException : public QueryException {
+ public:
+  CoordinatorModificationInMulticommandTxException()
+      : QueryException("Coordinator clause not allowed in multicommand transactions.") {}
+  SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorModificationInMulticommandTxException)
+};
+
 class ReplicationDisabledOnDiskStorage : public QueryException {
  public:
   ReplicationDisabledOnDiskStorage() : QueryException("Replication is not supported while in on-disk storage mode.") {}

src/query/frontend/ast/ast.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -239,6 +239,9 @@ constexpr utils::TypeInfo query::DumpQuery::kType{utils::TypeId::AST_DUMP_QUERY,
 constexpr utils::TypeInfo query::ReplicationQuery::kType{utils::TypeId::AST_REPLICATION_QUERY, "ReplicationQuery",
                                                          &query::Query::kType};
 
+constexpr utils::TypeInfo query::CoordinatorQuery::kType{utils::TypeId::AST_COORDINATOR_QUERY, "CoordinatorQuery",
+                                                         &query::Query::kType};
+
 constexpr utils::TypeInfo query::LockPathQuery::kType{utils::TypeId::AST_LOCK_PATH_QUERY, "LockPathQuery",
                                                       &query::Query::kType};
 

src/query/frontend/ast/ast.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -2849,6 +2849,7 @@ class AuthQuery : public memgraph::query::Query {
     TRANSACTION_MANAGEMENT,
     MULTI_DATABASE_EDIT,
     MULTI_DATABASE_USE,
+    COORDINATOR
   };
 
   enum class FineGrainedPrivilege { NOTHING, READ, UPDATE, CREATE_DELETE };
@@ -2927,7 +2928,8 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {AuthQuery::Privilege::
                                                           AuthQuery::Privilege::TRANSACTION_MANAGEMENT,
                                                           AuthQuery::Privilege::STORAGE_MODE,
                                                           AuthQuery::Privilege::MULTI_DATABASE_EDIT,
-                                                          AuthQuery::Privilege::MULTI_DATABASE_USE};
+                                                          AuthQuery::Privilege::MULTI_DATABASE_USE,
+                                                          AuthQuery::Privilege::COORDINATOR};
 
 class DatabaseInfoQuery : public memgraph::query::Query {
  public:
@@ -3039,8 +3041,9 @@ class ReplicationQuery : public memgraph::query::Query {
 
   memgraph::query::ReplicationQuery::Action action_;
   memgraph::query::ReplicationQuery::ReplicationRole role_;
-  std::string replica_name_;
+  std::string instance_name_;
   memgraph::query::Expression *socket_address_{nullptr};
+  memgraph::query::Expression *coordinator_socket_address_{nullptr};
   memgraph::query::Expression *port_{nullptr};
   memgraph::query::ReplicationQuery::SyncMode sync_mode_;
 
@@ -3048,10 +3051,56 @@ class ReplicationQuery : public memgraph::query::Query {
     ReplicationQuery *object = storage->Create<ReplicationQuery>();
     object->action_ = action_;
     object->role_ = role_;
-    object->replica_name_ = replica_name_;
+    object->instance_name_ = instance_name_;
     object->socket_address_ = socket_address_ ? socket_address_->Clone(storage) : nullptr;
     object->port_ = port_ ? port_->Clone(storage) : nullptr;
     object->sync_mode_ = sync_mode_;
+    object->coordinator_socket_address_ =
+        coordinator_socket_address_ ? coordinator_socket_address_->Clone(storage) : nullptr;
+
+    return object;
+  }
+
+ private:
+  friend class AstStorage;
+};
+
+class CoordinatorQuery : public memgraph::query::Query {
+ public:
+  static const utils::TypeInfo kType;
+  const utils::TypeInfo &GetTypeInfo() const override { return kType; }
+
+  enum class Action {
+    REGISTER_INSTANCE_ON_COORDINATOR,
+    REGISTER_MAIN_COORDINATOR_SERVER,
+    REGISTER_REPLICA_COORDINATOR_SERVER,
+    SET_INSTANCE_TO_MAIN,
+    SHOW_REPLICATION_CLUSTER,
+    DO_FAILOVER
+  };
+
+  enum class SyncMode { SYNC, ASYNC };
+
+  CoordinatorQuery() = default;
+
+  DEFVISITABLE(QueryVisitor<void>);
+
+  memgraph::query::CoordinatorQuery::Action action_;
+  std::string instance_name_;
+  memgraph::query::Expression *replication_socket_address_{nullptr};
+  memgraph::query::Expression *coordinator_socket_address_{nullptr};
+  memgraph::query::CoordinatorQuery::SyncMode sync_mode_;
+
+  CoordinatorQuery *Clone(AstStorage *storage) const override {
+    auto *object = storage->Create<CoordinatorQuery>();
+    object->action_ = action_;
+    object->instance_name_ = instance_name_;
+    object->replication_socket_address_ =
+        replication_socket_address_ ? replication_socket_address_->Clone(storage) : nullptr;
+    object->sync_mode_ = sync_mode_;
+    object->coordinator_socket_address_ =
+        coordinator_socket_address_ ? coordinator_socket_address_->Clone(storage) : nullptr;
+
     return object;
   }
 

src/query/frontend/ast/ast_visitor.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -108,6 +108,7 @@ class MultiDatabaseQuery;
 class ShowDatabasesQuery;
 class EdgeImportModeQuery;
 class PatternComprehension;
+class CoordinatorQuery;
 
 using TreeCompositeVisitor = utils::CompositeVisitor<
     SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -146,6 +147,7 @@ class QueryVisitor
                             SystemInfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
                             FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery, StreamQuery,
                             SettingQuery, VersionQuery, ShowConfigQuery, TransactionQueueQuery, StorageModeQuery,
-                            AnalyzeGraphQuery, MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery> {};
+                            AnalyzeGraphQuery, MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery,
+                            CoordinatorQuery> {};
 
 }  // namespace memgraph::query

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -302,6 +302,13 @@ antlrcpp::Any CypherMainVisitor::visitReplicationQuery(MemgraphCypher::Replicati
   return replication_query;
 }
 
+antlrcpp::Any CypherMainVisitor::visitCoordinatorQuery(MemgraphCypher::CoordinatorQueryContext *ctx) {
+  MG_ASSERT(ctx->children.size() == 1, "CoordinatorQuery should have exactly one child!");
+  auto *coordinator_query = std::any_cast<CoordinatorQuery *>(ctx->children[0]->accept(this));
+  query_ = coordinator_query;
+  return coordinator_query;
+}
+
 antlrcpp::Any CypherMainVisitor::visitEdgeImportModeQuery(MemgraphCypher::EdgeImportModeQueryContext *ctx) {
   auto *edge_import_mode_query = storage_->Create<EdgeImportModeQuery>();
   if (ctx->ACTIVE()) {
@@ -316,24 +323,34 @@ antlrcpp::Any CypherMainVisitor::visitEdgeImportModeQuery(MemgraphCypher::EdgeIm
 antlrcpp::Any CypherMainVisitor::visitSetReplicationRole(MemgraphCypher::SetReplicationRoleContext *ctx) {
   auto *replication_query = storage_->Create<ReplicationQuery>();
   replication_query->action_ = ReplicationQuery::Action::SET_REPLICATION_ROLE;
+
+  auto set_replication_port = [replication_query, ctx, this]() -> void {
+    if (ctx->port->numberLiteral() && ctx->port->numberLiteral()->integerLiteral()) {
+      replication_query->port_ = std::any_cast<Expression *>(ctx->port->accept(this));
+    } else {
+      throw SyntaxException("Port must be an integer literal!");
+    }
+  };
+
   if (ctx->MAIN()) {
+    replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN;
     if (ctx->WITH() || ctx->PORT()) {
       throw SemanticException("Main can't set a port!");
     }
-    replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN;
+
   } else if (ctx->REPLICA()) {
     replication_query->role_ = ReplicationQuery::ReplicationRole::REPLICA;
     if (ctx->WITH() && ctx->PORT()) {
-      if (ctx->port->numberLiteral() && ctx->port->numberLiteral()->integerLiteral()) {
-        replication_query->port_ = std::any_cast<Expression *>(ctx->port->accept(this));
-      } else {
-        throw SyntaxException("Port must be an integer literal!");
-      }
+      set_replication_port();
+    } else {
+      throw SemanticException("Replica must set a port!");
     }
   }
+
   return replication_query;
 }
-antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext *ctx) {
+
+antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext * /*ctx*/) {
   auto *replication_query = storage_->Create<ReplicationQuery>();
   replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICATION_ROLE;
   return replication_query;
@@ -342,7 +359,7 @@ antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowRe
 antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterReplicaContext *ctx) {
   auto *replication_query = storage_->Create<ReplicationQuery>();
   replication_query->action_ = ReplicationQuery::Action::REGISTER_REPLICA;
-  replication_query->replica_name_ = std::any_cast<std::string>(ctx->replicaName()->symbolicName()->accept(this));
+  replication_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
   if (ctx->SYNC()) {
     replication_query->sync_mode_ = memgraph::query::ReplicationQuery::SyncMode::SYNC;
   } else if (ctx->ASYNC()) {
@@ -351,26 +368,75 @@ antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterRe
 
   if (!ctx->socketAddress()->literal()->StringLiteral()) {
     throw SemanticException("Socket address should be a string literal!");
-  } else {
-    replication_query->socket_address_ = std::any_cast<Expression *>(ctx->socketAddress()->accept(this));
   }
+  replication_query->socket_address_ = std::any_cast<Expression *>(ctx->socketAddress()->accept(this));
 
   return replication_query;
 }
 
+// License check is done in the interpreter.
+antlrcpp::Any CypherMainVisitor::visitRegisterInstanceOnCoordinator(
+    MemgraphCypher::RegisterInstanceOnCoordinatorContext *ctx) {
+  auto *coordinator_query = storage_->Create<CoordinatorQuery>();
+  if (!ctx->replicationSocketAddress()->literal()->StringLiteral()) {
+    throw SemanticException("Replication socket address should be a string literal!");
+  }
+
+  if (!ctx->coordinatorSocketAddress()->literal()->StringLiteral()) {
+    throw SemanticException("Coordinator socket address should be a string literal!");
+  }
+  coordinator_query->action_ = CoordinatorQuery::Action::REGISTER_INSTANCE_ON_COORDINATOR;
+  coordinator_query->replication_socket_address_ =
+      std::any_cast<Expression *>(ctx->replicationSocketAddress()->accept(this));
+  coordinator_query->coordinator_socket_address_ =
+      std::any_cast<Expression *>(ctx->coordinatorSocketAddress()->accept(this));
+  coordinator_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
+  if (ctx->ASYNC()) {
+    coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::ASYNC;
+  } else {
+    coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::SYNC;
+  }
+
+  return coordinator_query;
+}
+
+// License check is done in the interpreter
+antlrcpp::Any CypherMainVisitor::visitShowReplicationCluster(MemgraphCypher::ShowReplicationClusterContext * /*ctx*/) {
+  auto *coordinator_query = storage_->Create<CoordinatorQuery>();
+  coordinator_query->action_ = CoordinatorQuery::Action::SHOW_REPLICATION_CLUSTER;
+  return coordinator_query;
+}
+
 antlrcpp::Any CypherMainVisitor::visitDropReplica(MemgraphCypher::DropReplicaContext *ctx) {
   auto *replication_query = storage_->Create<ReplicationQuery>();
   replication_query->action_ = ReplicationQuery::Action::DROP_REPLICA;
-  replication_query->replica_name_ = std::any_cast<std::string>(ctx->replicaName()->symbolicName()->accept(this));
+  replication_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
   return replication_query;
 }
 
-antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) {
+antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasContext * /*ctx*/) {
   auto *replication_query = storage_->Create<ReplicationQuery>();
   replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICAS;
   return replication_query;
 }
 
+// License check is done in the interpreter
+antlrcpp::Any CypherMainVisitor::visitDoFailover(MemgraphCypher::DoFailoverContext * /*ctx*/) {
+  auto *coordinator_query = storage_->Create<CoordinatorQuery>();
+  coordinator_query->action_ = CoordinatorQuery::Action::DO_FAILOVER;
+  query_ = coordinator_query;
+  return coordinator_query;
+}
+
+// License check is done in the interpreter
+antlrcpp::Any CypherMainVisitor::visitSetInstanceToMain(MemgraphCypher::SetInstanceToMainContext *ctx) {
+  auto *coordinator_query = storage_->Create<CoordinatorQuery>();
+  coordinator_query->action_ = CoordinatorQuery::Action::SET_INSTANCE_TO_MAIN;
+  coordinator_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
+  query_ = coordinator_query;
+  return coordinator_query;
+}
+
 antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) {
   auto *lock_query = storage_->Create<LockPathQuery>();
   if (ctx->STATUS()) {
@@ -1638,6 +1704,7 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
   if (ctx->STORAGE_MODE()) return AuthQuery::Privilege::STORAGE_MODE;
   if (ctx->MULTI_DATABASE_EDIT()) return AuthQuery::Privilege::MULTI_DATABASE_EDIT;
   if (ctx->MULTI_DATABASE_USE()) return AuthQuery::Privilege::MULTI_DATABASE_USE;
+  if (ctx->COORDINATOR()) return AuthQuery::Privilege::COORDINATOR;
   LOG_FATAL("Should not get here - unknown privilege!");
 }
 

src/query/frontend/ast/cypher_main_visitor.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -233,6 +233,31 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
    */
   antlrcpp::Any visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) override;
 
+  /**
+   * @return CoordinatorQuery*
+   */
+  antlrcpp::Any visitCoordinatorQuery(MemgraphCypher::CoordinatorQueryContext *ctx) override;
+
+  /**
+   * @return CoordinatorQuery*
+   */
+  antlrcpp::Any visitRegisterInstanceOnCoordinator(MemgraphCypher::RegisterInstanceOnCoordinatorContext *ctx) override;
+
+  /**
+   * @return CoordinatorQuery*
+   */
+  antlrcpp::Any visitSetInstanceToMain(MemgraphCypher::SetInstanceToMainContext *ctx) override;
+
+  /**
+   * @return CoordinatorQuery*
+   */
+  antlrcpp::Any visitShowReplicationCluster(MemgraphCypher::ShowReplicationClusterContext *ctx) override;
+
+  /**
+   * @return CoordinatorQuery*
+   */
+  antlrcpp::Any visitDoFailover(MemgraphCypher::DoFailoverContext *ctx) override;
+
   /**
    * @return LockPathQuery*
    */

src/query/frontend/opencypher/grammar/CypherLexer.g4

@@ -102,6 +102,7 @@ FILTER         : F I L T E R ;
 IN             : I N ;
 INDEX          : I N D E X ;
 INFO           : I N F O ;
+INSTANCE       : I N S T A N C E ;
 IS             : I S ;
 KB             : K B ;
 KEY            : K E Y ;
@@ -122,6 +123,7 @@ PROCEDURE      : P R O C E D U R E ;
 PROFILE        : P R O F I L E ;
 QUERY          : Q U E R Y ;
 REDUCE         : R E D U C E ;
+REGISTER       : R E G I S T E R;
 REMOVE         : R E M O V E ;
 RETURN         : R E T U R N ;
 SET            : S E T ;

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -48,10 +48,12 @@ memgraphCypherKeyword : cypherKeyword
                       | DATABASE
                       | DENY
                       | DROP
+                      | DO
                       | DUMP
                       | EDGE
                       | EDGE_TYPES
                       | EXECUTE
+                      | FAILOVER
                       | FOR
                       | FOREACH
                       | FREE
@@ -61,6 +63,7 @@ memgraphCypherKeyword : cypherKeyword
                       | GRANT
                       | HEADER
                       | IDENTIFIED
+                      | INSTANCE
                       | NODE_LABELS
                       | NULLIF
                       | IMPORT
@@ -151,6 +154,7 @@ query : cypherQuery
       | multiDatabaseQuery
       | showDatabases
       | edgeImportModeQuery
+      | coordinatorQuery
       ;
 
 cypherQuery : ( indexHints )? singleQuery ( cypherUnion )* ( queryMemoryLimit )? ;
@@ -183,6 +187,12 @@ replicationQuery : setReplicationRole
                  | showReplicas
                  ;
 
+coordinatorQuery : registerInstanceOnCoordinator
+                 | setInstanceToMain
+                 | showReplicationCluster
+                 | doFailover
+                 ;
+
 triggerQuery : createTrigger
              | dropTrigger
              | showTriggers
@@ -244,6 +254,8 @@ transactionQueueQuery : showTransactions
 
 showTransactions : SHOW TRANSACTIONS ;
 
+doFailover : DO FAILOVER ;
+
 terminateTransactions : TERMINATE TRANSACTIONS transactionIdList;
 
 loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
@@ -323,6 +335,7 @@ privilege : CREATE
           | STORAGE_MODE
           | MULTI_DATABASE_EDIT
           | MULTI_DATABASE_USE
+          | COORDINATOR
           ;
 
 granularPrivilege : NOTHING | READ | UPDATE | CREATE_DELETE ;
@@ -364,14 +377,23 @@ setReplicationRole : SET REPLICATION ROLE TO ( MAIN | REPLICA )
 
 showReplicationRole : SHOW REPLICATION ROLE ;
 
-replicaName : symbolicName ;
+showReplicationCluster : SHOW REPLICATION CLUSTER ;
+
+instanceName : symbolicName ;
 
 socketAddress : literal ;
 
-registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC )
+coordinatorSocketAddress : literal ;
+replicationSocketAddress : literal ;
+
+registerReplica : REGISTER REPLICA instanceName ( SYNC | ASYNC )
                 TO socketAddress ;
 
-dropReplica : DROP REPLICA replicaName ;
+registerInstanceOnCoordinator : REGISTER INSTANCE  instanceName ON coordinatorSocketAddress ( AS ASYNC ) ?  WITH  replicationSocketAddress ;
+
+setInstanceToMain : SET INSTANCE instanceName TO MAIN ;
+
+dropReplica : DROP REPLICA instanceName ;
 
 showReplicas : SHOW REPLICAS ;
 

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -39,15 +39,18 @@ BOOTSTRAP_SERVERS       : B O O T S T R A P UNDERSCORE S E R V E R S ;
 CALL                    : C A L L ;
 CHECK                   : C H E C K ;
 CLEAR                   : C L E A R ;
+CLUSTER                 : C L U S T E R ;
 COMMIT                  : C O M M I T ;
 COMMITTED               : C O M M I T T E D ;
 CONFIG                  : C O N F I G ;
 CONFIGS                 : C O N F I G S;
 CONSUMER_GROUP          : C O N S U M E R UNDERSCORE G R O U P ;
+COORDINATOR             : C O O R D I N A T O R ;
 CREATE_DELETE           : C R E A T E UNDERSCORE D E L E T E ;
 CREDENTIALS             : C R E D E N T I A L S ;
 CSV                     : C S V ;
 DATA                    : D A T A ;
+DO                      : D O ;
 DELIMITER               : D E L I M I T E R ;
 DATABASE                : D A T A B A S E ;
 DATABASES               : D A T A B A S E S ;
@@ -59,6 +62,7 @@ DURABILITY              : D U R A B I L I T Y ;
 EDGE                    : E D G E ;
 EDGE_TYPES              : E D G E UNDERSCORE T Y P E S ;
 EXECUTE                 : E X E C U T E ;
+FAILOVER                : F A I L O V E R ;
 FOR                     : F O R ;
 FOREACH                 : F O R E A C H;
 FREE                    : F R E E ;
@@ -75,6 +79,7 @@ IMPORT                  : I M P O R T ;
 INACTIVE                : I N A C T I V E ;
 IN_MEMORY_ANALYTICAL    : I N UNDERSCORE M E M O R Y UNDERSCORE A N A L Y T I C A L ;
 IN_MEMORY_TRANSACTIONAL : I N UNDERSCORE M E M O R Y UNDERSCORE T R A N S A C T I O N A L ;
+INSTANCE                : I N S T A N C E ;
 ISOLATION               : I S O L A T I O N ;
 KAFKA                   : K A F K A ;
 LABELS                  : L A B E L S ;
@@ -107,6 +112,7 @@ REVOKE                  : R E V O K E ;
 ROLE                    : R O L E ;
 ROLES                   : R O L E S ;
 QUOTE                   : Q U O T E ;
+SERVER                  : S E R V E R ;
 SERVICE_URL             : S E R V I C E UNDERSCORE U R L ;
 SESSION                 : S E S S I O N ;
 SETTING                 : S E T T I N G ;

src/query/frontend/semantic/required_privileges.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -115,6 +115,8 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
     AddPrivilege(AuthQuery::Privilege::MULTI_DATABASE_USE); /* OR EDIT */
   }
 
+  void Visit(CoordinatorQuery & /*coordinator_query*/) override { AddPrivilege(AuthQuery::Privilege::COORDINATOR); }
+
   bool PreVisit(Create & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::CREATE);
     return false;

src/query/frontend/stripped_lexer_constants.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -218,7 +218,8 @@ const trie::Trie kKeywords = {"union",
                               "directory",
                               "lock",
                               "unlock",
-                              "build"};
+                              "build",
+                              "instance"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

src/query/interpreter.cpp

@@ -35,9 +35,9 @@
 #include "auth/models.hpp"
 #include "csv/parsing.hpp"
 #include "dbms/database.hpp"
-#include "dbms/dbms_handler.hpp"
 #include "dbms/global.hpp"
 #include "dbms/inmemory/storage_helper.hpp"
+#include "flags/replication.hpp"
 #include "flags/run_time_configurable.hpp"
 #include "glue/communication.hpp"
 #include "license/license.hpp"
@@ -101,12 +101,17 @@
 #include "utils/typeinfo.hpp"
 #include "utils/variant_helpers.hpp"
 
+#include "dbms/coordinator_handler.hpp"
 #include "dbms/dbms_handler.hpp"
 #include "dbms/replication_handler.hpp"
 #include "query/auth_query_handler.hpp"
 #include "query/interpreter_context.hpp"
 #include "replication/state.hpp"
 
+#ifdef MG_ENTERPRISE
+#include "coordination/constants.hpp"
+#endif
+
 namespace memgraph::metrics {
 extern Event ReadQuery;
 extern Event WriteQuery;
@@ -121,6 +126,7 @@ extern const Event CommitedTransactions;
 extern const Event RollbackedTransactions;
 extern const Event ActiveTransactions;
 }  // namespace memgraph::metrics
+
 void memgraph::query::CurrentDB::SetupDatabaseTransaction(
     std::optional<storage::IsolationLevel> override_isolation_level, bool could_commit, bool unique) {
   auto &db_acc = *db_acc_;
@@ -259,17 +265,32 @@ bool IsAllShortestPathsQuery(const std::vector<memgraph::query::Clause *> &claus
   return false;
 }
 
-inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode) -> replication::ReplicationMode {
+inline auto convertFromCoordinatorToReplicationMode(const CoordinatorQuery::SyncMode &sync_mode)
+    -> replication_coordination_glue::ReplicationMode {
   switch (sync_mode) {
-    case ReplicationQuery::SyncMode::ASYNC: {
-      return replication::ReplicationMode::ASYNC;
+    case CoordinatorQuery::SyncMode::ASYNC: {
+      return replication_coordination_glue::ReplicationMode::ASYNC;
     }
-    case ReplicationQuery::SyncMode::SYNC: {
-      return replication::ReplicationMode::SYNC;
+    case CoordinatorQuery::SyncMode::SYNC: {
+      return replication_coordination_glue::ReplicationMode::SYNC;
     }
   }
   // TODO: C++23 std::unreachable()
-  return replication::ReplicationMode::ASYNC;
+  return replication_coordination_glue::ReplicationMode::ASYNC;
+}
+
+inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode)
+    -> replication_coordination_glue::ReplicationMode {
+  switch (sync_mode) {
+    case ReplicationQuery::SyncMode::ASYNC: {
+      return replication_coordination_glue::ReplicationMode::ASYNC;
+    }
+    case ReplicationQuery::SyncMode::SYNC: {
+      return replication_coordination_glue::ReplicationMode::SYNC;
+    }
+  }
+  // TODO: C++23 std::unreachable()
+  return replication_coordination_glue::ReplicationMode::ASYNC;
 }
 
 class ReplQueryHandler final : public query::ReplicationQueryHandler {
@@ -278,14 +299,18 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
 
   /// @throw QueryRuntimeException if an error ocurred.
   void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional<int64_t> port) override {
-    if (replication_role == ReplicationQuery::ReplicationRole::MAIN) {
-      if (!handler_.SetReplicationRoleMain()) {
-        throw QueryRuntimeException("Couldn't set role to main!");
-      }
-    } else {
-      if (!port || *port < 0 || *port > std::numeric_limits<uint16_t>::max()) {
+    auto ValidatePort = [](std::optional<int64_t> port) -> void {
+      if (*port < 0 || *port > std::numeric_limits<uint16_t>::max()) {
         throw QueryRuntimeException("Port number invalid!");
       }
+    };
+
+    if (replication_role == ReplicationQuery::ReplicationRole::MAIN) {
+      if (!handler_.SetReplicationRoleMain()) {
+        throw QueryRuntimeException("Couldn't set replication role to main!");
+      }
+    } else {
+      ValidatePort(port);
 
       auto const config = memgraph::replication::ReplicationServerConfig{
           .ip_address = memgraph::replication::kDefaultReplicationServerIp,
@@ -313,27 +338,33 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
   void RegisterReplica(const std::string &name, const std::string &socket_address,
                        const ReplicationQuery::SyncMode sync_mode,
                        const std::chrono::seconds replica_check_frequency) override {
+    // Coordinator is main by default so this check is OK although it should actually be nothing (neither main nor
+    // replica)
     if (handler_.IsReplica()) {
       // replica can't register another replica
       throw QueryRuntimeException("Replica can't register another replica!");
     }
 
-    auto repl_mode = convertToReplicationMode(sync_mode);
+    const auto repl_mode = convertToReplicationMode(sync_mode);
 
-    auto maybe_ip_and_port =
+    const auto maybe_ip_and_port =
         io::network::Endpoint::ParseSocketOrAddress(socket_address, memgraph::replication::kDefaultReplicationPort);
     if (maybe_ip_and_port) {
-      auto [ip, port] = *maybe_ip_and_port;
-      auto config = replication::ReplicationClientConfig{.name = name,
-                                                         .mode = repl_mode,
-                                                         .ip_address = ip,
-                                                         .port = port,
-                                                         .replica_check_frequency = replica_check_frequency,
-                                                         .ssl = std::nullopt};
-      auto ret = handler_.RegisterReplica(config);
-      if (ret.HasError()) {
+      const auto [ip, port] = *maybe_ip_and_port;
+      const auto replication_config =
+          replication::ReplicationClientConfig{.name = name,
+                                               .mode = repl_mode,
+                                               .ip_address = ip,
+                                               .port = port,
+                                               .replica_check_frequency = replica_check_frequency,
+                                               .ssl = std::nullopt};
+
+      const auto error = handler_.RegisterReplica(replication_config).HasError();
+
+      if (error) {
         throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
       }
+
     } else {
       throw QueryRuntimeException("Invalid socket address!");
     }
@@ -364,6 +395,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
 
     // TODO: Combine results? Have a single place with clients???
     //       Also authentication checks (replica + database visibility)
+
     std::vector<storage::ReplicaInfo> repl_infos{};
     dbms_handler_->ForOne([&repl_infos](dbms::Database *db) -> bool {
       auto infos = db->storage()->ReplicasInfo();
@@ -381,10 +413,10 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
       replica.name = repl_info.name;
       replica.socket_address = repl_info.endpoint.SocketAddress();
       switch (repl_info.mode) {
-        case memgraph::replication::ReplicationMode::SYNC:
+        case replication_coordination_glue::ReplicationMode::SYNC:
           replica.sync_mode = ReplicationQuery::SyncMode::SYNC;
           break;
-        case memgraph::replication::ReplicationMode::ASYNC:
+        case replication_coordination_glue::ReplicationMode::ASYNC:
           replica.sync_mode = ReplicationQuery::SyncMode::ASYNC;
           break;
       }
@@ -420,6 +452,219 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
   dbms::ReplicationHandler handler_;
 };
 
+class CoordQueryHandler final : public query::CoordinatorQueryHandler {
+ public:
+  explicit CoordQueryHandler(dbms::DbmsHandler *dbms_handler) : handler_ { *dbms_handler }
+#ifdef MG_ENTERPRISE
+  , coordinator_handler_(*dbms_handler)
+#endif
+  {
+  }
+
+#ifdef MG_ENTERPRISE
+  /// @throw QueryRuntimeException if an error ocurred.
+  void RegisterReplicaCoordinatorServer(const std::string &replication_socket_address,
+                                        const std::string &coordinator_socket_address,
+                                        const std::chrono::seconds instance_check_frequency,
+                                        const std::string &instance_name,
+                                        CoordinatorQuery::SyncMode sync_mode) override {
+    const auto maybe_replication_ip_port =
+        io::network::Endpoint::ParseSocketOrAddress(replication_socket_address, std::nullopt);
+    if (!maybe_replication_ip_port) {
+      throw QueryRuntimeException("Invalid replication socket address!");
+    }
+
+    const auto maybe_coordinator_ip_port =
+        io::network::Endpoint::ParseSocketOrAddress(coordinator_socket_address, std::nullopt);
+    if (!maybe_replication_ip_port) {
+      throw QueryRuntimeException("Invalid replication socket address!");
+    }
+
+    const auto [replication_ip, replication_port] = *maybe_replication_ip_port;
+    const auto [coordinator_server_ip, coordinator_server_port] = *maybe_coordinator_ip_port;
+    const auto repl_config = coordination::CoordinatorClientConfig::ReplicationClientInfo{
+        .instance_name = instance_name,
+        .replication_mode = convertFromCoordinatorToReplicationMode(sync_mode),
+        .replication_ip_address = replication_ip,
+        .replication_port = replication_port};
+
+    auto coordinator_client_config =
+        coordination::CoordinatorClientConfig{.instance_name = instance_name,
+                                              .ip_address = coordinator_server_ip,
+                                              .port = coordinator_server_port,
+                                              .health_check_frequency_sec = instance_check_frequency,
+                                              .replication_client_info = repl_config,
+                                              .ssl = std::nullopt};
+
+    auto status = coordinator_handler_.RegisterReplicaOnCoordinator(std::move(coordinator_client_config));
+    switch (status) {
+      using enum memgraph::coordination::RegisterMainReplicaCoordinatorStatus;
+      case NAME_EXISTS:
+        throw QueryRuntimeException("Couldn't register replica instance since instance with such name already exists!");
+      case END_POINT_EXISTS:
+        throw QueryRuntimeException(
+            "Couldn't register replica instance since instance with such endpoint already exists!");
+      case COULD_NOT_BE_PERSISTED:
+        throw QueryRuntimeException("Couldn't register replica instance since it couldn't be persisted!");
+      case NOT_COORDINATOR:
+        throw QueryRuntimeException("Couldn't register replica instance since this instance is not a coordinator!");
+      case SUCCESS:
+        break;
+    }
+  }
+
+  void RegisterMainCoordinatorServer(const std::string &coordinator_socket_address,
+                                     const std::chrono::seconds instance_check_frequency,
+                                     const std::string &instance_name) override {
+    const auto maybe_ip_and_port =
+        io::network::Endpoint::ParseSocketOrAddress(coordinator_socket_address, std::nullopt);
+    if (!maybe_ip_and_port) {
+      throw QueryRuntimeException("Invalid socket address!");
+    }
+    const auto [ip, port] = *maybe_ip_and_port;
+    auto coordinator_client_config =
+        coordination::CoordinatorClientConfig{.instance_name = instance_name,
+                                              .ip_address = ip,
+                                              .port = port,
+                                              .health_check_frequency_sec = instance_check_frequency,
+                                              .ssl = std::nullopt};
+
+    auto status = coordinator_handler_.RegisterMainOnCoordinator(std::move(coordinator_client_config));
+    switch (status) {
+      using enum memgraph::coordination::RegisterMainReplicaCoordinatorStatus;
+      case NAME_EXISTS:
+        throw QueryRuntimeException("Couldn't register main instance since instance with such name already exists!");
+      case END_POINT_EXISTS:
+        throw QueryRuntimeException(
+            "Couldn't register main instance since instance with such endpoint already exists!");
+      case COULD_NOT_BE_PERSISTED:
+        throw QueryRuntimeException("Couldn't register main instance since it couldn't be persisted!");
+      case NOT_COORDINATOR:
+        throw QueryRuntimeException("Couldn't register main instance since this instance is not a coordinator!");
+      case SUCCESS:
+        break;
+    }
+  }
+
+  void RegisterInstanceOnCoordinator(const std::string &coordinator_socket_address,
+                                     const std::string &replication_socket_address,
+                                     const std::chrono::seconds instance_check_frequency,
+                                     const std::string &instance_name, CoordinatorQuery::SyncMode sync_mode) override {
+    const auto maybe_replication_ip_port =
+        io::network::Endpoint::ParseSocketOrAddress(replication_socket_address, std::nullopt);
+    if (!maybe_replication_ip_port) {
+      throw QueryRuntimeException("Invalid replication socket address!");
+    }
+
+    const auto maybe_coordinator_ip_port =
+        io::network::Endpoint::ParseSocketOrAddress(coordinator_socket_address, std::nullopt);
+    if (!maybe_replication_ip_port) {
+      throw QueryRuntimeException("Invalid replication socket address!");
+    }
+
+    const auto [replication_ip, replication_port] = *maybe_replication_ip_port;
+    const auto [coordinator_server_ip, coordinator_server_port] = *maybe_coordinator_ip_port;
+    const auto repl_config = coordination::CoordinatorClientConfig::ReplicationClientInfo{
+        .instance_name = instance_name,
+        .replication_mode = convertFromCoordinatorToReplicationMode(sync_mode),
+        .replication_ip_address = replication_ip,
+        .replication_port = replication_port};
+
+    auto coordinator_client_config =
+        coordination::CoordinatorClientConfig{.instance_name = instance_name,
+                                              .ip_address = coordinator_server_ip,
+                                              .port = coordinator_server_port,
+                                              .health_check_frequency_sec = instance_check_frequency,
+                                              .replication_client_info = repl_config,
+                                              .ssl = std::nullopt};
+
+    auto status = coordinator_handler_.RegisterInstanceOnCoordinator(std::move(coordinator_client_config));
+    switch (status) {
+      using enum memgraph::coordination::RegisterInstanceCoordinatorStatus;
+      case NAME_EXISTS:
+        throw QueryRuntimeException("Couldn't register replica instance since instance with such name already exists!");
+      case END_POINT_EXISTS:
+        throw QueryRuntimeException(
+            "Couldn't register replica instance since instance with such endpoint already exists!");
+      case COULD_NOT_BE_PERSISTED:
+        throw QueryRuntimeException("Couldn't register replica instance since it couldn't be persisted!");
+      case NOT_COORDINATOR:
+        throw QueryRuntimeException("Couldn't register replica instance since this instance is not a coordinator!");
+      case SUCCESS:
+        break;
+    }
+  }
+
+  void SetInstanceToMain(const std::string &instance_name) override {
+    auto status = coordinator_handler_.SetInstanceToMain(instance_name);
+    switch (status) {
+      using enum memgraph::coordination::SetInstanceToMainCoordinatorStatus;
+      case NO_INSTANCE_WITH_NAME:
+        throw QueryRuntimeException("No instance with such name!");
+      case NOT_COORDINATOR:
+        throw QueryRuntimeException("Couldn't set replica instance to main since this instance is not a coordinator!");
+      case COULD_NOT_PROMOTE_TO_MAIN:
+        throw QueryRuntimeException(
+            "Couldn't set replica instance to main. Check coordinator and replica for more logs");
+      case SUCCESS:
+        break;
+    }
+  }
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  void DoFailover() const override {
+    if (!FLAGS_coordinator) {
+      throw QueryRuntimeException("Only coordinator can register coordinator server!");
+    }
+
+    auto status = coordinator_handler_.DoFailover();
+    switch (status) {
+      using enum memgraph::coordination::DoFailoverStatus;
+      case ALL_REPLICAS_DOWN:
+        throw QueryRuntimeException("Failover aborted since all replicas are down!");
+      case MAIN_ALIVE:
+        throw QueryRuntimeException("Failover aborted since main is alive!");
+      case CLUSTER_UNINITIALIZED:
+        throw QueryRuntimeException("Failover aborted since cluster is uninitialized!");
+      case SUCCESS:
+        break;
+    }
+  }
+
+  // TODO: Remove this method, probably not needed.
+  std::vector<MainReplicaStatus> ShowMainReplicaStatus(
+      const std::vector<coordination::CoordinatorInstanceStatus> &replicas,
+      const std::optional<coordination::CoordinatorInstanceStatus> &main) const override {
+    std::vector<MainReplicaStatus> result{};
+    result.reserve(replicas.size() + 1);  // replicas + 1 main
+    std::ranges::transform(replicas, std::back_inserter(result), [](const auto &replica) -> MainReplicaStatus {
+      return {replica.instance_name, replica.socket_address, replica.is_alive, false};
+    });
+    if (main) {
+      result.emplace_back(main->instance_name, main->socket_address, main->is_alive, true);
+    }
+    return result;
+  }
+
+#endif
+
+#ifdef MG_ENTERPRISE
+  std::vector<coordination::CoordinatorInstanceStatus> ShowReplicasOnCoordinator() const override {
+    return coordinator_handler_.ShowReplicasOnCoordinator();
+  }
+
+  std::optional<coordination::CoordinatorInstanceStatus> ShowMainOnCoordinator() const override {
+    return coordinator_handler_.ShowMainOnCoordinator();
+  }
+#endif
+
+ private:
+  dbms::ReplicationHandler handler_;
+#ifdef MG_ENTERPRISE
+  dbms::CoordinatorHandler coordinator_handler_;
+#endif
+};
+
 /// returns false if the replication role can't be set
 /// @throw QueryRuntimeException if an error ocurred.
 
@@ -723,6 +968,15 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
   Callback callback;
   switch (repl_query->action_) {
     case ReplicationQuery::Action::SET_REPLICATION_ROLE: {
+#ifdef MG_ENTERPRISE
+      if (FLAGS_coordinator) {
+        if (repl_query->role_ == ReplicationQuery::ReplicationRole::REPLICA) {
+          throw QueryRuntimeException("Coordinator cannot become a replica!");
+        }
+        throw QueryRuntimeException("Coordinator cannot become main!");
+      }
+#endif
+
       auto port = EvaluateOptionalExpression(repl_query->port_, evaluator);
       std::optional<int64_t> maybe_port;
       if (port.IsInt()) {
@@ -743,6 +997,12 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
       return callback;
     }
     case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
+#ifdef MG_ENTERPRISE
+      if (FLAGS_coordinator) {
+        throw QueryRuntimeException("Coordinator doesn't have a replication role!");
+      }
+#endif
+
       callback.header = {"replication role"};
       callback.fn = [handler = ReplQueryHandler{dbms_handler}] {
         auto mode = handler.ShowReplicationRole();
@@ -758,7 +1018,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
       return callback;
     }
     case ReplicationQuery::Action::REGISTER_REPLICA: {
-      const auto &name = repl_query->replica_name_;
+      const auto &name = repl_query->instance_name_;
       const auto &sync_mode = repl_query->sync_mode_;
       auto socket_address = repl_query->socket_address_->Accept(evaluator);
       const auto replica_check_frequency = config.replication_replica_check_frequency;
@@ -769,20 +1029,27 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
         return std::vector<std::vector<TypedValue>>();
       };
       notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICA,
-                                  fmt::format("Replica {} is registered.", repl_query->replica_name_));
+                                  fmt::format("Replica {} is registered.", repl_query->instance_name_));
       return callback;
     }
+
     case ReplicationQuery::Action::DROP_REPLICA: {
-      const auto &name = repl_query->replica_name_;
+      const auto &name = repl_query->instance_name_;
       callback.fn = [handler = ReplQueryHandler{dbms_handler}, name]() mutable {
         handler.DropReplica(name);
         return std::vector<std::vector<TypedValue>>();
       };
       notifications->emplace_back(SeverityLevel::INFO, NotificationCode::DROP_REPLICA,
-                                  fmt::format("Replica {} is dropped.", repl_query->replica_name_));
+                                  fmt::format("Replica {} is dropped.", repl_query->instance_name_));
       return callback;
     }
     case ReplicationQuery::Action::SHOW_REPLICAS: {
+#ifdef MG_ENTERPRISE
+      if (FLAGS_coordinator) {
+        throw QueryRuntimeException("Coordinator cannot call SHOW REPLICAS! Use SHOW REPLICATION CLUSTER instead.");
+      }
+#endif
+
       callback.header = {
           "name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master",
           "state"};
@@ -833,6 +1100,183 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
   }
 }
 
+Callback HandleCoordinatorQuery(CoordinatorQuery *coordinator_query, const Parameters &parameters,
+                                dbms::DbmsHandler *dbms_handler, const query::InterpreterConfig &config,
+                                std::vector<Notification> *notifications) {
+  Callback callback;
+  switch (coordinator_query->action_) {
+    case CoordinatorQuery::Action::REGISTER_MAIN_COORDINATOR_SERVER: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can register coordinator server!");
+      }
+
+      throw QueryRuntimeException("Query is disabled for now");
+      return callback;
+#endif
+    }
+    case CoordinatorQuery::Action::REGISTER_REPLICA_COORDINATOR_SERVER: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can register coordinator server!");
+      }
+
+      throw QueryRuntimeException("Query is disabled for now");
+      return callback;
+#endif
+    }
+    case CoordinatorQuery::Action::REGISTER_INSTANCE_ON_COORDINATOR: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can register coordinator server!");
+      }
+      // TODO: MemoryResource for EvaluationContext, it should probably be passed as
+      // the argument to Callback.
+      EvaluationContext evaluation_context{.timestamp = QueryTimestamp(), .parameters = parameters};
+      auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context};
+
+      auto coordinator_socket_address_tv = coordinator_query->coordinator_socket_address_->Accept(evaluator);
+      auto replication_socket_address_tv = coordinator_query->replication_socket_address_->Accept(evaluator);
+      callback.fn = [handler = CoordQueryHandler{dbms_handler}, coordinator_socket_address_tv,
+                     replication_socket_address_tv, main_check_frequency = config.replication_replica_check_frequency,
+                     instance_name = coordinator_query->instance_name_,
+                     sync_mode = coordinator_query->sync_mode_]() mutable {
+        handler.RegisterInstanceOnCoordinator(std::string(coordinator_socket_address_tv.ValueString()),
+                                              std::string(replication_socket_address_tv.ValueString()),
+                                              main_check_frequency, instance_name, sync_mode);
+        return std::vector<std::vector<TypedValue>>();
+      };
+
+      notifications->emplace_back(
+          SeverityLevel::INFO, NotificationCode::REGISTER_COORDINATOR_SERVER,
+          fmt::format("Coordinator has registered coordinator server on {} for instance {}.",
+                      coordinator_socket_address_tv.ValueString(), coordinator_query->instance_name_));
+      return callback;
+#endif
+    }
+    case CoordinatorQuery::Action::SET_INSTANCE_TO_MAIN: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can register coordinator server!");
+      }
+      // TODO: MemoryResource for EvaluationContext, it should probably be passed as
+      // the argument to Callback.
+      EvaluationContext evaluation_context{.timestamp = QueryTimestamp(), .parameters = parameters};
+      auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context};
+
+      callback.fn = [handler = CoordQueryHandler{dbms_handler},
+                     instance_name = coordinator_query->instance_name_]() mutable {
+        handler.SetInstanceToMain(instance_name);
+        return std::vector<std::vector<TypedValue>>();
+      };
+
+      return callback;
+#endif
+    }
+    case CoordinatorQuery::Action::SHOW_REPLICATION_CLUSTER: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can run SHOW REPLICATION CLUSTER.");
+      }
+
+      callback.header = {"name", "socket_address", "alive", "role"};
+      callback.fn = [handler = CoordQueryHandler{dbms_handler}, replica_nfields = callback.header.size()]() mutable {
+        const auto result_status =
+            handler.ShowMainReplicaStatus(handler.ShowReplicasOnCoordinator(), handler.ShowMainOnCoordinator());
+        std::vector<std::vector<TypedValue>> result{};
+        result.reserve(result_status.size());
+
+        std::ranges::transform(result_status, std::back_inserter(result),
+                               [](const auto &status) -> std::vector<TypedValue> {
+                                 return {TypedValue{status.name}, TypedValue{status.socket_address},
+                                         TypedValue{status.alive}, TypedValue{status.is_main ? "main" : "replica"}};
+                               });
+        return result;
+      };
+      return callback;
+#endif
+    }
+    case CoordinatorQuery::Action::DO_FAILOVER: {
+      if (!license::global_license_checker.IsEnterpriseValidFast()) {
+        throw QueryException("Trying to use enterprise feature without a valid license.");
+      }
+#ifdef MG_ENTERPRISE
+      if constexpr (!coordination::allow_ha) {
+        throw QueryRuntimeException(
+            "High availability is experimental feature. Please set MG_EXPERIMENTAL_HIGH_AVAILABILITY compile flag to "
+            "be able to use this functionality.");
+      }
+      if (!FLAGS_coordinator) {
+        throw QueryRuntimeException("Only coordinator can run DO FAILOVER!");
+      }
+
+      callback.header = {"name", "socket_address", "alive", "role"};
+      callback.fn = [handler = CoordQueryHandler{dbms_handler}]() mutable {
+        handler.DoFailover();
+
+        const auto result_status =
+            handler.ShowMainReplicaStatus(handler.ShowReplicasOnCoordinator(), handler.ShowMainOnCoordinator());
+        std::vector<std::vector<TypedValue>> result{};
+        result.reserve(result_status.size());
+
+        std::ranges::transform(result_status, std::back_inserter(result),
+                               [](const auto &status) -> std::vector<TypedValue> {
+                                 return {TypedValue{status.name}, TypedValue{status.socket_address},
+                                         TypedValue{status.alive}, TypedValue{status.is_main ? "main" : "replica"}};
+                               });
+        return result;
+      };
+
+      notifications->emplace_back(SeverityLevel::INFO, NotificationCode::DO_FAILOVER,
+                                  "DO FAILOVER called on coordinator.");
+      return callback;
+#endif
+    }
+      return callback;
+  }
+}
+
 stream::CommonStreamInfo GetCommonStreamInfo(StreamQuery *stream_query, ExpressionVisitor<TypedValue> &evaluator) {
   return {
       .batch_interval = GetOptionalValue<std::chrono::milliseconds>(stream_query->batch_interval_, evaluator)
@@ -2286,6 +2730,34 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit
   // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
 }
 
+PreparedQuery PrepareCoordinatorQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
+                                      std::vector<Notification> *notifications, dbms::DbmsHandler &dbms_handler,
+                                      const InterpreterConfig &config) {
+  if (in_explicit_transaction) {
+    throw CoordinatorModificationInMulticommandTxException();
+  }
+
+  auto *coordinator_query = utils::Downcast<CoordinatorQuery>(parsed_query.query);
+  auto callback =
+      HandleCoordinatorQuery(coordinator_query, parsed_query.parameters, &dbms_handler, config, notifications);
+
+  return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
+                       [callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
+                           AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
+                         if (UNLIKELY(!pull_plan)) {
+                           pull_plan = std::make_shared<PullPlanVector>(callback_fn());
+                         }
+
+                         if (pull_plan->Pull(stream, n)) [[likely]] {
+                           return QueryHandlerResult::COMMIT;
+                         }
+                         return std::nullopt;
+                       },
+                       RWType::NONE};
+  // False positive report for the std::make_shared above
+  // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
+}
+
 PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_transaction, CurrentDB &current_db) {
   if (in_explicit_transaction) {
     throw LockPathModificationInMulticommandTxException();
@@ -3765,6 +4237,13 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
     //   /* something */
     // }
 
+#ifdef MG_ENTERPRISE
+    if (FLAGS_coordinator && !utils::Downcast<CoordinatorQuery>(parsed_query.query) &&
+        !utils::Downcast<SettingQuery>(parsed_query.query)) {
+      throw QueryRuntimeException("Coordinator can run only coordinator queries!");
+    }
+#endif
+
     utils::Timer planning_timer;
     PreparedQuery prepared_query;
     utils::MemoryResource *memory_resource =
@@ -3807,6 +4286,10 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
       prepared_query =
           PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
                                   *interpreter_context_->dbms_handler, interpreter_context_->config);
+    } else if (utils::Downcast<CoordinatorQuery>(parsed_query.query)) {
+      prepared_query =
+          PrepareCoordinatorQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
+                                  *interpreter_context_->dbms_handler, interpreter_context_->config);
     } else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
       prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_);
     } else if (utils::Downcast<FreeMemoryQuery>(parsed_query.query)) {

src/query/interpreter.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,6 +51,10 @@
 #include "utils/timer.hpp"
 #include "utils/tsc.hpp"
 
+#ifdef MG_ENTERPRISE
+#include "coordination/coordinator_instance_status.hpp"
+#endif
+
 namespace memgraph::metrics {
 extern const Event FailedQuery;
 extern const Event FailedPrepare;
@@ -106,6 +110,75 @@ class ReplicationQueryHandler {
   virtual std::vector<Replica> ShowReplicas() const = 0;
 };
 
+class CoordinatorQueryHandler {
+ public:
+  CoordinatorQueryHandler() = default;
+  virtual ~CoordinatorQueryHandler() = default;
+
+  CoordinatorQueryHandler(const CoordinatorQueryHandler &) = default;
+  CoordinatorQueryHandler &operator=(const CoordinatorQueryHandler &) = default;
+
+  CoordinatorQueryHandler(CoordinatorQueryHandler &&) = default;
+  CoordinatorQueryHandler &operator=(CoordinatorQueryHandler &&) = default;
+
+  struct Replica {
+    std::string name;
+    std::string socket_address;
+    ReplicationQuery::SyncMode sync_mode;
+    std::optional<double> timeout;
+    uint64_t current_timestamp_of_replica;
+    uint64_t current_number_of_timestamp_behind_master;
+    ReplicationQuery::ReplicaState state;
+  };
+
+#ifdef MG_ENTERPRISE
+  struct MainReplicaStatus {
+    std::string_view name;
+    std::string_view socket_address;
+    bool alive;
+    bool is_main;
+
+    MainReplicaStatus(std::string_view name, std::string_view socket_address, bool alive, bool is_main)
+        : name{name}, socket_address{socket_address}, alive{alive}, is_main{is_main} {}
+  };
+#endif
+
+#ifdef MG_ENTERPRISE
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual void RegisterReplicaCoordinatorServer(const std::string &replication_socket_address,
+                                                const std::string &coordinator_socket_address,
+                                                const std::chrono::seconds instance_check_frequency,
+                                                const std::string &instance_name,
+                                                CoordinatorQuery::SyncMode sync_mode) = 0;
+  virtual void RegisterMainCoordinatorServer(const std::string &socket_address,
+                                             const std::chrono::seconds instance_check_frequency,
+                                             const std::string &instance_name) = 0;
+
+  virtual void RegisterInstanceOnCoordinator(const std::string &coordinator_socket_address,
+                                             const std::string &replication_socket_address,
+                                             const std::chrono::seconds instance_check_frequency,
+                                             const std::string &instance_name,
+                                             CoordinatorQuery::SyncMode sync_mode) = 0;
+
+  virtual void SetInstanceToMain(const std::string &instance_name) = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual std::vector<coordination::CoordinatorInstanceStatus> ShowReplicasOnCoordinator() const = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual std::optional<coordination::CoordinatorInstanceStatus> ShowMainOnCoordinator() const = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual void DoFailover() const = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual std::vector<MainReplicaStatus> ShowMainReplicaStatus(
+      const std::vector<coordination::CoordinatorInstanceStatus> &replicas,
+      const std::optional<coordination::CoordinatorInstanceStatus> &main) const = 0;
+
+#endif
+};
+
 class AnalyzeGraphQueryHandler {
  public:
   AnalyzeGraphQueryHandler() = default;

src/query/metadata.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -66,6 +66,12 @@ constexpr std::string_view GetCodeString(const NotificationCode code) {
       return "PlanHinting"sv;
     case NotificationCode::REGISTER_REPLICA:
       return "RegisterReplica"sv;
+#ifdef MG_ENTERPRISE
+    case NotificationCode::REGISTER_COORDINATOR_SERVER:
+      return "RegisterCoordinatorServer"sv;
+    case NotificationCode::DO_FAILOVER:
+      return "DoFailover"sv;
+#endif
     case NotificationCode::REPLICA_PORT_WARNING:
       return "ReplicaPortWarning"sv;
     case NotificationCode::SET_REPLICA:

src/query/metadata.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -42,6 +42,10 @@ enum class NotificationCode : uint8_t {
   PLAN_HINTING,
   REPLICA_PORT_WARNING,
   REGISTER_REPLICA,
+#ifdef MG_ENTERPRISE
+  REGISTER_COORDINATOR_SERVER,
+  DO_FAILOVER,
+#endif
   SET_REPLICA,
   START_STREAM,
   START_ALL_STREAMS,

src/replication/CMakeLists.txt

@@ -5,8 +5,6 @@ target_sources(mg-replication
         include/replication/state.hpp
         include/replication/epoch.hpp
         include/replication/config.hpp
-        include/replication/mode.hpp
-        include/replication/messages.hpp
         include/replication/role.hpp
         include/replication/status.hpp
         include/replication/replication_client.hpp
@@ -17,7 +15,6 @@ target_sources(mg-replication
         epoch.cpp
         config.cpp
         status.cpp
-        messages.cpp
         replication_client.cpp
         replication_server.cpp
 )
@@ -25,6 +22,6 @@ target_include_directories(mg-replication PUBLIC include)
 
 find_package(fmt REQUIRED)
 target_link_libraries(mg-replication
-    PUBLIC mg::utils mg::kvstore lib::json mg::rpc mg::slk
+    PUBLIC mg::utils mg::kvstore lib::json mg::rpc mg::slk mg::io mg::repl_coord_glue
     PRIVATE fmt::fmt
 )

src/replication/include/replication/config.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,7 @@
 #include <cstdint>
 #include <optional>
 #include <string>
-#include "replication/mode.hpp"
+#include "replication_coordination_glue/mode.hpp"
 
 namespace memgraph::replication {
 
@@ -24,7 +24,7 @@ inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
 
 struct ReplicationClientConfig {
   std::string name;
-  ReplicationMode mode{};
+  replication_coordination_glue::ReplicationMode mode{};
   std::string ip_address;
   uint16_t port{};
 

src/replication/include/replication/replication_client.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,7 +12,7 @@
 #pragma once
 
 #include "replication/config.hpp"
-#include "replication/messages.hpp"
+#include "replication_coordination_glue/messages.hpp"
 #include "rpc/client.hpp"
 #include "utils/scheduler.hpp"
 #include "utils/thread_pool.hpp"
@@ -42,7 +42,7 @@ struct ReplicationClient {
         try {
           bool success = false;
           {
-            auto stream{rpc_client_.Stream<memgraph::replication::FrequentHeartbeatRpc>()};
+            auto stream{rpc_client_.Stream<memgraph::replication_coordination_glue::FrequentHeartbeatRpc>()};
             success = stream.AwaitResponse().success;
           }
           if (success) {
@@ -60,7 +60,7 @@ struct ReplicationClient {
   rpc::Client rpc_client_;
   std::chrono::seconds replica_check_frequency_;
 
-  memgraph::replication::ReplicationMode mode_{memgraph::replication::ReplicationMode::SYNC};
+  replication_coordination_glue::ReplicationMode mode_{replication_coordination_glue::ReplicationMode::SYNC};
   // This thread pool is used for background tasks so we don't
   // block the main storage thread
   // We use only 1 thread for 2 reasons:

src/replication/include/replication/state.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,9 +14,9 @@
 #include "kvstore/kvstore.hpp"
 #include "replication/config.hpp"
 #include "replication/epoch.hpp"
-#include "replication/mode.hpp"
 #include "replication/replication_client.hpp"
 #include "replication/role.hpp"
+#include "replication_coordination_glue/mode.hpp"
 #include "replication_server.hpp"
 #include "status.hpp"
 #include "utils/result.hpp"

src/replication/include/replication/status.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -42,7 +42,7 @@ struct MainRole {
 
 // fragment of key: "__replication_role"
 struct ReplicaRole {
-  ReplicationServerConfig config;
+  ReplicationServerConfig config{};
   friend bool operator==(ReplicaRole const &, ReplicaRole const &) = default;
 };
 

src/replication/replication_client.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -28,7 +28,7 @@ ReplicationClient::ReplicationClient(const memgraph::replication::ReplicationCli
       mode_{config.mode} {}
 
 ReplicationClient::~ReplicationClient() {
-  auto endpoint = rpc_client_.Endpoint();
+  const auto endpoint = rpc_client_.Endpoint();
   try {
     spdlog::trace("Closing replication client on {}:{}", endpoint.address, endpoint.port);
   } catch (...) {

src/replication/replication_server.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,7 +10,7 @@
 // licenses/APL.txt.
 
 #include "replication/replication_server.hpp"
-#include "replication/messages.hpp"
+#include "replication_coordination_glue/messages.hpp"
 
 namespace memgraph::replication {
 namespace {
@@ -32,9 +32,9 @@ ReplicationServer::ReplicationServer(const memgraph::replication::ReplicationSer
     : rpc_server_context_{CreateServerContext(config)},
       rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_,
                   kReplicationServerThreads} {
-  rpc_server_.Register<FrequentHeartbeatRpc>([](auto *req_reader, auto *res_builder) {
+  rpc_server_.Register<replication_coordination_glue::FrequentHeartbeatRpc>([](auto *req_reader, auto *res_builder) {
     spdlog::debug("Received FrequentHeartbeatRpc");
-    FrequentHeartbeatHandler(req_reader, res_builder);
+    replication_coordination_glue::FrequentHeartbeatHandler(req_reader, res_builder);
   });
 }
 

src/replication/state.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -218,10 +218,12 @@ bool ReplicationState::TryPersistRegisteredReplica(const ReplicationClientConfig
 
 bool ReplicationState::SetReplicationRoleMain() {
   auto new_epoch = utils::GenerateUUID();
+
   if (!TryPersistRoleMain(new_epoch)) {
     return false;
   }
   replication_data_ = RoleMainData{ReplicationEpoch{new_epoch}};
+
   return true;
 }
 
@@ -236,6 +238,7 @@ bool ReplicationState::SetReplicationRoleReplica(const ReplicationServerConfig &
 utils::BasicResult<RegisterReplicaError, ReplicationClient *> ReplicationState::RegisterReplica(
     const ReplicationClientConfig &config) {
   auto const replica_handler = [](RoleReplicaData const &) { return RegisterReplicaError::NOT_MAIN; };
+
   ReplicationClient *client{nullptr};
   auto const main_handler = [&client, &config, this](RoleMainData &mainData) -> RegisterReplicaError {
     // name check

src/replication/status.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -95,7 +95,7 @@ void from_json(const nlohmann::json &j, ReplicationReplicaEntry &p) {
   auto seconds = j.at(kCheckFrequency).get<std::chrono::seconds::rep>();
   auto config = ReplicationClientConfig{
       .name = j.at(kReplicaName).get<std::string>(),
-      .mode = j.at(kSyncMode).get<ReplicationMode>(),
+      .mode = j.at(kSyncMode).get<replication_coordination_glue::ReplicationMode>(),
       .ip_address = j.at(kIpAddress).get<std::string>(),
       .port = j.at(kPort).get<uint16_t>(),
       .replica_check_frequency = std::chrono::seconds{seconds},

src/replication_coordination_glue/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(mg-repl_coord_glue STATIC )
+add_library(mg::repl_coord_glue ALIAS mg-repl_coord_glue)
+
+target_sources(mg-repl_coord_glue
+        PUBLIC
+        messages.hpp
+        mode.hpp
+
+        PRIVATE
+        messages.cpp
+)
+
+target_link_libraries(mg-repl_coord_glue mg-rpc mg-slk)

src/replication_coordination_glue/messages.cpp

@@ -9,31 +9,33 @@
 // by the Apache License, Version 2.0, included in the file
 // licenses/APL.txt.
 
-#include "replication/messages.hpp"
+#include "replication_coordination_glue/messages.hpp"
 #include "rpc/messages.hpp"
 #include "slk/serialization.hpp"
 #include "slk/streams.hpp"
 
 namespace memgraph::slk {
 // Serialize code for FrequentHeartbeatRes
-void Save(const memgraph::replication::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) {
+void Save(const memgraph::replication_coordination_glue::FrequentHeartbeatRes &self, memgraph::slk::Builder *builder) {
   memgraph::slk::Save(self.success, builder);
 }
-void Load(memgraph::replication::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) {
+void Load(memgraph::replication_coordination_glue::FrequentHeartbeatRes *self, memgraph::slk::Reader *reader) {
   memgraph::slk::Load(&self->success, reader);
 }
 
 // Serialize code for FrequentHeartbeatReq
-void Save(const memgraph::replication::FrequentHeartbeatReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
+void Save(const memgraph::replication_coordination_glue::FrequentHeartbeatReq & /*self*/,
+          memgraph::slk::Builder * /*builder*/) {
   /* Nothing to serialize */
 }
-void Load(memgraph::replication::FrequentHeartbeatReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
+void Load(memgraph::replication_coordination_glue::FrequentHeartbeatReq * /*self*/,
+          memgraph::slk::Reader * /*reader*/) {
   /* Nothing to serialize */
 }
 
 }  // namespace memgraph::slk
 
-namespace memgraph::replication {
+namespace memgraph::replication_coordination_glue {
 
 constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_REQ, "FrequentHeartbeatReq",
                                                       nullptr};
@@ -62,4 +64,4 @@ void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder
   memgraph::slk::Save(res, res_builder);
 }
 
-}  // namespace memgraph::replication
+}  // namespace memgraph::replication_coordination_glue

src/replication_coordination_glue/messages.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,11 +14,11 @@
 #include "rpc/messages.hpp"
 #include "slk/serialization.hpp"
 
-namespace memgraph::replication {
+namespace memgraph::replication_coordination_glue {
 
 struct FrequentHeartbeatReq {
-  static const utils::TypeInfo kType;                            // TODO: make constexpr?
-  static const utils::TypeInfo &GetTypeInfo() { return kType; }  // WHAT?
+  static const utils::TypeInfo kType;  // TODO: make constexpr?
+  static const utils::TypeInfo &GetTypeInfo() { return kType; }
 
   static void Load(FrequentHeartbeatReq *self, memgraph::slk::Reader *reader);
   static void Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder);
@@ -41,4 +41,4 @@ using FrequentHeartbeatRpc = rpc::RequestResponse<FrequentHeartbeatReq, Frequent
 
 void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder);
 
-}  // namespace memgraph::replication
+}  // namespace memgraph::replication_coordination_glue

src/replication_coordination_glue/mode.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,7 +10,9 @@
 // licenses/APL.txt.
 
 #pragma once
+
 #include <cstdint>
-namespace memgraph::replication {
+
+namespace memgraph::replication_coordination_glue {
 enum class ReplicationMode : std::uint8_t { SYNC, ASYNC };
-}
+}  // namespace memgraph::replication_coordination_glue

src/slk/serialization.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -55,6 +55,12 @@ class SlkDecodeException : public utils::BasicException {
 // here because C++ doesn't know how to resolve the function call if it isn't in
 // the global namespace.
 
+template <typename T>
+inline void Save(const std::vector<T> &obj, Builder *builder,
+                 std::function<void(const T &, Builder *)> item_save_function);
+template <typename T>
+inline void Load(std::vector<T> *obj, Reader *reader, std::function<void(T *, Reader *)> item_load_function);
+
 template <typename T>
 void Save(const std::vector<T> &obj, Builder *builder);
 template <typename T>
@@ -486,4 +492,17 @@ inline void Load(utils::TypeId *obj, Reader *reader) {
   *obj = utils::TypeId(utils::MemcpyCast<enum_type>(obj_encoded));
 }
 
+template <utils::Enum T>
+void Save(const T &enum_value, slk::Builder *builder) {
+  slk::Save(utils::UnderlyingCast(enum_value), builder);
+}
+
+template <utils::Enum T>
+void Load(T *enum_value, slk::Reader *reader) {
+  using UnderlyingType = std::underlying_type_t<T>;
+  UnderlyingType value;
+  slk::Load(&value, reader);
+  *enum_value = static_cast<T>(value);
+}
+
 }  // namespace memgraph::slk

src/storage/v2/property_store.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -1050,14 +1050,11 @@ bool PropertyStore::HasProperty(PropertyId property) const {
   return ExistsSpecificProperty(&reader, property) == ExpectedPropertyStatus::EQUAL;
 }
 
-/// TODO: andi write a unit test for it
 bool PropertyStore::HasAllProperties(const std::set<PropertyId> &properties) const {
   return std::all_of(properties.begin(), properties.end(), [this](const auto &prop) { return HasProperty(prop); });
 }
 
-/// TODO: andi write a unit test for it
 bool PropertyStore::HasAllPropertyValues(const std::vector<PropertyValue> &property_values) const {
-  /// TODO: andi extract this into a private method
   auto property_map = Properties();
   std::vector<PropertyValue> all_property_values;
   transform(property_map.begin(), property_map.end(), back_inserter(all_property_values),

src/storage/v2/property_store.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -55,7 +55,6 @@ class PropertyStore {
 
   /// Checks whether all property values in the vector `property_values` exist in the store. The time
   /// complexity of this function is O(n^2).
-  /// TODO: andi Not so sure it is quadratic complexity
   bool HasAllPropertyValues(const std::vector<PropertyValue> &property_values) const;
 
   /// Extracts property values for all property ids in the set `properties`. The time

src/storage/v2/replication/global.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -31,7 +31,7 @@ struct TimestampInfo {
 
 struct ReplicaInfo {
   std::string name;
-  memgraph::replication::ReplicationMode mode;
+  replication_coordination_glue::ReplicationMode mode;
   io::network::Endpoint endpoint;
   replication::ReplicaState state;
   TimestampInfo timestamp_info;

src/storage/v2/replication/replication_client.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -201,7 +201,7 @@ bool ReplicationStorageClient::FinalizeTransactionReplication(Storage *storage)
     }
   };
 
-  if (client_.mode_ == memgraph::replication::ReplicationMode::ASYNC) {
+  if (client_.mode_ == replication_coordination_glue::ReplicationMode::ASYNC) {
     client_.thread_pool_.AddTask([task = std::move(task)] { (void)task(); });
     return true;
   }

src/storage/v2/replication/replication_client.hpp

@@ -13,8 +13,8 @@
 
 #include "replication/config.hpp"
 #include "replication/epoch.hpp"
-#include "replication/messages.hpp"
 #include "replication/replication_client.hpp"
+#include "replication_coordination_glue/messages.hpp"
 #include "rpc/client.hpp"
 #include "storage/v2/durability/storage_global_operation.hpp"
 #include "storage/v2/id_types.hpp"
@@ -93,7 +93,7 @@ class ReplicationStorageClient {
   ~ReplicationStorageClient() = default;
 
   // TODO Remove the client related functions
-  auto Mode() const -> memgraph::replication::ReplicationMode { return client_.mode_; }
+  auto Mode() const -> memgraph::replication_coordination_glue::ReplicationMode { return client_.mode_; }
   auto Name() const -> std::string const & { return client_.name_; }
   auto Endpoint() const -> io::network::Endpoint const & { return client_.rpc_client_.Endpoint(); }
 

src/storage/v2/replication/replication_storage_state.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -59,7 +59,7 @@ bool ReplicationStorageState::FinalizeTransaction(uint64_t timestamp, Storage *s
       client->IfStreamingTransaction([&](auto &stream) { stream.AppendTransactionEnd(timestamp); });
       const auto finalized = client->FinalizeTransactionReplication(storage);
 
-      if (client->Mode() == memgraph::replication::ReplicationMode::SYNC) {
+      if (client->Mode() == replication_coordination_glue::ReplicationMode::SYNC) {
         finalized_on_all_replicas = finalized && finalized_on_all_replicas;
       }
     }

src/storage/v2/replication/slk.hpp

@@ -1,4 +1,4 @@
-// Copyright 2022 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,17 +25,4 @@ void Load(storage::Gid *gid, slk::Reader *reader);
 void Save(const storage::PropertyValue &value, slk::Builder *builder);
 void Load(storage::PropertyValue *value, slk::Reader *reader);
 
-template <utils::Enum T>
-void Save(const T &enum_value, slk::Builder *builder) {
-  slk::Save(utils::UnderlyingCast(enum_value), builder);
-}
-
-template <utils::Enum T>
-void Load(T *enum_value, slk::Reader *reader) {
-  using UnderlyingType = std::underlying_type_t<T>;
-  UnderlyingType value;
-  slk::Load(&value, reader);
-  *enum_value = static_cast<T>(value);
-}
-
 }  // namespace memgraph::slk

src/utils/scheduler.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -58,28 +58,40 @@ class Scheduler {
         // the start of the program. Since Server will log some messages on
         // the program start we let him log first and we make sure by first
         // waiting that funcion f will not log before it.
+        // Check for pause also.
         std::unique_lock<std::mutex> lk(mutex_);
         auto now = std::chrono::system_clock::now();
         start_time += pause;
         if (start_time > now) {
-          condition_variable_.wait_until(lk, start_time, [&] { return is_working_.load() == false; });
+          condition_variable_.wait_until(lk, start_time, [&] { return !is_working_.load(); });
         } else {
           start_time = now;
         }
 
+        pause_cv_.wait(lk, [&] { return !is_paused_.load(); });
+
         if (!is_working_) break;
         f();
       }
     });
   }
 
+  void Resume() {
+    is_paused_.store(false);
+    pause_cv_.notify_one();
+  }
+
+  void Pause() { is_paused_.store(true); }
+
   /**
    * @brief Stops the thread execution. This is a blocking call and may take as
    * much time as one call to the function given previously to Run takes.
    * @throw std::system_error
    */
   void Stop() {
+    is_paused_.store(false);
     is_working_.store(false);
+    pause_cv_.notify_one();
     condition_variable_.notify_one();
     if (thread_.joinable()) thread_.join();
   }
@@ -97,6 +109,16 @@ class Scheduler {
    */
   std::atomic<bool> is_working_{false};
 
+  /**
+   * Variable is true when thread is paused.
+   */
+  std::atomic<bool> is_paused_{false};
+
+  /*
+   * Wait until the thread is resumed.
+   */
+  std::condition_variable pause_cv_;
+
   /**
    * Mutex used to synchronize threads using condition variable.
    */

src/utils/typeinfo.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -84,6 +84,12 @@ enum class TypeId : uint64_t {
   REP_TIMESTAMP_REQ,
   REP_TIMESTAMP_RES,
 
+  // Coordinator
+  COORD_FAILOVER_REQ,
+  COORD_FAILOVER_RES,
+  COORD_SET_REPL_MAIN_REQ,
+  COORD_SET_REPL_MAIN_RES,
+
   // AST
   AST_LABELIX,
   AST_PROPERTYIX,
@@ -191,6 +197,7 @@ enum class TypeId : uint64_t {
   AST_SHOW_DATABASES,
   AST_EDGE_IMPORT_MODE_QUERY,
   AST_PATTERN_COMPREHENSION,
+  AST_COORDINATOR_QUERY,
   // Symbol
   SYMBOL,
 };

tests/e2e/CMakeLists.txt

@@ -77,6 +77,11 @@ add_subdirectory(query_modules_storage_modes)
 add_subdirectory(garbage_collection)
 add_subdirectory(query_planning)
 
+if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
+  add_subdirectory(high_availability_experimental)
+endif ()
+
+
 copy_e2e_python_files(pytest_runner pytest_runner.sh "")
 copy_e2e_python_files(x x.sh "")
 file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/memgraph-selfsigned.crt DESTINATION ${CMAKE_CURRENT_BINARY_DIR})

tests/e2e/configuration/CMakeLists.txt

@@ -1,5 +1,5 @@
 function(copy_configuration_check_e2e_python_files FILE_NAME)
-    copy_e2e_python_files(write_procedures ${FILE_NAME})
+    copy_e2e_python_files(configuration ${FILE_NAME})
 endfunction()
 
 copy_configuration_check_e2e_python_files(default_config.py)

tests/e2e/configuration/default_config.py

@@ -66,6 +66,8 @@ startup_config_dict = {
         "Time in seconds after which inactive Bolt sessions will be closed.",
     ),
     "cartesian_product_enabled": ("true", "true", "Enable cartesian product expansion."),
+    "coordinator": ("false", "false", "Controls whether the instance is a replication coordinator."),
+    "coordinator_server_port": ("0", "0", "Port on which coordinator servers will be started."),
     "data_directory": ("mg_data", "mg_data", "Path to directory in which to save all permanent data."),
     "data_recovery_on_startup": (
         "false",

tests/e2e/configuration/storage_info.py

@@ -11,7 +11,6 @@
 
 import sys
 
-import default_config
 import mgclient
 import pytest
 

tests/e2e/high_availability_experimental/CMakeLists.txt

@@ -0,0 +1,13 @@
+find_package(gflags REQUIRED)
+
+copy_e2e_python_files(ha_experimental coordinator.py)
+copy_e2e_python_files(ha_experimental client_initiated_failover.py)
+copy_e2e_python_files(ha_experimental automatic_failover.py)
+copy_e2e_python_files(ha_experimental uninitialized_cluster.py)
+copy_e2e_python_files(ha_experimental common.py)
+copy_e2e_python_files(ha_experimental conftest.py)
+copy_e2e_python_files(ha_experimental workloads.yaml)
+
+copy_e2e_python_files_from_parent_folder(ha_experimental ".." memgraph.py)
+copy_e2e_python_files_from_parent_folder(ha_experimental ".." interactive_mg_runner.py)
+copy_e2e_python_files_from_parent_folder(ha_experimental ".." mg_utils.py)

tests/e2e/high_availability_experimental/automatic_failover.py

@@ -0,0 +1,95 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import os
+import sys
+
+import interactive_mg_runner
+import pytest
+from common import execute_and_fetch_all
+from mg_utils import mg_sleep_and_assert
+
+interactive_mg_runner.SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+interactive_mg_runner.PROJECT_DIR = os.path.normpath(
+    os.path.join(interactive_mg_runner.SCRIPT_DIR, "..", "..", "..", "..")
+)
+interactive_mg_runner.BUILD_DIR = os.path.normpath(os.path.join(interactive_mg_runner.PROJECT_DIR, "build"))
+interactive_mg_runner.MEMGRAPH_BINARY = os.path.normpath(os.path.join(interactive_mg_runner.BUILD_DIR, "memgraph"))
+
+MEMGRAPH_INSTANCES_DESCRIPTION = {
+    "instance_1": {
+        "args": ["--bolt-port", "7688", "--log-level", "TRACE", "--coordinator-server-port", "10011"],
+        "log_file": "replica1.log",
+        "setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"],
+    },
+    "instance_2": {
+        "args": ["--bolt-port", "7689", "--log-level", "TRACE", "--coordinator-server-port", "10012"],
+        "log_file": "replica2.log",
+        "setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"],
+    },
+    "instance_3": {
+        "args": ["--bolt-port", "7687", "--log-level", "TRACE", "--coordinator-server-port", "10013"],
+        "log_file": "main.log",
+        "setup_queries": [
+            "REGISTER REPLICA instance_1 SYNC TO '127.0.0.1:10001'",
+            "REGISTER REPLICA instance_2 SYNC TO '127.0.0.1:10002'",
+        ],
+    },
+    "coordinator": {
+        "args": ["--bolt-port", "7690", "--log-level=TRACE", "--coordinator"],
+        "log_file": "replica3.log",
+        "setup_queries": [
+            "REGISTER REPLICA instance_1 SYNC TO '127.0.0.1:10001' WITH COORDINATOR SERVER ON '127.0.0.1:10011';",
+            "REGISTER REPLICA instance_2 SYNC TO '127.0.0.1:10002' WITH COORDINATOR SERVER ON '127.0.0.1:10012';",
+            "REGISTER MAIN instance_3 WITH COORDINATOR SERVER ON '127.0.0.1:10013';",
+        ],
+    },
+}
+
+
+def test_simple_automatic_failover(connection):
+    interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
+
+    main_cursor = connection(7687, "instance_3").cursor()
+    expected_data_on_main = {
+        ("instance_1", "127.0.0.1:10001", "sync", 0, 0, "ready"),
+        ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
+    }
+    actual_data_on_main = set(execute_and_fetch_all(main_cursor, "SHOW REPLICAS;"))
+    assert actual_data_on_main == expected_data_on_main
+
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
+
+    coord_cursor = connection(7690, "coordinator").cursor()
+
+    def retrieve_data_show_repl_cluster():
+        return set(execute_and_fetch_all(coord_cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data_on_coord = {
+        ("instance_1", "127.0.0.1:10011", True, "main"),
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_3", "127.0.0.1:10013", False, "main"),  # TODO: (andi) Include or exclude dead main from the result?
+    }
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data_show_repl_cluster)
+
+    new_main_cursor = connection(7688, "instance_1").cursor()
+
+    def retrieve_data_show_replicas():
+        return set(execute_and_fetch_all(new_main_cursor, "SHOW REPLICAS;"))
+
+    expected_data_on_new_main = {
+        ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
+    }
+    mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/high_availability_experimental/client_initiated_failover.py

@@ -0,0 +1,207 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import os
+import sys
+
+import interactive_mg_runner
+import pytest
+from common import execute_and_fetch_all
+from mg_utils import mg_sleep_and_assert
+
+interactive_mg_runner.SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+interactive_mg_runner.PROJECT_DIR = os.path.normpath(
+    os.path.join(interactive_mg_runner.SCRIPT_DIR, "..", "..", "..", "..")
+)
+interactive_mg_runner.BUILD_DIR = os.path.normpath(os.path.join(interactive_mg_runner.PROJECT_DIR, "build"))
+interactive_mg_runner.MEMGRAPH_BINARY = os.path.normpath(os.path.join(interactive_mg_runner.BUILD_DIR, "memgraph"))
+
+MEMGRAPH_INSTANCES_DESCRIPTION = {
+    "instance_1": {
+        "args": ["--bolt-port", "7688", "--log-level", "TRACE", "--coordinator-server-port", "10011"],
+        "log_file": "replica1.log",
+        "setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"],
+    },
+    "instance_2": {
+        "args": ["--bolt-port", "7689", "--log-level", "TRACE", "--coordinator-server-port", "10012"],
+        "log_file": "replica2.log",
+        "setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"],
+    },
+    "instance_3": {
+        "args": ["--bolt-port", "7687", "--log-level", "TRACE", "--coordinator-server-port", "10013"],
+        "log_file": "main.log",
+        "setup_queries": [
+            "REGISTER REPLICA instance_1 SYNC TO '127.0.0.1:10001'",
+            "REGISTER REPLICA instance_2 SYNC TO '127.0.0.1:10002'",
+        ],
+    },
+    "coordinator": {
+        "args": ["--bolt-port", "7690", "--log-level=TRACE", "--coordinator"],
+        "log_file": "replica3.log",
+        "setup_queries": [
+            "REGISTER REPLICA instance_1 SYNC TO '127.0.0.1:10001' WITH COORDINATOR SERVER ON '127.0.0.1:10011';",
+            "REGISTER REPLICA instance_2 SYNC TO '127.0.0.1:10002' WITH COORDINATOR SERVER ON '127.0.0.1:10012';",
+            "REGISTER MAIN instance_3 WITH COORDINATOR SERVER ON '127.0.0.1:10013';",
+        ],
+    },
+}
+
+
+def test_show_replication_cluster(connection):
+    # Goal of this test is to check the SHOW REPLICATION CLUSTER command.
+    # 1. We start all replicas, main and coordinator manually: we want to be able to kill them ourselves without relying on external tooling to kill processes.
+    # 2. We check that all replicas and main have the correct state: they should all be alive.
+    # 3. We kill one replica. It should not appear anymore in the SHOW REPLICATION CLUSTER command.
+    # 4. We kill main. It should not appear anymore in the SHOW REPLICATION CLUSTER command.
+
+    # 1.
+    interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
+
+    cursor = connection(7690, "coordinator").cursor()
+
+    # 2.
+
+    # We leave some time for the coordinator to realise the replicas are down.
+    def retrieve_data():
+        return set(execute_and_fetch_all(cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data = {
+        ("instance_1", "127.0.0.1:10011", True, "replica"),
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_3", "127.0.0.1:10013", True, "main"),
+    }
+    mg_sleep_and_assert(expected_data, retrieve_data)
+
+    # 3.
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
+
+    expected_data = {
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_3", "127.0.0.1:10013", True, "main"),
+        ("instance_1", "127.0.0.1:10011", False, "replica"),
+    }
+    mg_sleep_and_assert(expected_data, retrieve_data)
+
+    # 4.
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
+
+    expected_data = {
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_1", "127.0.0.1:10011", False, "replica"),
+        ("instance_3", "127.0.0.1:10013", False, "main"),
+    }
+    mg_sleep_and_assert(expected_data, retrieve_data)
+
+
+def test_simple_client_initiated_failover(connection):
+    # 1. Start all instances
+    # 2. Kill main
+    # 3. Run DO FAILOVER on COORDINATOR
+    # 4. Assert new config on coordinator by running show replication cluster
+    # 5. Assert replicas on new main
+
+    # 1.
+    interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
+
+    # 2.
+    main_cursor = connection(7687, "instance_3").cursor()
+    expected_data_on_main = {
+        ("instance_1", "127.0.0.1:10001", "sync", 0, 0, "ready"),
+        ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
+    }
+    actual_data_on_main = set(execute_and_fetch_all(main_cursor, "SHOW REPLICAS;"))
+    assert actual_data_on_main == expected_data_on_main
+
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
+    coord_cursor = connection(7690, "coordinator").cursor()
+
+    def retrieve_data_show_repl_cluster():
+        return set(execute_and_fetch_all(coord_cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data_on_coord = {
+        ("instance_1", "127.0.0.1:10011", True, "replica"),
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_3", "127.0.0.1:10013", False, "main"),
+    }
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data_show_repl_cluster)
+
+    # 3.
+    execute_and_fetch_all(coord_cursor, "DO FAILOVER")
+
+    expected_data_on_coord = {
+        ("instance_1", "127.0.0.1:10011", True, "main"),
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+    }
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data_show_repl_cluster)
+
+    # 4.
+    new_main_cursor = connection(7688, "instance_1").cursor()
+
+    def retrieve_data_show_replicas():
+        return set(execute_and_fetch_all(new_main_cursor, "SHOW REPLICAS;"))
+
+    expected_data_on_new_main = {
+        ("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
+    }
+    mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas)
+
+
+def test_failover_fails_all_replicas_down(connection):
+    interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
+
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_2")
+    interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
+
+    coord_cursor = connection(7690, "coordinator").cursor()
+
+    def retrieve_data():
+        return set(execute_and_fetch_all(coord_cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data_on_coord = {
+        ("instance_1", "127.0.0.1:10011", False, "replica"),
+        ("instance_2", "127.0.0.1:10012", False, "replica"),
+        ("instance_3", "127.0.0.1:10013", False, "main"),
+    }
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data)
+
+    # 4.
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(coord_cursor, "DO FAILOVER;")
+    assert str(e.value) == "Failover aborted since all replicas are down!"
+
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data)
+
+
+def test_failover_fails_main_is_alive(connection):
+    interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
+
+    coord_cursor = connection(7690, "coordinator").cursor()
+
+    def retrieve_data():
+        return set(execute_and_fetch_all(coord_cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data_on_coord = {
+        ("instance_1", "127.0.0.1:10011", True, "replica"),
+        ("instance_2", "127.0.0.1:10012", True, "replica"),
+        ("instance_3", "127.0.0.1:10013", True, "main"),
+    }
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data)
+
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(coord_cursor, "DO FAILOVER;")
+    assert str(e.value) == "Failover aborted since main is alive!"
+
+    mg_sleep_and_assert(expected_data_on_coord, retrieve_data)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/high_availability_experimental/common.py

@@ -0,0 +1,25 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import typing
+
+import mgclient
+
+
+def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]:
+    cursor.execute(query, params)
+    return cursor.fetchall()
+
+
+def connect(**kwargs) -> mgclient.Connection:
+    connection = mgclient.connect(**kwargs)
+    connection.autocommit = True
+    return connection

tests/e2e/high_availability_experimental/conftest.py

@@ -0,0 +1,43 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import pytest
+from common import connect, execute_and_fetch_all
+
+
+# The fixture here is more complex because the connection has to be
+# parameterized based on the test parameters (info has to be available on both
+# sides).
+#
+# https://docs.pytest.org/en/latest/example/parametrize.html#indirect-parametrization
+# is not an elegant/feasible solution here.
+#
+# The solution was independently developed and then I stumbled upon the same
+# approach here https://stackoverflow.com/a/68286553/4888809 which I think is
+# optimal.
+@pytest.fixture(scope="function")
+def connection():
+    connection_holder = None
+    role_holder = None
+
+    def inner_connection(port, role):
+        nonlocal connection_holder, role_holder
+        connection_holder = connect(host="localhost", port=port)
+        role_holder = role
+        return connection_holder
+
+    yield inner_connection
+
+    # Only main instance can be cleaned up because replicas do NOT accept
+    # writes.
+    if role_holder == "main":
+        cursor = connection_holder.cursor()
+        execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n;")

tests/e2e/high_availability_experimental/coordinator.py

@@ -0,0 +1,98 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import execute_and_fetch_all
+from mg_utils import mg_sleep_and_assert
+
+
+def test_disable_cypher_queries(connection):
+    cursor = connection(7690, "coordinator").cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "CREATE (n:TestNode {prop: 'test'})")
+    assert str(e.value) == "Coordinator can run only coordinator queries!"
+
+
+def test_coordinator_cannot_be_replica_role(connection):
+    cursor = connection(7690, "coordinator").cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "SET REPLICATION ROLE TO REPLICA WITH PORT 10001;")
+    assert str(e.value) == "Coordinator can run only coordinator queries!"
+
+
+def test_coordinator_cannot_run_show_repl_role(connection):
+    cursor = connection(7690, "coordinator").cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "SHOW REPLICATION ROLE;")
+    assert str(e.value) == "Coordinator can run only coordinator queries!"
+
+
+def test_coordinator_show_replication_cluster(connection):
+    cursor = connection(7690, "coordinator").cursor()
+
+    def retrieve_data():
+        return set(execute_and_fetch_all(cursor, "SHOW REPLICATION CLUSTER;"))
+
+    expected_data = {
+        ("main", "127.0.0.1:10013", True, "main"),
+        ("replica_1", "127.0.0.1:10011", True, "replica"),
+        ("replica_2", "127.0.0.1:10012", True, "replica"),
+    }
+    mg_sleep_and_assert(expected_data, retrieve_data)
+
+
+def test_coordinator_cannot_call_show_replicas(connection):
+    cursor = connection(7690, "coordinator").cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "SHOW REPLICAS;")
+    assert str(e.value) == "Coordinator can run only coordinator queries!"
+
+
+@pytest.mark.parametrize(
+    "port, role",
+    [(7687, "main"), (7688, "replica"), (7689, "replica")],
+)
+def test_main_and_replicas_cannot_call_show_repl_cluster(port, role, connection):
+    cursor = connection(port, role).cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "SHOW REPLICATION CLUSTER;")
+    assert str(e.value) == "Only coordinator can run SHOW REPLICATION CLUSTER."
+
+
+@pytest.mark.parametrize(
+    "port, role",
+    [(7687, "main"), (7688, "replica"), (7689, "replica")],
+)
+def test_main_and_replicas_cannot_register_coord_server(port, role, connection):
+    cursor = connection(port, role).cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(
+            cursor,
+            "REGISTER REPLICA instance_1 SYNC TO '127.0.0.1:10001' WITH COORDINATOR SERVER ON '127.0.0.1:10011';",
+        )
+    assert str(e.value) == "Only coordinator can register coordinator server!"
+
+
+@pytest.mark.parametrize(
+    "port, role",
+    [(7687, "main"), (7688, "replica"), (7689, "replica")],
+)
+def test_main_and_replicas_cannot_run_do_failover(port, role, connection):
+    cursor = connection(port, role).cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "DO FAILOVER;")
+    assert str(e.value) == "Only coordinator can run DO FAILOVER!"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/high_availability_experimental/uninitialized_cluster.py

@@ -0,0 +1,26 @@
+# Copyright 2022 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import sys
+
+import pytest
+from common import execute_and_fetch_all
+
+
+def test_failover_on_non_setup_cluster(connection):
+    cursor = connection(7690, "coordinator").cursor()
+    with pytest.raises(Exception) as e:
+        execute_and_fetch_all(cursor, "DO FAILOVER;")
+    assert str(e.value) == "Failover aborted since cluster is uninitialized!"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-rA"]))

tests/e2e/high_availability_experimental/workloads.yaml

@@ -0,0 +1,67 @@
+ha_cluster: &ha_cluster
+  cluster:
+    replica_1:
+      args: ["--bolt-port", "7688", "--log-level=TRACE", "--coordinator-server-port=10011"]
+      log_file: "replication-e2e-replica1.log"
+      setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
+    replica_2:
+      args: ["--bolt-port", "7689", "--log-level=TRACE", "--coordinator-server-port=10012"]
+      log_file: "replication-e2e-replica2.log"
+      setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
+    main:
+      args: ["--bolt-port", "7687", "--log-level=TRACE", "--coordinator-server-port=10013"]
+      log_file: "replication-e2e-main.log"
+      setup_queries: [
+        "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001'",
+        "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002'",
+      ]
+    coordinator:
+      args: ["--bolt-port", "7690", "--log-level=TRACE", "--coordinator"]
+      log_file: "replication-e2e-coordinator.log"
+      setup_queries: [
+        "REGISTER MAIN main WITH COORDINATOR SERVER ON '127.0.0.1:10013'",
+        "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001' WITH COORDINATOR SERVER ON '127.0.0.1:10011'",
+        "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002' WITH COORDINATOR SERVER ON '127.0.0.1:10012'",
+      ]
+
+noninitialized_cluster: &noninitialized_cluster
+  cluster:
+    replica_1:
+      args: ["--bolt-port", "7688", "--log-level=TRACE", "--coordinator-server-port=10011"]
+      log_file: "replication-e2e-replica1.log"
+      setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
+    replica_2:
+      args: ["--bolt-port", "7689", "--log-level=TRACE", "--coordinator-server-port=10012"]
+      log_file: "replication-e2e-replica2.log"
+      setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
+    main:
+      args: ["--bolt-port", "7687", "--log-level=TRACE", "--coordinator-server-port=10013"]
+      log_file: "replication-e2e-main.log"
+      setup_queries: [
+        "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001'",
+        "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002'",
+      ]
+    coordinator:
+      args: ["--bolt-port", "7690", "--log-level=TRACE", "--coordinator"]
+      log_file: "replication-e2e-coordinator.log"
+      setup_queries: []
+
+
+workloads:
+  - name: "Coordinator"
+    binary: "tests/e2e/pytest_runner.sh"
+    args: ["high_availability_experimental/coordinator.py"]
+    <<: *ha_cluster
+
+  - name: "Uninitialized cluster"
+    binary: "tests/e2e/pytest_runner.sh"
+    args: ["high_availability_experimental/uninitialized_cluster.py"]
+    <<: *noninitialized_cluster
+
+  - name: "Client initiated failover"
+    binary: "tests/e2e/pytest_runner.sh"
+    args: ["high_availability_experimental/client_initiated_failover.py"]
+
+  - name: "Automatic failover"
+    binary: "tests/e2e/pytest_runner.sh"
+    args: ["high_availability_experimental/automatic_failover.py"]

tests/e2e/replication/CMakeLists.txt

@@ -9,11 +9,11 @@ target_link_libraries(memgraph__e2e__replication__indices gflags mgclient mg-uti
 add_executable(memgraph__e2e__replication__read_write_benchmark read_write_benchmark.cpp)
 target_link_libraries(memgraph__e2e__replication__read_write_benchmark gflags json mgclient mg-utils mg-io Threads::Threads)
 
-copy_e2e_python_files(replication_show common.py)
-copy_e2e_python_files(replication_show conftest.py)
-copy_e2e_python_files(replication_show show.py)
-copy_e2e_python_files(replication_show show_while_creating_invalid_state.py)
-copy_e2e_python_files(replication_show edge_delete.py)
-copy_e2e_python_files_from_parent_folder(replication_show ".." memgraph.py)
-copy_e2e_python_files_from_parent_folder(replication_show ".." interactive_mg_runner.py)
-copy_e2e_python_files_from_parent_folder(replication_show ".." mg_utils.py)
+copy_e2e_python_files(replication common.py)
+copy_e2e_python_files(replication conftest.py)
+copy_e2e_python_files(replication show.py)
+copy_e2e_python_files(replication show_while_creating_invalid_state.py)
+copy_e2e_python_files(replication edge_delete.py)
+copy_e2e_python_files_from_parent_folder(replication ".." memgraph.py)
+copy_e2e_python_files_from_parent_folder(replication ".." interactive_mg_runner.py)
+copy_e2e_python_files_from_parent_folder(replication ".." mg_utils.py)

tests/e2e/replication/workloads.yaml

@@ -11,7 +11,7 @@ template_validation_queries: &template_validation_queries
 template_simple_cluster: &template_simple_cluster
   cluster:
     replica_1:
-      args: [ "--bolt-port", "7688", "--log-level=TRACE" ]
+      args: [ "--bolt-port", "7688", "--log-level=TRACE"]
       log_file: "replication-e2e-replica1.log"
       setup_queries: [ "SET REPLICATION ROLE TO REPLICA WITH PORT 10001;" ]
     replica_2:
@@ -25,6 +25,7 @@ template_simple_cluster: &template_simple_cluster
         "REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:10001'",
         "REGISTER REPLICA replica_2 ASYNC TO '127.0.0.1:10002'",
       ]
+
 template_cluster: &template_cluster
   cluster:
     replica_1:
@@ -50,7 +51,6 @@ template_cluster: &template_cluster
         "REGISTER REPLICA replica_2 SYNC TO '127.0.0.1:10002'",
         "REGISTER REPLICA replica_3 ASYNC TO '127.0.0.1:10003'"
       ]
-      <<: *template_validation_queries
 
 workloads:
   - name: "Constraints"

tests/e2e/run.sh

@@ -25,7 +25,7 @@ if [ "$#" -eq 0 ]; then
   # NOTE: If you want to run all tests under specific folder/section just
   # replace the dot (root directory below) with the folder name, e.g.
   # `--workloads-root-directory replication`.
-  python3 runner.py --workloads-root-directory .
+  python3 runner.py --workloads-root-directory "$SCRIPT_DIR/../../build"
 elif [ "$#" -eq 1 ]; then
   if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
     print_help
@@ -34,7 +34,7 @@ elif [ "$#" -eq 1 ]; then
   # NOTE: --workload-name comes from each individual folder/section
   # workloads.yaml file. E.g. `streams/workloads.yaml` has a list of
   # `workloads:` and each workload has it's `-name`.
-  python3 runner.py --workloads-root-directory . --workload-name "$1"
+  python3 runner.py --workloads-root-directory "$SCRIPT_DIR/../../build" --workload-name "$1"
 else
   print_help
 fi

tests/stress/test_config.py

@@ -1,194 +0,0 @@
-import itertools
-import os
-from dataclasses import dataclass
-from typing import List
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-STATS_FILE = os.path.join(SCRIPT_DIR, ".long_running_stats")
-
-
-class DatasetConstants:
-    TEST = "test"
-    OPTIONS = "options"
-    TIMEOUT = "timeout"
-    MODE = "mode"
-    MEMGRAPH_OPTIONS = "memgraph_options"
-
-
-@dataclass
-class DatabaseMode:
-    storage_mode: str
-    isolation_level: str
-
-
-class StorageModeConstants:
-    IN_MEMORY_TRANSACTIONAL = "IN_MEMORY_TRANSACTIONAL"
-    IN_MEMORY_ANALYTICAL = "IN_MEMORY_ANALYTICAL"
-
-    @classmethod
-    def to_list(cls) -> List[str]:
-        return [cls.IN_MEMORY_TRANSACTIONAL, cls.IN_MEMORY_ANALYTICAL]
-
-
-class IsolationLevelConstants:
-    SNAPSHOT_ISOLATION = "SNAPSHOT ISOLATION"
-    READ_COMMITED = "READ COMMITED"
-    READ_UNCOMMITED = "READ UNCOMMITED"
-
-    @classmethod
-    def to_list(cls) -> List[str]:
-        return [cls.SNAPSHOT_SERIALIZATION, cls.READ_COMMITED, cls.READ_UNCOMMITED]
-
-
-def get_default_database_mode() -> DatabaseMode:
-    return DatabaseMode(StorageModeConstants.IN_MEMORY_TRANSACTIONAL, IsolationLevelConstants.SNAPSHOT_ISOLATION)
-
-
-def get_all_database_modes() -> List[DatabaseMode]:
-    return [
-        DatabaseMode(x[0], x[1])
-        for x in itertools.product(StorageModeConstants.to_list(), IsolationLevelConstants.to_list())
-    ]
-
-
-# dataset calibrated for running on Apollo (total 4min)
-# bipartite.py runs for approx. 30s
-# create_match.py runs for approx. 30s
-# long_running runs for 1min
-# long_running runs for 2min
-SMALL_DATASET = [
-    {
-        DatasetConstants.TEST: "bipartite.py",
-        DatasetConstants.OPTIONS: ["--u-count", "100", "--v-count", "100"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-    {
-        DatasetConstants.TEST: "detach_delete.py",
-        DatasetConstants.OPTIONS: ["--worker-count", "4", "--repetition-count", "100"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-    {
-        DatasetConstants.TEST: "memory_tracker.py",
-        DatasetConstants.OPTIONS: ["--worker-count", "5", "--repetition-count", "100"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-        DatasetConstants.MEMGRAPH_OPTIONS: ["--memory-limit=2048"],
-    },
-    {
-        DatasetConstants.TEST: "memory_limit.py",
-        DatasetConstants.OPTIONS: ["--worker-count", "5", "--repetition-count", "100"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-        DatasetConstants.MEMGRAPH_OPTIONS: ["--memory-limit=2048"],
-    },
-    {
-        DatasetConstants.TEST: "create_match.py",
-        DatasetConstants.OPTIONS: ["--vertex-count", "40000", "--create-pack-size", "100"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-    {
-        DatasetConstants.TEST: "parser.cpp",
-        DatasetConstants.OPTIONS: ["--per-worker-query-count", "1000"],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-    {
-        DatasetConstants.TEST: "long_running.cpp",
-        DatasetConstants.OPTIONS: [
-            "--vertex-count",
-            "1000",
-            "--edge-count",
-            "5000",
-            "--max-time",
-            "1",
-            "--verify",
-            "20",
-        ],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-    {
-        DatasetConstants.TEST: "long_running.cpp",
-        DatasetConstants.OPTIONS: [
-            "--vertex-count",
-            "10000",
-            "--edge-count",
-            "50000",
-            "--max-time",
-            "2",
-            "--verify",
-            "30",
-            "--stats-file",
-            STATS_FILE,
-        ],
-        DatasetConstants.TIMEOUT: 5,
-        DatasetConstants.MODE: [get_default_database_mode()],
-    },
-]
-
-# dataset calibrated for running on daily stress instance (total 9h)
-# bipartite.py and create_match.py run for approx. 15min
-# long_running runs for 5min x 6 times = 30min
-# long_running runs for 8h
-LARGE_DATASET = (
-    [
-        {
-            DatasetConstants.TEST: "bipartite.py",
-            DatasetConstants.OPTIONS: ["--u-count", "300", "--v-count", "300"],
-            DatasetConstants.TIMEOUT: 30,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-        {
-            DatasetConstants.TEST: "detach_delete.py",
-            DatasetConstants.OPTIONS: ["--worker-count", "4", "--repetition-count", "300"],
-            DatasetConstants.TIMEOUT: 5,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-        {
-            DatasetConstants.TEST: "create_match.py",
-            DatasetConstants.OPTIONS: ["--vertex-count", "500000", "--create-pack-size", "500"],
-            DatasetConstants.TIMEOUT: 30,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
-    + [
-        {
-            DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.OPTIONS: [
-                "--vertex-count",
-                "10000",
-                "--edge-count",
-                "40000",
-                "--max-time",
-                "5",
-                "--verify",
-                "60",
-            ],
-            DatasetConstants.TIMEOUT: 16,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
-    * 6
-    + [
-        {
-            DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.OPTIONS: [
-                "--vertex-count",
-                "200000",
-                "--edge-count",
-                "1000000",
-                "--max-time",
-                "480",
-                "--verify",
-                "300",
-                "--stats-file",
-                STATS_FILE,
-            ],
-            DatasetConstants.TIMEOUT: 500,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
-)