PoC for auth replication

Co-authored-by: Antonio Andelic <antonio.andelic@memgraph.io>

.github/workflows/diff.yaml

@@ -288,3 +288,43 @@ jobs:
           cd tests/stress
           source ve3/bin/activate
           python3 durability --num-steps 5
+
+  release_jepsen_test:
+    name: "Release Jepsen Test"
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
+    #continue-on-error: true
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS memgraph
+
+      - name: Run Jepsen tests
+        run: |
+          cd tests/jepsen
+          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
+
+      - name: Save Jepsen report
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: "Jepsen Report"
+          path: tests/jepsen/Jepsen.tar.gz

.github/workflows/test_all_workers.yaml

@@ -36,6 +36,8 @@ jobs:
         run: |
           source /opt/toolchain-v2/activate
           ./tools/check-build-system
+          docker --version
+          docker ps | grep jepsen
 
   HP-DL360G6-v2-1:
     name: "HP-DL360G6-v2-1"

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Change Log
 
+## Future
+
+### Breaking Changes
+
+* Added extra information in durability files to support replication making
+  it incompatible with the durability files generated by older versions of
+  Memgraph. Even though the replication is an Enterprise feature, the files
+  are compatible with the Community version.
+
 ## v1.2.0
 
 ### Breaking Changes

docs/feature_spec/replication.md

@@ -0,0 +1,202 @@
+# Replication
+
+## High Level Context
+
+Replication is a method that ensures that multiple database instances are
+storing the same data. To enable replication, there must be at least two
+instances of Memgraph in a cluster. Each instance has one of either two roles:
+main or replica. The main instance is the instance that accepts writes to the
+database and replicates its state to the replicas. In a cluster, there can only
+be one main. There can be one or more replicas. None of the replicas will accept
+write queries, but they will always accept read queries (there is an exception
+to this rule and is described below). Replicas can also be configured to be
+replicas of replicas, not necessarily replicas of the main. Each instance will
+always be reachable using the standard supported communication protocols. The
+replication will replicate WAL data. All data is transported through a custom
+binary protocol that will try remain backward compatible, so that replication
+immediately allows for zero downtime upgrades.
+
+Each replica can be configured to accept replicated data in one of the following
+modes:
+ - synchronous
+ - asynchronous
+ - semi-synchronous
+
+### Synchronous Replication
+
+When the data is replicated to a replica synchronously, all of the data of a
+currently pending transaction must be sent to the synchronous replica before the
+transaction is able to commit its changes.
+
+This mode has a positive implication that all data that is committed to the
+main will always be replicated to the synchronous replica. It also has a
+negative performance implication because non-responsive replicas could grind all
+query execution to a halt.
+
+This mode is good when you absolutely need to be sure that all data is always
+consistent between the main and the replica.
+
+### Asynchronous Replication
+
+When the data is replicated to a replica asynchronously, all pending
+transactions are immediately committed and their data is replicated to the
+asynchronous replica in the background.
+
+This mode has a positive performance implication in which it won't slow down
+query execution. It also has a negative implication that the data between the
+main and the replica is almost never in a consistent state (when the data is
+being changed).
+
+This mode is good when you don't care about consistency and only need an
+eventually consistent cluster, but you care about performance.
+
+### Semi-synchronous Replication
+
+When the data is replicated to a replica semi-synchronously, the data is
+replicated using both the synchronous and asynchronous methodology. The data is
+always replicated synchronously, but, if the replica for any reason doesn't
+respond within a preset timeout, the pending transaction is committed and the
+data is replicated to the replica asynchronously.
+
+This mode has a positive implication that all data that is committed is
+*mostly* replicated to the semi-synchronous replica. It also has a negative
+performance implication as the synchronous replication mode.
+
+This mode is useful when you want the replication to be synchronous to ensure
+that the data within the cluster is consistent, but you don't want the main
+to grind to a halt when you have a non-responsive replica.
+
+### Addition of a New Replica
+
+Each replica, when added to the cluster (in any mode), will first start out as
+an asynchronous replica. That will allow replicas that have fallen behind to
+first catch-up to the current state of the database. When the replica is in a
+state that it isn't lagging behind the main it will then be promoted (in a brief
+stop-the-world operation) to a semi-synchronous or synchronous replica. Slaves
+that are added as asynchronous replicas will remain asynchronous.
+
+## User Facing Setup
+
+### How to Setup a Memgraph Cluster with Replication?
+
+Replication configuration is done primarily through openCypher commands. This
+allows the cluster to be dynamically rearranged (new leader election, addition
+of a new replica, etc.).
+
+Each Memgraph instance when first started will be a main. You have to change
+the role of all replica nodes using the following openCypher query before you
+can enable replication on the main:
+
+```plaintext
+SET REPLICATION ROLE TO (MAIN|REPLICA) WITH PORT <port_number>;
+```
+
+Note that the "WITH PORT <port_number>" part of the query sets the replication port,
+but it applies only to the replica. In other words, if you try to set the
+replication port as the main, a semantic exception will be thrown.
+After you have set your replica instance to the correct operating role, you can
+enable replication in the main instance by issuing the following openCypher
+command:
+```plaintext
+REGISTER REPLICA name (SYNC|ASYNC) [WITH TIMEOUT 0.5] TO <socket_address>;
+```
+
+The socket address must be a string of the following form:
+
+```plaintext
+"IP_ADDRESS:PORT_NUMBER"
+```
+
+where IP_ADDRESS is a valid IP address, and PORT_NUMBER is a valid port number,
+both given in decimal notation.
+Note that in this case they must be separated by a single colon.
+Alternatively, one can give the socket address as:
+
+```plaintext
+"IP_ADDRESS"
+```
+
+where IP_ADDRESS must be a valid IP address, and the port number will be
+assumed to be the default one (we specify it to be 10000).
+
+Each Memgraph instance will remember what the configuration was set to and will
+automatically resume with its role when restarted.
+
+### How to Setup an Advanced Replication Scenario?
+
+The configuration allows for a more advanced scenario like this:
+```plaintext
+main -[asynchronous]-> replica 1 -[semi-synchronous]-> replica 2
+```
+
+To configure the above scenario, issue the following commands:
+```plaintext
+SET REPLICATION ROLE TO REPLICA;  # on replica 1
+SET REPLICATION ROLE TO REPLICA;  # on replica 2
+
+REGISTER REPLICA replica1 ASYNC TO <replica1_sa>;  # on main
+REGISTER REPLICA replica2 SYNC WITH TIMEOUT 0.5 TO <replica2_sa>;  # on replica 1
+```
+
+### How to See the Current Replication Status?
+
+To see the replication ROLE of the current Memgraph instance, you can issue the
+following query:
+
+```plaintext
+SHOW REPLICATION ROLE;
+```
+
+To see the replicas of the current Memgraph instance, you can issue the
+following query:
+
+```plaintext
+SHOW REPLICAS;
+```
+
+To delete a replica, issue the following query:
+
+```plaintext
+DELETE REPLICA 'name';
+```
+
+### How to Promote a New Main?
+
+When you have an already set-up cluster, to promote a new main, just set the
+replica that you want to be a main to the main role.
+
+```plaintext
+SET REPLICATION ROLE TO MAIN;  # on desired replica
+```
+
+After the command is issued, if the original main is still alive, it won't be
+able to replicate its data to the replica (the new main) anymore and will enter
+an error state. You must ensure that at any given point in time there aren't
+two mains in the cluster.
+
+## Integration with Memgraph
+
+WAL `Delta`s are replicated between the replication main and replica. With
+`Delta`s, all `StorageGlobalOperation`s are also replicated. Replication is
+essentially the same as appending to the WAL.
+
+Synchronous replication will occur in `Commit` and each
+`StorageGlobalOperation` handler. The storage itself guarantees that `Commit`
+will be called single-threadedly and that no `StorageGlobalOperation` will be
+executed during an active transaction. Asynchronous replication will load its
+data from already written WAL files and transmit the data to the replica. All
+data will be replicated using our RPC protocol (SLK encoded).
+
+For each replica the replication main (or replica) will keep track of the
+replica's state. That way, it will know which operations must be transmitted to
+the replica and which operations can be skipped. When a replica is very stale,
+a snapshot will be transmitted to it so that it can quickly synchronize with
+the current state. All following operations will transmit WAL deltas.
+
+## Reading materials
+
+1. [PostgreSQL comparison of different solutions](https://www.postgresql.org/docs/12/different-replication-solutions.html)
+2. [PostgreSQL docs](https://www.postgresql.org/docs/12/runtime-config-replication.html)
+3. [MySQL reference manual](https://dev.mysql.com/doc/refman/8.0/en/replication.html)
+4. [MySQL docs](https://dev.mysql.com/doc/refman/8.0/en/replication-setup-slaves.html)
+5. [MySQL master switch](https://dev.mysql.com/doc/refman/8.0/en/replication-solutions-switch.html)

src/auth/auth.cpp

@@ -9,6 +9,7 @@
 #include <glog/logging.h>
 
 #include "auth/exceptions.hpp"
+#include "io/network/endpoint.hpp"
 #include "utils/flag_validation.hpp"
 #include "utils/string.hpp"
 
@@ -182,6 +183,7 @@ void Auth::SaveUser(const User &user) {
   if (!success) {
     throw AuthException("Couldn't save user '{}'!", user.username());
   }
+  storage_.Replicate();
 }
 
 std::optional<User> Auth::AddUser(const std::string &username,
@@ -204,6 +206,7 @@ bool Auth::RemoveUser(const std::string &username_orig) {
   if (!storage_.DeleteMultiple(keys)) {
     throw AuthException("Couldn't remove user '{}'!", username);
   }
+  storage_.Replicate();
   return true;
 }
 
@@ -218,6 +221,7 @@ std::vector<auth::User> Auth::AllUsers() {
       ret.push_back(*user);
     }
   }
+
   return ret;
 }
 
@@ -244,6 +248,7 @@ void Auth::SaveRole(const Role &role) {
   if (!storage_.Put(kRolePrefix + role.rolename(), role.Serialize().dump())) {
     throw AuthException("Couldn't save role '{}'!", role.rolename());
   }
+  storage_.Replicate();
 }
 
 std::optional<Role> Auth::AddRole(const std::string &rolename) {
@@ -270,6 +275,7 @@ bool Auth::RemoveRole(const std::string &rolename_orig) {
   if (!storage_.DeleteMultiple(keys)) {
     throw AuthException("Couldn't remove role '{}'!", rolename);
   }
+  storage_.Replicate();
   return true;
 }
 

src/auth/auth.hpp

@@ -7,8 +7,12 @@
 #include "auth/exceptions.hpp"
 #include "auth/models.hpp"
 #include "auth/module.hpp"
+#include "communication/context.hpp"
 #include "kvstore/kvstore.hpp"
 
+#include "rpc/client.hpp"
+#include "rpc/server.hpp"
+
 namespace auth {
 
 /**

src/auth/models.cpp

@@ -40,6 +40,8 @@ std::string PermissionToString(Permission permission) {
       return "CONSTRAINT";
     case Permission::DUMP:
       return "DUMP";
+    case Permission::REPLICATION:
+      return "REPLICATION";
     case Permission::AUTH:
       return "AUTH";
   }

src/auth/models.hpp

@@ -20,6 +20,7 @@ enum class Permission : uint64_t {
   STATS = 0x00000080,
   CONSTRAINT = 0x00000100,
   DUMP = 0x00000200,
+  REPLICATION = 0x00000400,
   AUTH = 0x00010000,
 };
 
@@ -28,7 +29,7 @@ const std::vector<Permission> kPermissionsAll = {
     Permission::MATCH,  Permission::CREATE, Permission::MERGE,
     Permission::DELETE, Permission::SET,    Permission::REMOVE,
     Permission::INDEX,  Permission::STATS,  Permission::CONSTRAINT,
-    Permission::DUMP,   Permission::AUTH};
+    Permission::DUMP,   Permission::AUTH,   Permission::REPLICATION};
 
 // Function that converts a permission to its string representation.
 std::string PermissionToString(Permission permission);

src/glue/auth.cpp

@@ -24,6 +24,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
       return auth::Permission::CONSTRAINT;
     case query::AuthQuery::Privilege::DUMP:
       return auth::Permission::DUMP;
+    case query::AuthQuery::Privilege::REPLICATION:
+      return auth::Permission::REPLICATION;
     case query::AuthQuery::Privilege::AUTH:
       return auth::Permission::AUTH;
   }

src/io/network/endpoint.cpp

@@ -7,34 +7,89 @@
 #include "glog/logging.h"
 
 #include "io/network/endpoint.hpp"
+#include "utils/string.hpp"
 
 namespace io::network {
 
-Endpoint::Endpoint() {}
-Endpoint::Endpoint(const std::string &address, uint16_t port)
-    : address_(address), port_(port) {
+Endpoint::IpFamily Endpoint::GetIpFamily(const std::string &ip_address) {
   in_addr addr4;
   in6_addr addr6;
-  int ipv4_result = inet_pton(AF_INET, address_.c_str(), &addr4);
-  int ipv6_result = inet_pton(AF_INET6, address_.c_str(), &addr6);
-  if (ipv4_result == 1)
-    family_ = 4;
-  else if (ipv6_result == 1)
-    family_ = 6;
-  CHECK(family_ != 0) << "Not a valid IPv4 or IPv6 address: " << address;
+  int ipv4_result = inet_pton(AF_INET, ip_address.c_str(), &addr4);
+  int ipv6_result = inet_pton(AF_INET6, ip_address.c_str(), &addr6);
+  if (ipv4_result == 1) {
+    return IpFamily::IP4;
+  } else if (ipv6_result == 1) {
+    return IpFamily::IP6;
+  } else {
+    return IpFamily::NONE;
+  }
 }
 
-bool Endpoint::operator==(const Endpoint &other) const {
-  return address_ == other.address_ && port_ == other.port_ &&
-         family_ == other.family_;
+std::optional<std::pair<std::string, uint16_t>>
+Endpoint::ParseSocketOrIpAddress(
+    const std::string &address,
+    const std::optional<uint16_t> default_port = {}) {
+  /// expected address format:
+  ///   - "ip_address:port_number"
+  ///   - "ip_address"
+  /// We parse the address first. If it's an IP address, a default port must
+  // be given, or we return nullopt. If it's a socket address, we try to parse
+  // it into an ip address and a port number; even if a default port is given,
+  // it won't be used, as we expect that it is given in the address string.
+  const std::string delimiter = ":";
+  std::string ip_address;
+
+  std::vector<std::string> parts = utils::Split(address, delimiter);
+  if (parts.size() == 1) {
+    if (default_port) {
+      return std::pair{address, *default_port};
+    }
+  } else if (parts.size() == 2) {
+    ip_address = std::move(parts[0]);
+    int64_t int_port{0};
+    try {
+      int_port = utils::ParseInt(parts[1]);
+    } catch (utils::BasicException &e) {
+      LOG(ERROR) << "Invalid port number: " << parts[1];
+      return std::nullopt;
+    }
+    if (int_port < 0) {
+      LOG(ERROR) << "Port number must be a positive integer!";
+      return std::nullopt;
+    }
+    if (int_port > std::numeric_limits<uint16_t>::max()) {
+      LOG(ERROR) << "Port number exceeded maximum possible size!";
+      return std::nullopt;
+    }
+
+    return std::pair{ip_address, static_cast<uint16_t>(int_port)};
+  }
+
+  return std::nullopt;
+}
+
+std::string Endpoint::SocketAddress() const {
+  auto ip_address = address.empty() ? "EMPTY" : address;
+  return ip_address + ":" + std::to_string(port);
+}
+
+Endpoint::Endpoint() {}
+Endpoint::Endpoint(std::string ip_address, uint16_t port)
+    : address(std::move(ip_address)), port(port) {
+  IpFamily ip_family = GetIpFamily(address);
+  CHECK(ip_family != IpFamily::NONE)
+      << "Not a valid IPv4 or IPv6 address: " << ip_address;
+  family = ip_family;
 }
 
 std::ostream &operator<<(std::ostream &os, const Endpoint &endpoint) {
-  if (endpoint.family() == 6) {
-    return os << "[" << endpoint.address() << "]"
-              << ":" << endpoint.port();
+  // no need to cover the IpFamily::NONE case, as you can't even construct an
+  // Endpoint object if the IpFamily is NONE (i.e. the IP address is invalid)
+  if (endpoint.family == Endpoint::IpFamily::IP6) {
+    return os << "[" << endpoint.address << "]"
+              << ":" << endpoint.port;
   }
-  return os << endpoint.address() << ":" << endpoint.port();
+  return os << endpoint.address << ":" << endpoint.port;
 }
 
 }  // namespace io::network

src/io/network/endpoint.hpp

@@ -3,6 +3,7 @@
 #include <netinet/in.h>
 #include <cstdint>
 #include <iostream>
+#include <optional>
 #include <string>
 
 namespace io::network {
@@ -12,22 +13,35 @@ namespace io::network {
  * It is used when connecting to an address and to get the current
  * connection address.
  */
-class Endpoint {
- public:
+struct Endpoint {
   Endpoint();
-  Endpoint(const std::string &address, uint16_t port);
+  Endpoint(std::string ip_address, uint16_t port);
 
-  // TODO: Remove these since members are public
-  std::string address() const { return address_; }
-  uint16_t port() const { return port_; }
-  unsigned char family() const { return family_; }
+  enum class IpFamily : std::uint8_t { NONE, IP4, IP6 };
 
-  bool operator==(const Endpoint &other) const;
+  std::string SocketAddress() const;
+
+  bool operator==(const Endpoint &other) const = default;
   friend std::ostream &operator<<(std::ostream &os, const Endpoint &endpoint);
 
-  std::string address_;
-  uint16_t port_{0};
-  unsigned char family_{0};
+  std::string address;
+  uint16_t port{0};
+  IpFamily family{IpFamily::NONE};
+
+  /**
+   * Tries to parse the given string as either a socket address or ip address.
+   * Expected address format:
+   *   - "ip_address:port_number"
+   *   - "ip_address"
+   * We parse the address first. If it's an IP address, a default port must
+   * be given, or we return nullopt. If it's a socket address, we try to parse
+   * it into an ip address and a port number; even if a default port is given,
+   * it won't be used, as we expect that it is given in the address string.
+   */
+  static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrIpAddress(
+      const std::string &address, const std::optional<uint16_t> default_port);
+
+  static IpFamily GetIpFamily(const std::string &ip_address);
 };
 
 }  // namespace io::network

src/io/network/socket.cpp

@@ -60,8 +60,8 @@ bool Socket::IsOpen() const { return socket_ != -1; }
 bool Socket::Connect(const Endpoint &endpoint) {
   if (socket_ != -1) return false;
 
-  auto info = AddrInfo::Get(endpoint.address().c_str(),
-                            std::to_string(endpoint.port()).c_str());
+  auto info = AddrInfo::Get(endpoint.address.c_str(),
+                            std::to_string(endpoint.port).c_str());
 
   for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
     int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
@@ -84,8 +84,8 @@ bool Socket::Connect(const Endpoint &endpoint) {
 bool Socket::Bind(const Endpoint &endpoint) {
   if (socket_ != -1) return false;
 
-  auto info = AddrInfo::Get(endpoint.address().c_str(),
-                            std::to_string(endpoint.port()).c_str());
+  auto info = AddrInfo::Get(endpoint.address.c_str(),
+                            std::to_string(endpoint.port).c_str());
 
   for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
     int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
@@ -122,7 +122,7 @@ bool Socket::Bind(const Endpoint &endpoint) {
     return false;
   }
 
-  endpoint_ = Endpoint(endpoint.address(), ntohs(portdata.sin6_port));
+  endpoint_ = Endpoint(endpoint.address, ntohs(portdata.sin6_port));
 
   return true;
 }

src/kvstore/CMakeLists.txt

@@ -1,7 +1,25 @@
+set(kvstore_src_files kvstore.cpp)
+if(MG_ENTERPRISE)
+    define_add_lcp(add_lcp_kvstore lcp_kvstore_cpp_files generated_lcp_kvstore_files)
+
+    add_lcp_kvstore(replication/rpc.lcp SLK_SERIALIZE)
+
+    add_custom_target(generate_lcp_kvstore DEPENDS ${generated_lcp_kvstore_files})
+
+    set(kvstore_src_files
+        ${kvstore_src_files}
+        ${lcp_kvstore_cpp_files})
+endif()
+
 # STATIC library used to store key-value pairs
-add_library(mg-kvstore STATIC kvstore.cpp)
+add_library(mg-kvstore STATIC ${kvstore_src_files})
 target_link_libraries(mg-kvstore stdc++fs mg-utils rocksdb bzip2 zlib glog gflags)
 
+if(MG_ENTERPRISE)
+    add_dependencies(mg-kvstore generate_lcp_kvstore)
+    target_link_libraries(mg-kvstore mg-rpc mg-slk)
+endif()
+
 # STATIC library for dummy key-value storage
 # add_library(mg-kvstore-dummy STATIC kvstore_dummy.cpp)
 # target_link_libraries(mg-kvstore-dummy mg-utils)

src/kvstore/kvstore.cpp

@@ -1,11 +1,16 @@
 #include <rocksdb/db.h>
 #include <rocksdb/options.h>
+#include <rocksdb/write_batch.h>
 
 #include "kvstore/kvstore.hpp"
+#include "kvstore/replication/rpc.hpp"
 #include "utils/file.hpp"
 
 namespace kvstore {
 
+DEFINE_bool(main, false, "Set to true to be the main");
+DEFINE_bool(replica, false, "Set to true to be the replica");
+
 struct KVStore::impl {
   std::filesystem::path storage;
   std::unique_ptr<rocksdb::DB> db;
@@ -14,6 +19,37 @@ struct KVStore::impl {
 
 KVStore::KVStore(std::filesystem::path storage)
     : pimpl_(std::make_unique<impl>()) {
+  if (FLAGS_main) {
+    DLOG(INFO) << "SETTING CLIENT FOR AUTH";
+    rpc_context_.emplace();
+    rpc_client_.emplace(io::network::Endpoint("127.0.0.1", 10000),
+                        &*rpc_context_);
+  } else if (FLAGS_replica) {
+    rpc_server_context_.emplace();
+    rpc_server_.emplace(io::network::Endpoint("127.0.0.1", 10000),
+                        &*rpc_server_context_);
+    rpc_server_->Register<AppendKvstoreRpc>([this](auto *req_reader,
+                                                   auto *res_builder) {
+      AppendKvstoreReq req;
+      slk::Load(&req, req_reader);
+      DLOG(INFO) << "Received AppendKvstoreRpc";
+      size_t updates_num;
+      slk::Load(&updates_num, req_reader);
+      for (size_t i = 0; i < updates_num; ++i) {
+        size_t update_size;
+        slk::Load(&update_size, req_reader);
+        std::vector<uint8_t> update_data(update_size);
+        req_reader->Load(update_data.data(), update_size);
+        rocksdb::WriteBatch write_batch(std::string(
+            reinterpret_cast<const char *>(update_data.data()), update_size));
+        pimpl_->db->Write(rocksdb::WriteOptions(), &write_batch);
+      }
+      const auto next_sequence_num = pimpl_->db->GetLatestSequenceNumber();
+      AppendKvstoreRes res{true, next_sequence_num};
+      slk::Save(res, res_builder);
+    });
+    rpc_server_->Start();
+  }
   pimpl_->storage = storage;
   if (!utils::EnsureDir(pimpl_->storage))
     throw KVStoreError("Folder for the key-value store " +
@@ -27,7 +63,12 @@ KVStore::KVStore(std::filesystem::path storage)
   pimpl_->db.reset(db);
 }
 
-KVStore::~KVStore() {}
+KVStore::~KVStore() {
+  if (rpc_server_) {
+    rpc_server_->Shutdown();
+    rpc_server_->AwaitShutdown();
+  }
+}
 
 KVStore::KVStore(KVStore &&other) { pimpl_ = std::move(other.pimpl_); }
 
@@ -96,6 +137,34 @@ bool KVStore::PutAndDeleteMultiple(
   return s.ok();
 }
 
+void KVStore::Replicate() {
+  auto stream = rpc_client_->Stream<AppendKvstoreRpc>();
+  auto *builder = stream.GetBuilder();
+
+  std::vector<std::vector<std::uint8_t>> updates;
+  std::unique_ptr<rocksdb::TransactionLogIterator> iter;
+  auto status = pimpl_->db->GetUpdatesSince(next_sequence_num_, &iter);
+  if (status.ok()) {
+    for (; iter && iter->Valid(); iter->Next()) {
+      auto result = iter->GetBatch();
+      const auto &str = result.writeBatchPtr->Data();
+      std::vector<std::uint8_t> raw_data;
+      raw_data.resize(str.size());
+      memcpy(raw_data.data(), str.data(), str.size());
+      updates.emplace_back(std::move(raw_data));
+    }
+  }
+
+  slk::Save(updates.size(), builder);
+  for (const auto &update : updates) {
+    slk::Save(update.size(), builder);
+    builder->Save(update.data(), update.size());
+  }
+
+  const auto response = stream.AwaitResponse();
+  next_sequence_num_ = response.next_sequence_num;
+};
+
 // iterator
 
 struct KVStore::iterator::impl {

src/kvstore/kvstore.hpp

@@ -7,6 +7,8 @@
 #include <string>
 #include <vector>
 
+#include "rpc/client.hpp"
+#include "rpc/server.hpp"
 #include "utils/exceptions.hpp"
 
 namespace kvstore {
@@ -117,6 +119,8 @@ class KVStore final {
   bool PutAndDeleteMultiple(const std::map<std::string, std::string> &items,
                             const std::vector<std::string> &keys);
 
+  void Replicate();
+
   /**
    * Returns total number of stored (key, value) pairs. The function takes an
    * optional prefix parameter used for filtering keys that start with that
@@ -202,6 +206,14 @@ class KVStore final {
  private:
   struct impl;
   std::unique_ptr<impl> pimpl_;
+
+  // RocksDB WAL sequence number always starts with 0 for a newly created DB
+  std::uint64_t next_sequence_num_ = 0;
+  std::optional<communication::ClientContext> rpc_context_;
+  std::optional<rpc::Client> rpc_client_;
+
+  std::optional<communication::ServerContext> rpc_server_context_;
+  std::optional<rpc::Server> rpc_server_;
 };
 
 }  // namespace kvstore

src/kvstore/replication/.gitignore

@@ -0,0 +1,2 @@
+# autogenerated files
+rpc.hpp

src/kvstore/replication/rpc.lcp

@@ -0,0 +1,18 @@
+#>cpp
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+#include "rpc/messages.hpp"
+#include "slk/serialization.hpp"
+#include "slk/streams.hpp"
+cpp<#
+(lcp:namespace kvstore)
+
+(lcp:define-rpc append-kvstore
+  (:request ())
+  (:response
+    ((success :bool)
+     (next-sequence-num :uint64_t))))
+(lcp:pop-namespace) ;; storage

src/memgraph.cpp

@@ -228,8 +228,8 @@ class BoltSession final
     for (const auto &kv : params)
       params_pv.emplace(kv.first, glue::ToPropertyValue(kv.second));
 #ifdef MG_ENTERPRISE
-    audit_log_->Record(endpoint_.address(), user_ ? user_->username() : "",
-                       query, storage::PropertyValue(params_pv));
+    audit_log_->Record(endpoint_.address, user_ ? user_->username() : "", query,
+                       storage::PropertyValue(params_pv));
 #endif
     try {
       auto result = interpreter_.Prepare(query, params_pv);

src/query/constants.hpp

@@ -0,0 +1,8 @@
+#pragma once
+#include <cstdint>
+#include <string>
+
+namespace query {
+constexpr uint16_t kDefaultReplicationPort = 10000;
+constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
+}  // namespace query

src/query/exceptions.hpp

@@ -167,4 +167,11 @@ class InvalidArgumentsException : public QueryException {
                                    argument_name, message)) {}
 };
 
+class ReplicationModificationInMulticommandTxException : public QueryException {
+ public:
+  ReplicationModificationInMulticommandTxException()
+      : QueryException(
+            "Replication clause not allowed in multicommand transactions.") {}
+};
+
 }  // namespace query

src/query/frontend/ast/ast.lcp

@@ -2191,7 +2191,7 @@ cpp<#
       (:serialize))
     (lcp:define-enum privilege
         (create delete match merge set remove index stats auth constraint
-         dump)
+         dump replication)
       (:serialize))
     #>cpp
     AuthQuery() = default;
@@ -2226,7 +2226,8 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
     AuthQuery::Privilege::SET,        AuthQuery::Privilege::REMOVE,
     AuthQuery::Privilege::INDEX,      AuthQuery::Privilege::STATS,
     AuthQuery::Privilege::AUTH,
-    AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP};
+    AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP,
+    AuthQuery::Privilege::REPLICATION};
 cpp<#
 
 (lcp:define-class info-query (query)
@@ -2296,4 +2297,40 @@ cpp<#
   (:serialize (:slk))
   (:clone))
 
+(lcp:define-class replication-query (query)
+  ((action "Action" :scope :public)
+   (role "ReplicationRole" :scope :public)
+   (replica_name "std::string" :scope :public)
+   (socket_address "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression"))
+   (port "Expression *" :initval "nullptr" :scope :public)
+   (sync_mode "SyncMode" :scope :public)
+   (timeout "Expression *" :initval "nullptr" :scope :public
+             :slk-save #'slk-save-ast-pointer
+             :slk-load (slk-load-ast-pointer "Expression")))
+
+  (:public
+    (lcp:define-enum action
+        (set-replication-role show-replication-role register-replica
+         drop-replica show-replicas)
+      (:serialize))
+    (lcp:define-enum replication-role
+        (main replica)
+      (:serialize))
+    (lcp:define-enum sync-mode
+        (sync async)
+      (:serialize))
+    #>cpp
+    ReplicationQuery() = default;
+
+    DEFVISITABLE(QueryVisitor<void>);
+    cpp<#)
+  (:private
+    #>cpp
+    friend class AstStorage;
+    cpp<#)
+  (:serialize (:slk))
+  (:clone))
+
 (lcp:pop-namespace) ;; namespace query

src/query/frontend/ast/ast_visitor.hpp

@@ -72,6 +72,7 @@ class InfoQuery;
 class ConstraintQuery;
 class RegexMatch;
 class DumpQuery;
+class ReplicationQuery;
 
 using TreeCompositeVisitor = ::utils::CompositeVisitor<
     SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator,
@@ -115,7 +116,7 @@ class ExpressionVisitor
 template <class TResult>
 class QueryVisitor
     : public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery,
-                              IndexQuery, AuthQuery, InfoQuery,
-                              ConstraintQuery, DumpQuery> {};
+                              IndexQuery, AuthQuery, InfoQuery, ConstraintQuery,
+                              DumpQuery, ReplicationQuery> {};
 
 }  // namespace query

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -196,6 +196,95 @@ antlrcpp::Any CypherMainVisitor::visitDumpQuery(
   return dump_query;
 }
 
+antlrcpp::Any CypherMainVisitor::visitReplicationQuery(
+    MemgraphCypher::ReplicationQueryContext *ctx) {
+  CHECK(ctx->children.size() == 1)
+      << "ReplicationQuery should have exactly one child!";
+  auto *replication_query =
+      ctx->children[0]->accept(this).as<ReplicationQuery *>();
+  query_ = replication_query;
+  return replication_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitSetReplicationRole(
+    MemgraphCypher::SetReplicationRoleContext *ctx) {
+  auto *replication_query = storage_->Create<ReplicationQuery>();
+  replication_query->action_ = ReplicationQuery::Action::SET_REPLICATION_ROLE;
+  if (ctx->MAIN()) {
+    if (ctx->WITH() || ctx->PORT()) {
+      throw SemanticException("Main can't set a port!");
+    }
+    replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN;
+  } else if (ctx->REPLICA()) {
+    replication_query->role_ = ReplicationQuery::ReplicationRole::REPLICA;
+    if (ctx->WITH() && ctx->PORT()) {
+      if (ctx->port->numberLiteral() &&
+          ctx->port->numberLiteral()->integerLiteral()) {
+        replication_query->port_ = ctx->port->accept(this);
+      } else {
+        throw SyntaxException("Port must be an integer literal!");
+      }
+    }
+  }
+  return replication_query;
+}
+antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(
+    MemgraphCypher::ShowReplicationRoleContext *ctx) {
+  auto *replication_query = storage_->Create<ReplicationQuery>();
+  replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICATION_ROLE;
+  return replication_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitRegisterReplica(
+    MemgraphCypher::RegisterReplicaContext *ctx) {
+  auto *replication_query = storage_->Create<ReplicationQuery>();
+  replication_query->action_ = ReplicationQuery::Action::REGISTER_REPLICA;
+  replication_query->replica_name_ =
+      ctx->replicaName()->symbolicName()->accept(this).as<std::string>();
+  if (ctx->SYNC()) {
+    replication_query->sync_mode_ = query::ReplicationQuery::SyncMode::SYNC;
+    if (ctx->WITH() && ctx->TIMEOUT()) {
+      if (ctx->timeout->numberLiteral()) {
+        // we accept both double and integer literals
+        replication_query->timeout_ = ctx->timeout->accept(this);
+      } else {
+        throw SemanticException(
+            "Timeout should be a integer or double literal!");
+      }
+    }
+  } else if (ctx->ASYNC()) {
+    if (ctx->WITH() && ctx->TIMEOUT()) {
+      throw SyntaxException(
+          "Timeout can be set only for the SYNC replication mode!");
+    }
+    replication_query->sync_mode_ = query::ReplicationQuery::SyncMode::ASYNC;
+  }
+
+  if (!ctx->socketAddress()->literal()->StringLiteral()) {
+    throw SemanticException("Socket address should be a string literal!");
+  } else {
+    replication_query->socket_address_ = ctx->socketAddress()->accept(this);
+  }
+
+  return replication_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitDropReplica(
+    MemgraphCypher::DropReplicaContext *ctx) {
+  auto *replication_query = storage_->Create<ReplicationQuery>();
+  replication_query->action_ = ReplicationQuery::Action::DROP_REPLICA;
+  replication_query->replica_name_ =
+      ctx->replicaName()->symbolicName()->accept(this).as<std::string>();
+  return replication_query;
+}
+
+antlrcpp::Any CypherMainVisitor::visitShowReplicas(
+    MemgraphCypher::ShowReplicasContext *ctx) {
+  auto *replication_query = storage_->Create<ReplicationQuery>();
+  replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICAS;
+  return replication_query;
+}
+
 antlrcpp::Any CypherMainVisitor::visitCypherUnion(
     MemgraphCypher::CypherUnionContext *ctx) {
   bool distinct = !ctx->ALL();

src/query/frontend/ast/cypher_main_visitor.hpp

@@ -186,6 +186,42 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
    */
   antlrcpp::Any visitDumpQuery(MemgraphCypher::DumpQueryContext *ctx) override;
 
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitReplicationQuery(
+      MemgraphCypher::ReplicationQueryContext *ctx) override;
+
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitSetReplicationRole(
+      MemgraphCypher::SetReplicationRoleContext *ctx) override;
+
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitShowReplicationRole(
+      MemgraphCypher::ShowReplicationRoleContext *ctx) override;
+
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitRegisterReplica(
+      MemgraphCypher::RegisterReplicaContext *ctx) override;
+
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitDropReplica(
+      MemgraphCypher::DropReplicaContext *ctx) override;
+
+  /**
+   * @return ReplicationQuery*
+   */
+  antlrcpp::Any visitShowReplicas(
+      MemgraphCypher::ShowReplicasContext *ctx) override;
+
   /**
    * @return CypherUnion*
    */

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -8,6 +8,7 @@ import Cypher ;
 
 memgraphCypherKeyword : cypherKeyword
                       | ALTER
+                      | ASYNC
                       | AUTH
                       | CLEAR
                       | DATABASE
@@ -18,12 +19,21 @@ memgraphCypherKeyword : cypherKeyword
                       | FROM
                       | GRANT
                       | IDENTIFIED
+                      | MAIN
+                      | MODE
                       | PASSWORD
+                      | PORT
                       | PRIVILEGES
+                      | REGISTER
+                      | REPLICA
+                      | REPLICAS
+                      | REPLICATION
                       | REVOKE
                       | ROLE
                       | ROLES
                       | STATS
+                      | SYNC
+                      | TIMEOUT
                       | TO
                       | USER
                       | USERS
@@ -42,6 +52,7 @@ query : cypherQuery
       | constraintQuery
       | authQuery
       | dumpQuery
+      | replicationQuery
       ;
 
 authQuery : createRole
@@ -61,6 +72,13 @@ authQuery : createRole
           | showUsersForRole
           ;
 
+replicationQuery : setReplicationRole
+                 | showReplicationRole
+                 | registerReplica
+                 | dropReplica
+                 | showReplicas
+                 ;
+
 userOrRoleName : symbolicName ;
 
 createRole : CREATE ROLE role=userOrRoleName ;
@@ -100,3 +118,20 @@ showRoleForUser : SHOW ROLE FOR user=userOrRoleName ;
 showUsersForRole : SHOW USERS FOR role=userOrRoleName ;
 
 dumpQuery: DUMP DATABASE ;
+
+setReplicationRole  : SET REPLICATION ROLE TO ( MAIN | REPLICA )
+                      ( WITH PORT port=literal ) ? ;
+
+showReplicationRole : SHOW REPLICATION ROLE ;
+
+replicaName : symbolicName ;
+
+socketAddress : literal ;
+
+registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC )
+                ( WITH TIMEOUT timeout=literal ) ?
+                TO socketAddress ;
+
+dropReplica : DROP REPLICA replicaName ;
+
+showReplicas  : SHOW REPLICAS ;

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -11,6 +11,7 @@ lexer grammar MemgraphCypherLexer ;
 import CypherLexer ;
 
 ALTER          : A L T E R ;
+ASYNC          : A S Y N C ;
 AUTH           : A U T H ;
 CLEAR          : C L E A R ;
 DATABASE       : D A T A B A S E ;
@@ -22,12 +23,21 @@ FROM           : F R O M ;
 GRANT          : G R A N T ;
 GRANTS         : G R A N T S ;
 IDENTIFIED     : I D E N T I F I E D ;
+MAIN           : M A I N ;
+MODE           : M O D E ;
 PASSWORD       : P A S S W O R D ;
+PORT           : P O R T ;
 PRIVILEGES     : P R I V I L E G E S ;
+REGISTER       : R E G I S T E R ;
+REPLICA        : R E P L I C A ;
+REPLICAS       : R E P L I C A S ;
+REPLICATION    : R E P L I C A T I O N ;
 REVOKE         : R E V O K E ;
 ROLE           : R O L E ;
 ROLES          : R O L E S ;
 STATS          : S T A T S ;
+SYNC           : S Y N C ;
+TIMEOUT        : T I M E O U T ;
 TO             : T O ;
 USER           : U S E R ;
 USERS          : U S E R S ;

src/query/frontend/semantic/required_privileges.cpp

@@ -59,6 +59,26 @@ class PrivilegeExtractor : public QueryVisitor<void>,
     AddPrivilege(AuthQuery::Privilege::DUMP);
   }
 
+  void Visit(ReplicationQuery &replication_query) override {
+    switch (replication_query.action_) {
+      case ReplicationQuery::Action::SET_REPLICATION_ROLE:
+        AddPrivilege(AuthQuery::Privilege::REPLICATION);
+        break;
+      case ReplicationQuery::Action::SHOW_REPLICATION_ROLE:
+        AddPrivilege(AuthQuery::Privilege::REPLICATION);
+        break;
+      case ReplicationQuery::Action::REGISTER_REPLICA:
+        AddPrivilege(AuthQuery::Privilege::REPLICATION);
+        break;
+      case ReplicationQuery::Action::DROP_REPLICA:
+        AddPrivilege(AuthQuery::Privilege::REPLICATION);
+        break;
+      case ReplicationQuery::Action::SHOW_REPLICAS:
+        AddPrivilege(AuthQuery::Privilege::REPLICATION);
+        break;
+    }
+  }
+
   bool PreVisit(Create &) override {
     AddPrivilege(AuthQuery::Privilege::CREATE);
     return false;

src/query/interpreter.cpp

@@ -5,6 +5,7 @@
 #include <glog/logging.h>
 
 #include "glue/communication.hpp"
+#include "query/constants.hpp"
 #include "query/context.hpp"
 #include "query/db_accessor.hpp"
 #include "query/dump.hpp"
@@ -162,6 +163,176 @@ TypedValue EvaluateOptionalExpression(Expression *expression,
   return expression ? expression->Accept(*eval) : TypedValue();
 }
 
+#ifdef MG_ENTERPRISE
+class ReplQueryHandler final : public query::ReplicationQueryHandler {
+ public:
+  explicit ReplQueryHandler(storage::Storage *db) : db_(db) {}
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role,
+                          std::optional<int64_t> port) override {
+    if (replication_role == ReplicationQuery::ReplicationRole::MAIN) {
+      if (!db_->SetMainReplicationRole()) {
+        throw QueryRuntimeException("Couldn't set role to main!");
+      }
+    }
+    if (replication_role == ReplicationQuery::ReplicationRole::REPLICA) {
+      if (!port || *port < 0 || *port > std::numeric_limits<uint16_t>::max()) {
+        throw QueryRuntimeException("Port number invalid!");
+      }
+      if (!db_->SetReplicaRole(
+              io::network::Endpoint(query::kDefaultReplicationServerIp,
+                                    static_cast<uint16_t>(*port)))) {
+        throw QueryRuntimeException("Couldn't set role to replica!");
+      }
+    }
+  }
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  ReplicationQuery::ReplicationRole ShowReplicationRole() const override {
+    switch (db_->GetReplicationRole()) {
+      case storage::ReplicationRole::MAIN:
+        return ReplicationQuery::ReplicationRole::MAIN;
+      case storage::ReplicationRole::REPLICA:
+        return ReplicationQuery::ReplicationRole::REPLICA;
+    }
+    throw QueryRuntimeException(
+        "Couldn't show replication role - invalid role set!");
+  }
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  void RegisterReplica(const std::string &name,
+                       const std::string &socket_address,
+                       const ReplicationQuery::SyncMode sync_mode,
+                       const std::optional<double> timeout) override {
+    if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
+      // replica can't register another replica
+      throw QueryRuntimeException("Replica can't register another replica!");
+    }
+
+    storage::replication::ReplicationMode repl_mode;
+    switch (sync_mode) {
+      case ReplicationQuery::SyncMode::ASYNC: {
+        repl_mode = storage::replication::ReplicationMode::ASYNC;
+        break;
+      }
+      case ReplicationQuery::SyncMode::SYNC: {
+        repl_mode = storage::replication::ReplicationMode::SYNC;
+        break;
+      }
+    }
+
+    auto maybe_ip_and_port = io::network::Endpoint::ParseSocketOrIpAddress(
+        socket_address, query::kDefaultReplicationPort);
+    if (maybe_ip_and_port) {
+      auto [ip, port] = *maybe_ip_and_port;
+      auto ret =
+          db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
+                               {.timeout = timeout, .ssl = std::nullopt});
+      if (ret.HasError()) {
+        throw QueryRuntimeException(
+            fmt::format("Couldn't register replica '{}'!", name));
+      }
+    } else {
+      throw QueryRuntimeException("Invalid socket address!");
+    }
+  }
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  void DropReplica(const std::string &replica_name) override {
+    if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
+      // replica can't unregister a replica
+      throw QueryRuntimeException("Replica can't unregister a replica!");
+    }
+    if (!db_->UnregisterReplica(replica_name)) {
+      throw QueryRuntimeException(
+          fmt::format("Couldn't unregister the replica '{}'", replica_name));
+    }
+  }
+
+  using Replica = ReplicationQueryHandler::Replica;
+  std::vector<Replica> ShowReplicas() const override {
+    if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
+      // replica can't show registered replicas (it shouldn't have any)
+      throw QueryRuntimeException(
+          "Replica can't show registered replicas (it shouldn't have any)!");
+    }
+
+    auto repl_infos = db_->ReplicasInfo();
+    std::vector<Replica> replicas;
+    replicas.reserve(repl_infos.size());
+
+    const auto from_info = [](const auto &repl_info) -> Replica {
+      Replica replica;
+      replica.name = repl_info.name;
+      replica.socket_address = repl_info.endpoint.SocketAddress();
+      switch (repl_info.mode) {
+        case storage::replication::ReplicationMode::SYNC:
+          replica.sync_mode = ReplicationQuery::SyncMode::SYNC;
+          break;
+        case storage::replication::ReplicationMode::ASYNC:
+          replica.sync_mode = ReplicationQuery::SyncMode::ASYNC;
+          break;
+      }
+      if (repl_info.timeout) {
+        replica.timeout = *repl_info.timeout;
+      }
+
+      return replica;
+    };
+
+    std::transform(repl_infos.begin(), repl_infos.end(),
+                   std::back_inserter(replicas), from_info);
+    return replicas;
+  }
+
+ private:
+  storage::Storage *db_;
+};
+/// returns false if the replication role can't be set
+/// @throw QueryRuntimeException if an error ocurred.
+#else
+
+class NoReplicationInCommunity : public query::QueryRuntimeException {
+ public:
+  NoReplicationInCommunity()
+      : query::QueryRuntimeException::QueryRuntimeException(
+            "Replication is not supported in Memgraph Community!") {}
+};
+
+class ReplQueryHandler : public query::ReplicationQueryHandler {
+ public:
+  // Dummy ctor - just there to make the replication query handler work
+  // in both community and enterprise versions.
+  explicit ReplQueryHandler(storage::Storage *db) {}
+  void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role,
+                          std::optional<int64_t> port) override {
+    throw NoReplicationInCommunity();
+  }
+
+  ReplicationQuery::ReplicationRole ShowReplicationRole() const override {
+    throw NoReplicationInCommunity();
+  }
+
+  void RegisterReplica(const std::string &name,
+                       const std::string &socket_address,
+                       const ReplicationQuery::SyncMode sync_mode,
+                       const std::optional<double> timeout) {
+    throw NoReplicationInCommunity();
+  }
+
+  void DropReplica(const std::string &replica_name) override {
+    throw NoReplicationInCommunity();
+  }
+
+  using Replica = ReplicationQueryHandler::Replica;
+
+  std::vector<Replica> ShowReplicas() const override {
+    throw NoReplicationInCommunity();
+  }
+};
+#endif
+
 Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth,
                          const Parameters &parameters,
                          DbAccessor *db_accessor) {
@@ -323,6 +494,118 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth,
   }
 }
 
+Callback HandleReplicationQuery(ReplicationQuery *repl_query,
+                                ReplQueryHandler *handler,
+                                const Parameters &parameters,
+                                DbAccessor *db_accessor) {
+  Frame frame(0);
+  SymbolTable symbol_table;
+  EvaluationContext evaluation_context;
+  evaluation_context.timestamp =
+      std::chrono::duration_cast<std::chrono::milliseconds>(
+          std::chrono::system_clock::now().time_since_epoch())
+          .count();
+  evaluation_context.parameters = parameters;
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context,
+                                db_accessor, storage::View::OLD);
+
+  Callback callback;
+  switch (repl_query->action_) {
+    case ReplicationQuery::Action::SET_REPLICATION_ROLE: {
+      auto port = EvaluateOptionalExpression(repl_query->port_, &evaluator);
+      std::optional<int64_t> maybe_port;
+      if (port.IsInt()) {
+        maybe_port = port.ValueInt();
+      }
+      callback.fn = [handler, role = repl_query->role_, maybe_port] {
+        handler->SetReplicationRole(role, maybe_port);
+        return std::vector<std::vector<TypedValue>>();
+      };
+      return callback;
+    }
+    case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
+      callback.header = {"replication mode"};
+      callback.fn = [handler] {
+        auto mode = handler->ShowReplicationRole();
+        switch (mode) {
+          case ReplicationQuery::ReplicationRole::MAIN: {
+            return std::vector<std::vector<TypedValue>>{{TypedValue("main")}};
+          }
+          case ReplicationQuery::ReplicationRole::REPLICA: {
+            return std::vector<std::vector<TypedValue>>{
+                {TypedValue("replica")}};
+          }
+        }
+      };
+      return callback;
+    }
+    case ReplicationQuery::Action::REGISTER_REPLICA: {
+      const auto &name = repl_query->replica_name_;
+      const auto &sync_mode = repl_query->sync_mode_;
+      auto socket_address = repl_query->socket_address_->Accept(evaluator);
+      auto timeout =
+          EvaluateOptionalExpression(repl_query->timeout_, &evaluator);
+      std::optional<double> maybe_timeout;
+      if (timeout.IsDouble()) {
+        maybe_timeout = timeout.ValueDouble();
+      } else if (timeout.IsInt()) {
+        maybe_timeout = static_cast<double>(timeout.ValueInt());
+      }
+      callback.fn = [handler, name, socket_address, sync_mode, maybe_timeout] {
+        CHECK(socket_address.IsString());
+        handler->RegisterReplica(name,
+                                 std::string(socket_address.ValueString()),
+                                 sync_mode, maybe_timeout);
+        return std::vector<std::vector<TypedValue>>();
+      };
+      return callback;
+    }
+    case ReplicationQuery::Action::DROP_REPLICA: {
+      const auto &name = repl_query->replica_name_;
+      callback.fn = [handler, name] {
+        handler->DropReplica(name);
+        return std::vector<std::vector<TypedValue>>();
+      };
+      return callback;
+    }
+    case ReplicationQuery::Action::SHOW_REPLICAS: {
+      callback.header = {"name", "socket_address", "sync_mode", "timeout"};
+      callback.fn = [handler, replica_nfields = callback.header.size()] {
+        const auto &replicas = handler->ShowReplicas();
+        auto typed_replicas = std::vector<std::vector<TypedValue>>{};
+        typed_replicas.reserve(replicas.size());
+        for (auto &replica : replicas) {
+          std::vector<TypedValue> typed_replica;
+          typed_replica.reserve(replica_nfields);
+
+          typed_replica.emplace_back(TypedValue(replica.name));
+          typed_replica.emplace_back(TypedValue(replica.socket_address));
+          switch (replica.sync_mode) {
+            case ReplicationQuery::SyncMode::SYNC:
+              typed_replica.emplace_back(TypedValue("sync"));
+              break;
+            case ReplicationQuery::SyncMode::ASYNC:
+              typed_replica.emplace_back(TypedValue("async"));
+              break;
+          }
+          typed_replica.emplace_back(
+              TypedValue(static_cast<int64_t>(replica.sync_mode)));
+          if (replica.timeout) {
+            typed_replica.emplace_back(TypedValue(*replica.timeout));
+          } else {
+            typed_replica.emplace_back(TypedValue());
+          }
+
+          typed_replicas.emplace_back(std::move(typed_replica));
+        }
+        return typed_replicas;
+      };
+      return callback;
+    }
+      return callback;
+  }
+}
+
 Interpreter::Interpreter(InterpreterContext *interpreter_context)
     : interpreter_context_(interpreter_context) {
   CHECK(interpreter_context_) << "Interpreter context must not be NULL";
@@ -896,6 +1179,32 @@ PreparedQuery PrepareAuthQuery(
       }};
 }
 
+PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query,
+                                      bool in_explicit_transaction,
+                                      InterpreterContext *interpreter_context,
+                                      DbAccessor *dba) {
+  if (in_explicit_transaction) {
+    throw ReplicationModificationInMulticommandTxException();
+  }
+
+  auto *replication_query =
+      utils::Downcast<ReplicationQuery>(parsed_query.query);
+  ReplQueryHandler handler{interpreter_context->db};
+  auto callback = HandleReplicationQuery(replication_query, &handler,
+                                         parsed_query.parameters, dba);
+
+  return PreparedQuery{
+      callback.header, std::move(parsed_query.required_privileges),
+      [pull_plan = std::make_shared<PullPlanVector>(callback.fn())](
+          AnyStream *stream,
+          std::optional<int> n) -> std::optional<QueryHandlerResult> {
+        if (pull_plan->Pull(stream, n)) {
+          return QueryHandlerResult::COMMIT;
+        }
+        return std::nullopt;
+      }};
+}
+
 PreparedQuery PrepareInfoQuery(
     ParsedQuery parsed_query, bool in_explicit_transaction,
     std::map<std::string, TypedValue> *summary,
@@ -1279,6 +1588,10 @@ Interpreter::PrepareResult Interpreter::Prepare(
           std::move(parsed_query), in_explicit_transaction_,
           &query_execution->summary, interpreter_context_,
           &query_execution->execution_memory);
+    } else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
+      prepared_query = PrepareReplicationQuery(
+          std::move(parsed_query), in_explicit_transaction_,
+          interpreter_context_, &*execution_db_accessor_);
     } else {
       LOG(FATAL) << "Should not get here -- unknown query type!";
     }

src/query/interpreter.hpp

@@ -98,6 +98,45 @@ class AuthQueryHandler {
 
 enum class QueryHandlerResult { COMMIT, ABORT, NOTHING };
 
+class ReplicationQueryHandler {
+ public:
+  ReplicationQueryHandler() = default;
+  virtual ~ReplicationQueryHandler() = default;
+
+  ReplicationQueryHandler(const ReplicationQueryHandler &) = delete;
+  ReplicationQueryHandler &operator=(const ReplicationQueryHandler &) = delete;
+
+  ReplicationQueryHandler(ReplicationQueryHandler &&) = delete;
+  ReplicationQueryHandler &operator=(ReplicationQueryHandler &&) = delete;
+
+  struct Replica {
+    std::string name;
+    std::string socket_address;
+    ReplicationQuery::SyncMode sync_mode;
+    std::optional<double> timeout;
+  };
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual void SetReplicationRole(
+      ReplicationQuery::ReplicationRole replication_role,
+      std::optional<int64_t> port) = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual ReplicationQuery::ReplicationRole ShowReplicationRole() const = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual void RegisterReplica(const std::string &name,
+                               const std::string &socket_address,
+                               const ReplicationQuery::SyncMode sync_mode,
+                               const std::optional<double> timeout) = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual void DropReplica(const std::string &replica_name) = 0;
+
+  /// @throw QueryRuntimeException if an error ocurred.
+  virtual std::vector<Replica> ShowReplicas() const = 0;
+};
+
 /**
  * A container for data related to the preparation of a query.
  */

src/rpc/client.hpp

@@ -119,7 +119,7 @@ class Client {
   ///                            RPC call (eg. connection failed, remote end
   ///                            died, etc.)
   template <class TRequestResponse, class... Args>
-  StreamHandler<TRequestResponse> Stream(Args &&... args) {
+  StreamHandler<TRequestResponse> Stream(Args &&...args) {
     return StreamWithLoad<TRequestResponse>(
         [](auto *reader) {
           typename TRequestResponse::Response response;
@@ -133,7 +133,7 @@ class Client {
   template <class TRequestResponse, class... Args>
   StreamHandler<TRequestResponse> StreamWithLoad(
       std::function<typename TRequestResponse::Response(slk::Reader *)> load,
-      Args &&... args) {
+      Args &&...args) {
     typename TRequestResponse::Request request(std::forward<Args>(args)...);
     auto req_type = TRequestResponse::Request::kType;
     VLOG(12) << "[RpcClient] sent " << req_type.name;
@@ -177,7 +177,7 @@ class Client {
   ///                            RPC call (eg. connection failed, remote end
   ///                            died, etc.)
   template <class TRequestResponse, class... Args>
-  typename TRequestResponse::Response Call(Args &&... args) {
+  typename TRequestResponse::Response Call(Args &&...args) {
     auto stream = Stream<TRequestResponse>(std::forward<Args>(args)...);
     return stream.AwaitResponse();
   }
@@ -186,7 +186,7 @@ class Client {
   template <class TRequestResponse, class... Args>
   typename TRequestResponse::Response CallWithLoad(
       std::function<typename TRequestResponse::Response(slk::Reader *)> load,
-      Args &&... args) {
+      Args &&...args) {
     auto stream = StreamWithLoad(load, std::forward<Args>(args)...);
     return stream.AwaitResponse();
   }
@@ -194,6 +194,8 @@ class Client {
   /// Call this function from another thread to abort a pending RPC call.
   void Abort();
 
+  const auto &Endpoint() const { return endpoint_; }
+
  private:
   io::network::Endpoint endpoint_;
   communication::ClientContext *context_;

src/storage/v2/CMakeLists.txt

@@ -10,5 +10,26 @@ set(storage_v2_src_files
     vertex_accessor.cpp
     storage.cpp)
 
+if(MG_ENTERPRISE)
+    define_add_lcp(add_lcp_storage lcp_storage_cpp_files generated_lcp_storage_files)
+
+    add_lcp_storage(replication/rpc.lcp SLK_SERIALIZE)
+
+    add_custom_target(generate_lcp_storage DEPENDS ${generated_lcp_storage_files})
+
+    set(storage_v2_src_files
+        ${storage_v2_src_files}
+        replication/replication_client.cpp
+        replication/replication_server.cpp
+        replication/serialization.cpp
+        replication/slk.cpp
+        ${lcp_storage_cpp_files})
+endif()
+
 add_library(mg-storage-v2 STATIC ${storage_v2_src_files})
 target_link_libraries(mg-storage-v2 Threads::Threads mg-utils glog gflags)
+
+if(MG_ENTERPRISE)
+    add_dependencies(mg-storage-v2 generate_lcp_storage)
+    target_link_libraries(mg-storage-v2 mg-rpc mg-slk)
+endif()

src/storage/v2/durability/durability.cpp

@@ -50,54 +50,10 @@ void VerifyStorageDirectoryOwnerAndProcessUserOrDie(
       << ". Please start the process as user " << user_directory << "!";
 }
 
-std::optional<RecoveryInfo> RecoverData(
+std::vector<SnapshotDurabilityInfo> GetSnapshotFiles(
     const std::filesystem::path &snapshot_directory,
-    const std::filesystem::path &wal_directory, std::string *uuid,
-    utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
-    std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
-    Indices *indices, Constraints *constraints, Config::Items items,
-    uint64_t *wal_seq_num) {
-  if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory))
-    return std::nullopt;
-
-  // Helper lambda used to recover all discovered indices and constraints. The
-  // indices and constraints must be recovered after the data recovery is done
-  // to ensure that the indices and constraints are consistent at the end of the
-  // recovery process.
-  auto recover_indices_and_constraints = [&](const auto &indices_constraints) {
-    // Recover label indices.
-    for (const auto &item : indices_constraints.indices.label) {
-      if (!indices->label_index.CreateIndex(item, vertices->access()))
-        throw RecoveryFailure("The label index must be created here!");
-    }
-
-    // Recover label+property indices.
-    for (const auto &item : indices_constraints.indices.label_property) {
-      if (!indices->label_property_index.CreateIndex(item.first, item.second,
-                                                     vertices->access()))
-        throw RecoveryFailure("The label+property index must be created here!");
-    }
-
-    // Recover existence constraints.
-    for (const auto &item : indices_constraints.constraints.existence) {
-      auto ret = CreateExistenceConstraint(constraints, item.first, item.second,
-                                           vertices->access());
-      if (ret.HasError() || !ret.GetValue())
-        throw RecoveryFailure("The existence constraint must be created here!");
-    }
-
-    // Recover unique constraints.
-    for (const auto &item : indices_constraints.constraints.unique) {
-      auto ret = constraints->unique_constraints.CreateConstraint(
-          item.first, item.second, vertices->access());
-      if (ret.HasError() ||
-          ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
-        throw RecoveryFailure("The unique constraint must be created here!");
-    }
-  };
-
-  // Array of all discovered snapshots, ordered by name.
-  std::vector<std::pair<std::filesystem::path, std::string>> snapshot_files;
+    const std::string_view uuid) {
+  std::vector<SnapshotDurabilityInfo> snapshot_files;
   std::error_code error_code;
   if (utils::DirExists(snapshot_directory)) {
     for (const auto &item :
@@ -105,7 +61,10 @@ std::optional<RecoveryInfo> RecoverData(
       if (!item.is_regular_file()) continue;
       try {
         auto info = ReadSnapshotInfo(item.path());
-        snapshot_files.emplace_back(item.path(), info.uuid);
+        if (uuid.empty() || info.uuid == uuid) {
+          snapshot_files.emplace_back(item.path(), std::move(info.uuid),
+                                      info.start_timestamp);
+        }
       } catch (const RecoveryFailure &) {
         continue;
       }
@@ -114,16 +73,102 @@ std::optional<RecoveryInfo> RecoverData(
                        << error_code.message() << "!";
   }
 
+  return snapshot_files;
+}
+
+std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(
+    const std::filesystem::path &wal_directory, const std::string_view uuid,
+    const std::optional<size_t> current_seq_num) {
+  if (!utils::DirExists(wal_directory)) return std::nullopt;
+
+  std::vector<WalDurabilityInfo> wal_files;
+  std::error_code error_code;
+  for (const auto &item :
+       std::filesystem::directory_iterator(wal_directory, error_code)) {
+    if (!item.is_regular_file()) continue;
+    try {
+      auto info = ReadWalInfo(item.path());
+      if ((uuid.empty() || info.uuid == uuid) &&
+          (!current_seq_num || info.seq_num < *current_seq_num))
+        wal_files.emplace_back(info.seq_num, info.from_timestamp,
+                               info.to_timestamp, std::move(info.uuid),
+                               std::move(info.epoch_id), item.path());
+    } catch (const RecoveryFailure &e) {
+      DLOG(WARNING) << "Failed to read " << item.path();
+      continue;
+    }
+  }
+  CHECK(!error_code) << "Couldn't recover data because an error occurred: "
+                     << error_code.message() << "!";
+
+  std::sort(wal_files.begin(), wal_files.end());
+  return std::move(wal_files);
+}
+
+// Function used to recover all discovered indices and constraints. The
+// indices and constraints must be recovered after the data recovery is done
+// to ensure that the indices and constraints are consistent at the end of the
+// recovery process.
+void RecoverIndicesAndConstraints(
+    const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
+    Constraints *constraints, utils::SkipList<Vertex> *vertices) {
+  // Recover label indices.
+  for (const auto &item : indices_constraints.indices.label) {
+    if (!indices->label_index.CreateIndex(item, vertices->access()))
+      throw RecoveryFailure("The label index must be created here!");
+  }
+
+  // Recover label+property indices.
+  for (const auto &item : indices_constraints.indices.label_property) {
+    if (!indices->label_property_index.CreateIndex(item.first, item.second,
+                                                   vertices->access()))
+      throw RecoveryFailure("The label+property index must be created here!");
+  }
+
+  // Recover existence constraints.
+  for (const auto &item : indices_constraints.constraints.existence) {
+    auto ret = CreateExistenceConstraint(constraints, item.first, item.second,
+                                         vertices->access());
+    if (ret.HasError() || !ret.GetValue())
+      throw RecoveryFailure("The existence constraint must be created here!");
+  }
+
+  // Recover unique constraints.
+  for (const auto &item : indices_constraints.constraints.unique) {
+    auto ret = constraints->unique_constraints.CreateConstraint(
+        item.first, item.second, vertices->access());
+    if (ret.HasError() ||
+        ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
+      throw RecoveryFailure("The unique constraint must be created here!");
+  }
+}
+
+std::optional<RecoveryInfo> RecoverData(
+    const std::filesystem::path &snapshot_directory,
+    const std::filesystem::path &wal_directory, std::string *uuid,
+    std::string *epoch_id,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
+    utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
+    std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
+    Indices *indices, Constraints *constraints, Config::Items items,
+    uint64_t *wal_seq_num) {
+  if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory))
+    return std::nullopt;
+
+  auto snapshot_files = GetSnapshotFiles(snapshot_directory);
+
   RecoveryInfo recovery_info;
   RecoveredIndicesAndConstraints indices_constraints;
   std::optional<uint64_t> snapshot_timestamp;
   if (!snapshot_files.empty()) {
+    // Order the files by name
     std::sort(snapshot_files.begin(), snapshot_files.end());
+
     // UUID used for durability is the UUID of the last snapshot file.
-    *uuid = snapshot_files.back().second;
+    *uuid = snapshot_files.back().uuid;
     std::optional<RecoveredSnapshot> recovered_snapshot;
     for (auto it = snapshot_files.rbegin(); it != snapshot_files.rend(); ++it) {
-      const auto &[path, file_uuid] = *it;
+      const auto &[path, file_uuid, _] = *it;
       if (file_uuid != *uuid) {
         LOG(WARNING) << "The snapshot file " << path
                      << " isn't related to the latest snapshot file!";
@@ -131,8 +176,8 @@ std::optional<RecoveryInfo> RecoverData(
       }
       LOG(INFO) << "Starting snapshot recovery from " << path;
       try {
-        recovered_snapshot = LoadSnapshot(path, vertices, edges, name_id_mapper,
-                                          edge_count, items);
+        recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history,
+                                          name_id_mapper, edge_count, items);
         LOG(INFO) << "Snapshot recovery successful!";
         break;
       } catch (const RecoveryFailure &e) {
@@ -148,20 +193,39 @@ std::optional<RecoveryInfo> RecoverData(
     recovery_info = recovered_snapshot->recovery_info;
     indices_constraints = std::move(recovered_snapshot->indices_constraints);
     snapshot_timestamp = recovered_snapshot->snapshot_info.start_timestamp;
+    *epoch_id = std::move(recovered_snapshot->snapshot_info.epoch_id);
+
     if (!utils::DirExists(wal_directory)) {
-      recover_indices_and_constraints(indices_constraints);
+      RecoverIndicesAndConstraints(indices_constraints, indices, constraints,
+                                   vertices);
       return recovered_snapshot->recovery_info;
     }
   } else {
+    std::error_code error_code;
     if (!utils::DirExists(wal_directory)) return std::nullopt;
-    // Array of all discovered WAL files, ordered by name.
-    std::vector<std::pair<std::filesystem::path, std::string>> wal_files;
+    // We use this smaller struct that contains only a subset of information
+    // necessary for the rest of the recovery function.
+    // Also, the struct is sorted primarily on the path it contains.
+    struct WalFileInfo {
+      explicit WalFileInfo(std::filesystem::path path, std::string uuid,
+                           std::string epoch_id)
+          : path(std::move(path)),
+            uuid(std::move(uuid)),
+            epoch_id(std::move(epoch_id)) {}
+      std::filesystem::path path;
+      std::string uuid;
+      std::string epoch_id;
+
+      auto operator<=>(const WalFileInfo &) const = default;
+    };
+    std::vector<WalFileInfo> wal_files;
     for (const auto &item :
          std::filesystem::directory_iterator(wal_directory, error_code)) {
       if (!item.is_regular_file()) continue;
       try {
         auto info = ReadWalInfo(item.path());
-        wal_files.emplace_back(item.path(), info.uuid);
+        wal_files.emplace_back(item.path(), std::move(info.uuid),
+                               std::move(info.epoch_id));
       } catch (const RecoveryFailure &e) {
         continue;
       }
@@ -171,26 +235,17 @@ std::optional<RecoveryInfo> RecoverData(
     if (wal_files.empty()) return std::nullopt;
     std::sort(wal_files.begin(), wal_files.end());
     // UUID used for durability is the UUID of the last WAL file.
-    *uuid = wal_files.back().second;
+    // Same for the epoch id.
+    *uuid = std::move(wal_files.back().uuid);
+    *epoch_id = std::move(wal_files.back().epoch_id);
   }
 
+  auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
+  if (!maybe_wal_files) return std::nullopt;
+
   // Array of all discovered WAL files, ordered by sequence number.
-  std::vector<std::tuple<uint64_t, uint64_t, uint64_t, std::filesystem::path>>
-      wal_files;
-  for (const auto &item :
-       std::filesystem::directory_iterator(wal_directory, error_code)) {
-    if (!item.is_regular_file()) continue;
-    try {
-      auto info = ReadWalInfo(item.path());
-      if (info.uuid != *uuid) continue;
-      wal_files.emplace_back(info.seq_num, info.from_timestamp,
-                             info.to_timestamp, item.path());
-    } catch (const RecoveryFailure &e) {
-      continue;
-    }
-  }
-  CHECK(!error_code) << "Couldn't recover data because an error occurred: "
-                     << error_code.message() << "!";
+  auto &wal_files = *maybe_wal_files;
+
   // By this point we should have recovered from a snapshot, or we should have
   // found some WAL files to recover from in the above `else`. This is just a
   // sanity check to circumvent the following case: The database didn't recover
@@ -203,10 +258,9 @@ std::optional<RecoveryInfo> RecoverData(
          "files that match the last WAL file!";
 
   if (!wal_files.empty()) {
-    std::sort(wal_files.begin(), wal_files.end());
     {
-      const auto &[seq_num, from_timestamp, to_timestamp, path] = wal_files[0];
-      if (seq_num != 0) {
+      const auto &first_wal = wal_files[0];
+      if (first_wal.seq_num != 0) {
         // We don't have all WAL files. We need to see whether we need them all.
         if (!snapshot_timestamp) {
           // We didn't recover from a snapshot and we must have all WAL files
@@ -214,7 +268,7 @@ std::optional<RecoveryInfo> RecoverData(
           // data from them.
           LOG(FATAL) << "There are missing prefix WAL files and data can't be "
                         "recovered without them!";
-        } else if (to_timestamp >= *snapshot_timestamp) {
+        } else if (first_wal.to_timestamp >= *snapshot_timestamp) {
           // We recovered from a snapshot and we must have at least one WAL file
           // whose all deltas were created before the snapshot in order to
           // verify that nothing is missing from the beginning of the WAL chain.
@@ -224,16 +278,29 @@ std::optional<RecoveryInfo> RecoverData(
       }
     }
     std::optional<uint64_t> previous_seq_num;
-    for (const auto &[seq_num, from_timestamp, to_timestamp, path] :
-         wal_files) {
-      if (previous_seq_num && *previous_seq_num + 1 != seq_num) {
+    auto last_loaded_timestamp = snapshot_timestamp;
+    for (auto &wal_file : wal_files) {
+      if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) {
         LOG(FATAL) << "You are missing a WAL file with the sequence number "
                    << *previous_seq_num + 1 << "!";
       }
-      previous_seq_num = seq_num;
+      previous_seq_num = wal_file.seq_num;
+
+      if (wal_file.epoch_id != *epoch_id) {
+        // This way we skip WALs finalized only because of role change.
+        // We can also set the last timestamp to 0 if last loaded timestamp
+        // is nullopt as this can only happen if the WAL file with seq = 0
+        // does not contain any deltas and we didn't find any snapshots.
+        if (last_loaded_timestamp) {
+          epoch_history->emplace_back(wal_file.epoch_id,
+                                      *last_loaded_timestamp);
+        }
+        *epoch_id = std::move(wal_file.epoch_id);
+      }
       try {
-        auto info = LoadWal(path, &indices_constraints, snapshot_timestamp,
-                            vertices, edges, name_id_mapper, edge_count, items);
+        auto info =
+            LoadWal(wal_file.path, &indices_constraints, last_loaded_timestamp,
+                    vertices, edges, name_id_mapper, edge_count, items);
         recovery_info.next_vertex_id =
             std::max(recovery_info.next_vertex_id, info.next_vertex_id);
         recovery_info.next_edge_id =
@@ -241,16 +308,21 @@ std::optional<RecoveryInfo> RecoverData(
         recovery_info.next_timestamp =
             std::max(recovery_info.next_timestamp, info.next_timestamp);
       } catch (const RecoveryFailure &e) {
-        LOG(FATAL) << "Couldn't recover WAL deltas from " << path
+        LOG(FATAL) << "Couldn't recover WAL deltas from " << wal_file.path
                    << " because of: " << e.what();
       }
+
+      if (recovery_info.next_timestamp != 0) {
+        last_loaded_timestamp.emplace(recovery_info.next_timestamp - 1);
+      }
     }
     // The sequence number needs to be recovered even though `LoadWal` didn't
     // load any deltas from that file.
     *wal_seq_num = *previous_seq_num + 1;
   }
 
-  recover_indices_and_constraints(indices_constraints);
+  RecoverIndicesAndConstraints(indices_constraints, indices, constraints,
+                               vertices);
   return recovery_info;
 }
 

src/storage/v2/durability/durability.hpp

@@ -5,10 +5,12 @@
 #include <filesystem>
 #include <optional>
 #include <string>
+#include <variant>
 
 #include "storage/v2/config.hpp"
 #include "storage/v2/constraints.hpp"
 #include "storage/v2/durability/metadata.hpp"
+#include "storage/v2/durability/wal.hpp"
 #include "storage/v2/edge.hpp"
 #include "storage/v2/indices.hpp"
 #include "storage/v2/name_id_mapper.hpp"
@@ -23,12 +25,85 @@ namespace storage::durability {
 void VerifyStorageDirectoryOwnerAndProcessUserOrDie(
     const std::filesystem::path &storage_directory);
 
+// Used to capture the snapshot's data related to durability
+struct SnapshotDurabilityInfo {
+  explicit SnapshotDurabilityInfo(std::filesystem::path path, std::string uuid,
+                                  const uint64_t start_timestamp)
+      : path(std::move(path)),
+        uuid(std::move(uuid)),
+        start_timestamp(start_timestamp) {}
+
+  std::filesystem::path path;
+  std::string uuid;
+  uint64_t start_timestamp;
+
+  auto operator<=>(const SnapshotDurabilityInfo &) const = default;
+};
+
+/// Get list of snapshot files with their UUID.
+/// @param snapshot_directory Directory containing the Snapshot files.
+/// @param uuid UUID of the Snapshot files. If not empty, fetch only Snapshot
+/// file with the specified UUID. Otherwise, fetch only Snapshot files in the
+/// snapshot_directory.
+/// @return List of snapshot files defined with its path and UUID.
+std::vector<SnapshotDurabilityInfo> GetSnapshotFiles(
+    const std::filesystem::path &snapshot_directory,
+    std::string_view uuid = "");
+
+/// Used to capture a WAL's data related to durability
+struct WalDurabilityInfo {
+  explicit WalDurabilityInfo(const uint64_t seq_num,
+                             const uint64_t from_timestamp,
+                             const uint64_t to_timestamp, std::string uuid,
+                             std::string epoch_id, std::filesystem::path path)
+      : seq_num(seq_num),
+        from_timestamp(from_timestamp),
+        to_timestamp(to_timestamp),
+        uuid(std::move(uuid)),
+        epoch_id(std::move(epoch_id)),
+        path(std::move(path)) {}
+
+  uint64_t seq_num;
+  uint64_t from_timestamp;
+  uint64_t to_timestamp;
+  std::string uuid;
+  std::string epoch_id;
+  std::filesystem::path path;
+
+  auto operator<=>(const WalDurabilityInfo &) const = default;
+};
+
+/// Get list of WAL files ordered by the sequence number
+/// @param wal_directory Directory containing the WAL files.
+/// @param uuid UUID of the WAL files. If not empty, fetch only WAL files
+/// with the specified UUID. Otherwise, fetch all WAL files in the
+/// wal_directory.
+/// @param current_seq_num Sequence number of the WAL file which is currently
+/// being written. If specified, load only finalized WAL files, i.e. WAL files
+/// with seq_num < current_seq_num.
+/// @return List of WAL files. Each WAL file is defined with its sequence
+/// number, from timestamp, to timestamp and path.
+std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(
+    const std::filesystem::path &wal_directory, std::string_view uuid = "",
+    std::optional<size_t> current_seq_num = {});
+
+// Helper function used to recover all discovered indices and constraints. The
+// indices and constraints must be recovered after the data recovery is done
+// to ensure that the indices and constraints are consistent at the end of the
+// recovery process.
+/// @throw RecoveryFailure
+void RecoverIndicesAndConstraints(
+    const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
+    Constraints *constraints, utils::SkipList<Vertex> *vertices);
+
 /// Recovers data either from a snapshot and/or WAL files.
 /// @throw RecoveryFailure
 /// @throw std::bad_alloc
 std::optional<RecoveryInfo> RecoverData(
     const std::filesystem::path &snapshot_directory,
     const std::filesystem::path &wal_directory, std::string *uuid,
+    std::string *epoch_id,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
     utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
     std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
     Indices *indices, Constraints *constraints, Config::Items items,

src/storage/v2/durability/marker.hpp

@@ -24,6 +24,7 @@ enum class Marker : uint8_t {
   SECTION_INDICES = 0x24,
   SECTION_CONSTRAINTS = 0x25,
   SECTION_DELTA = 0x26,
+  SECTION_EPOCH_HISTORY = 0x27,
   SECTION_OFFSETS = 0x42,
 
   DELTA_VERTEX_CREATE = 0x50,
@@ -66,6 +67,7 @@ static const Marker kMarkersAll[] = {
     Marker::SECTION_INDICES,
     Marker::SECTION_CONSTRAINTS,
     Marker::SECTION_DELTA,
+    Marker::SECTION_EPOCH_HISTORY,
     Marker::SECTION_OFFSETS,
     Marker::DELTA_VERTEX_CREATE,
     Marker::DELTA_VERTEX_DELETE,

src/storage/v2/durability/serialization.cpp

@@ -24,6 +24,16 @@ void Encoder::Initialize(const std::filesystem::path &path,
         sizeof(version_encoded));
 }
 
+void Encoder::OpenExisting(const std::filesystem::path &path) {
+  file_.Open(path, utils::OutputFile::Mode::APPEND_TO_EXISTING);
+}
+
+void Encoder::Close() {
+  if (file_.IsOpen()) {
+    file_.Close();
+  }
+}
+
 void Encoder::Write(const uint8_t *data, uint64_t size) {
   file_.Write(data, size);
 }
@@ -119,6 +129,18 @@ void Encoder::Finalize() {
   file_.Close();
 }
 
+void Encoder::DisableFlushing() { file_.DisableFlushing(); }
+
+void Encoder::EnableFlushing() { file_.EnableFlushing(); }
+
+void Encoder::TryFlushing() { file_.TryFlushing(); }
+
+std::pair<const uint8_t *, size_t> Encoder::CurrentFileBuffer() const {
+  return file_.CurrentBuffer();
+}
+
+size_t Encoder::GetSize() { return file_.GetSize(); }
+
 //////////////////////////
 // Decoder implementation.
 //////////////////////////
@@ -295,6 +317,7 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::DELTA_VERTEX_CREATE:
     case Marker::DELTA_VERTEX_DELETE:
@@ -390,6 +413,7 @@ bool Decoder::SkipPropertyValue() {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::DELTA_VERTEX_CREATE:
     case Marker::DELTA_VERTEX_DELETE:

src/storage/v2/durability/serialization.hpp

@@ -33,6 +33,9 @@ class Encoder final : public BaseEncoder {
   void Initialize(const std::filesystem::path &path,
                   const std::string_view &magic, uint64_t version);
 
+  void OpenExisting(const std::filesystem::path &path);
+
+  void Close();
   // Main write function, the only one that is allowed to write to the `file_`
   // directly.
   void Write(const uint8_t *data, uint64_t size);
@@ -51,6 +54,18 @@ class Encoder final : public BaseEncoder {
 
   void Finalize();
 
+  // Disable flushing of the internal buffer.
+  void DisableFlushing();
+  // Enable flushing of the internal buffer.
+  void EnableFlushing();
+  // Try flushing the internal buffer.
+  void TryFlushing();
+  // Get the current internal buffer with its size.
+  std::pair<const uint8_t *, size_t> CurrentFileBuffer() const;
+
+  // Get the total size of the current file.
+  size_t GetSize();
+
  private:
   utils::OutputFile file_;
 };

src/storage/v2/durability/snapshot.cpp

@@ -9,6 +9,7 @@
 #include "storage/v2/edge_ref.hpp"
 #include "storage/v2/mvcc.hpp"
 #include "storage/v2/vertex_accessor.hpp"
+#include "utils/file_locker.hpp"
 
 namespace storage::durability {
 
@@ -113,6 +114,7 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
     info.offset_indices = read_offset();
     info.offset_constraints = read_offset();
     info.offset_mapper = read_offset();
+    info.offset_epoch_history = read_offset();
     info.offset_metadata = read_offset();
   }
 
@@ -129,6 +131,10 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
     if (!maybe_uuid) throw RecoveryFailure("Invalid snapshot data!");
     info.uuid = std::move(*maybe_uuid);
 
+    auto maybe_epoch_id = snapshot.ReadString();
+    if (!maybe_epoch_id) throw RecoveryFailure("Invalid snapshot data!");
+    info.epoch_id = std::move(*maybe_epoch_id);
+
     auto maybe_timestamp = snapshot.ReadUint();
     if (!maybe_timestamp) throw RecoveryFailure("Invalid snapshot data!");
     info.start_timestamp = *maybe_timestamp;
@@ -145,12 +151,12 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
   return info;
 }
 
-RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
-                               utils::SkipList<Vertex> *vertices,
-                               utils::SkipList<Edge> *edges,
-                               NameIdMapper *name_id_mapper,
-                               std::atomic<uint64_t> *edge_count,
-                               Config::Items items) {
+RecoveredSnapshot LoadSnapshot(
+    const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
+    NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
+    Config::Items items) {
   RecoveryInfo ret;
   RecoveredIndicesAndConstraints indices_constraints;
 
@@ -159,7 +165,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
   if (!version)
     throw RecoveryFailure("Couldn't read snapshot magic and/or version!");
   if (!IsVersionSupported(*version))
-    throw RecoveryFailure("Invalid snapshot version!");
+    throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version));
 
   // Cleanup of loaded data in case of failure.
   bool success = false;
@@ -167,6 +173,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
     if (!success) {
       edges->clear();
       vertices->clear();
+      epoch_history->clear();
     }
   });
 
@@ -566,6 +573,34 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
     }
   }
 
+  // Recover epoch history
+  {
+    if (!snapshot.SetPosition(info.offset_epoch_history))
+      throw RecoveryFailure("Couldn't read data from snapshot!");
+
+    const auto marker = snapshot.ReadMarker();
+    if (!marker || *marker != Marker::SECTION_EPOCH_HISTORY)
+      throw RecoveryFailure("Invalid snapshot data!");
+
+    const auto history_size = snapshot.ReadUint();
+    if (!history_size) {
+      throw RecoveryFailure("Invalid snapshot data!");
+    }
+
+    for (int i = 0; i < *history_size; ++i) {
+      auto maybe_epoch_id = snapshot.ReadString();
+      if (!maybe_epoch_id) {
+        throw RecoveryFailure("Invalid snapshot data!");
+      }
+      const auto maybe_last_commit_timestamp = snapshot.ReadUint();
+      if (!maybe_last_commit_timestamp) {
+        throw RecoveryFailure("Invalid snapshot data!");
+      }
+      epoch_history->emplace_back(std::move(*maybe_epoch_id),
+                                  *maybe_last_commit_timestamp);
+    }
+  }
+
   // Recover timestamp.
   ret.next_timestamp = info.start_timestamp + 1;
 
@@ -575,14 +610,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
   return {info, ret, std::move(indices_constraints)};
 }
 
-void CreateSnapshot(Transaction *transaction,
-                    const std::filesystem::path &snapshot_directory,
-                    const std::filesystem::path &wal_directory,
-                    uint64_t snapshot_retention_count,
-                    utils::SkipList<Vertex> *vertices,
-                    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
-                    Indices *indices, Constraints *constraints,
-                    Config::Items items, const std::string &uuid) {
+void CreateSnapshot(
+    Transaction *transaction, const std::filesystem::path &snapshot_directory,
+    const std::filesystem::path &wal_directory,
+    uint64_t snapshot_retention_count, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
+    Indices *indices, Constraints *constraints, Config::Items items,
+    const std::string &uuid, const std::string_view epoch_id,
+    const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
+    utils::FileRetainer *file_retainer) {
   // Ensure that the storage directory exists.
   utils::EnsureDirOrDie(snapshot_directory);
 
@@ -601,6 +637,7 @@ void CreateSnapshot(Transaction *transaction,
   uint64_t offset_constraints = 0;
   uint64_t offset_mapper = 0;
   uint64_t offset_metadata = 0;
+  uint64_t offset_epoch_history = 0;
   {
     snapshot.WriteMarker(Marker::SECTION_OFFSETS);
     offset_offsets = snapshot.GetPosition();
@@ -609,6 +646,7 @@ void CreateSnapshot(Transaction *transaction,
     snapshot.WriteUint(offset_indices);
     snapshot.WriteUint(offset_constraints);
     snapshot.WriteUint(offset_mapper);
+    snapshot.WriteUint(offset_epoch_history);
     snapshot.WriteUint(offset_metadata);
   }
 
@@ -812,11 +850,23 @@ void CreateSnapshot(Transaction *transaction,
     }
   }
 
+  // Write epoch history
+  {
+    offset_epoch_history = snapshot.GetPosition();
+    snapshot.WriteMarker(Marker::SECTION_EPOCH_HISTORY);
+    snapshot.WriteUint(epoch_history.size());
+    for (const auto &[epoch_id, last_commit_timestamp] : epoch_history) {
+      snapshot.WriteString(epoch_id);
+      snapshot.WriteUint(last_commit_timestamp);
+    }
+  }
+
   // Write metadata.
   {
     offset_metadata = snapshot.GetPosition();
     snapshot.WriteMarker(Marker::SECTION_METADATA);
     snapshot.WriteString(uuid);
+    snapshot.WriteString(epoch_id);
     snapshot.WriteUint(transaction->start_timestamp);
     snapshot.WriteUint(edges_count);
     snapshot.WriteUint(vertices_count);
@@ -830,6 +880,7 @@ void CreateSnapshot(Transaction *transaction,
     snapshot.WriteUint(offset_indices);
     snapshot.WriteUint(offset_constraints);
     snapshot.WriteUint(offset_mapper);
+    snapshot.WriteUint(offset_epoch_history);
     snapshot.WriteUint(offset_metadata);
   }
 
@@ -865,10 +916,7 @@ void CreateSnapshot(Transaction *transaction,
           old_snapshot_files.size() - (snapshot_retention_count - 1);
       for (size_t i = 0; i < num_to_erase; ++i) {
         const auto &[start_timestamp, snapshot_path] = old_snapshot_files[i];
-        if (!utils::DeleteFile(snapshot_path)) {
-          LOG(WARNING) << "Couldn't delete snapshot file " << snapshot_path
-                       << "!";
-        }
+        file_retainer->DeleteFile(snapshot_path);
       }
       old_snapshot_files.erase(old_snapshot_files.begin(),
                                old_snapshot_files.begin() + num_to_erase);

src/storage/v2/durability/snapshot.hpp

@@ -12,6 +12,7 @@
 #include "storage/v2/name_id_mapper.hpp"
 #include "storage/v2/transaction.hpp"
 #include "storage/v2/vertex.hpp"
+#include "utils/file_locker.hpp"
 #include "utils/skip_list.hpp"
 
 namespace storage::durability {
@@ -23,9 +24,11 @@ struct SnapshotInfo {
   uint64_t offset_indices;
   uint64_t offset_constraints;
   uint64_t offset_mapper;
+  uint64_t offset_epoch_history;
   uint64_t offset_metadata;
 
   std::string uuid;
+  std::string epoch_id;
   uint64_t start_timestamp;
   uint64_t edges_count;
   uint64_t vertices_count;
@@ -45,21 +48,22 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
 
 /// Function used to load the snapshot data into the storage.
 /// @throw RecoveryFailure
-RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path,
-                               utils::SkipList<Vertex> *vertices,
-                               utils::SkipList<Edge> *edges,
-                               NameIdMapper *name_id_mapper,
-                               std::atomic<uint64_t> *edge_count,
-                               Config::Items items);
+RecoveredSnapshot LoadSnapshot(
+    const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges,
+    std::deque<std::pair<std::string, uint64_t>> *epoch_history,
+    NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
+    Config::Items items);
 
 /// Function used to create a snapshot using the given transaction.
-void CreateSnapshot(Transaction *transaction,
-                    const std::filesystem::path &snapshot_directory,
-                    const std::filesystem::path &wal_directory,
-                    uint64_t snapshot_retention_count,
-                    utils::SkipList<Vertex> *vertices,
-                    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
-                    Indices *indices, Constraints *constraints,
-                    Config::Items items, const std::string &uuid);
+void CreateSnapshot(
+    Transaction *transaction, const std::filesystem::path &snapshot_directory,
+    const std::filesystem::path &wal_directory,
+    uint64_t snapshot_retention_count, utils::SkipList<Vertex> *vertices,
+    utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
+    Indices *indices, Constraints *constraints, Config::Items items,
+    const std::string &uuid, std::string_view epoch_id,
+    const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
+    utils::FileRetainer *file_retainer);
 
 }  // namespace storage::durability

src/storage/v2/durability/version.hpp

@@ -9,9 +9,9 @@ namespace storage::durability {
 // The current version of snapshot and WAL encoding / decoding.
 // IMPORTANT: Please bump this version for every snapshot and/or WAL format
 // change!!!
-const uint64_t kVersion{13};
+const uint64_t kVersion{14};
 
-const uint64_t kOldestSupportedVersion{12};
+const uint64_t kOldestSupportedVersion{14};
 const uint64_t kUniqueConstraintVersion{13};
 
 // Magic values written to the start of a snapshot/WAL file to identify it.

src/storage/v2/durability/wal.cpp

@@ -6,6 +6,7 @@
 #include "storage/v2/durability/version.hpp"
 #include "storage/v2/edge.hpp"
 #include "storage/v2/vertex.hpp"
+#include "utils/file_locker.hpp"
 
 namespace storage::durability {
 
@@ -164,6 +165,7 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
     case Marker::SECTION_INDICES:
     case Marker::SECTION_CONSTRAINTS:
     case Marker::SECTION_DELTA:
+    case Marker::SECTION_EPOCH_HISTORY:
     case Marker::SECTION_OFFSETS:
     case Marker::VALUE_FALSE:
     case Marker::VALUE_TRUE:
@@ -171,39 +173,6 @@ WalDeltaData::Type MarkerToWalDeltaDataType(Marker marker) {
   }
 }
 
-bool IsWalDeltaDataTypeTransactionEnd(WalDeltaData::Type type) {
-  switch (type) {
-    // These delta actions are all found inside transactions so they don't
-    // indicate a transaction end.
-    case WalDeltaData::Type::VERTEX_CREATE:
-    case WalDeltaData::Type::VERTEX_DELETE:
-    case WalDeltaData::Type::VERTEX_ADD_LABEL:
-    case WalDeltaData::Type::VERTEX_REMOVE_LABEL:
-    case WalDeltaData::Type::EDGE_CREATE:
-    case WalDeltaData::Type::EDGE_DELETE:
-    case WalDeltaData::Type::VERTEX_SET_PROPERTY:
-    case WalDeltaData::Type::EDGE_SET_PROPERTY:
-      return false;
-
-    // This delta explicitly indicates that a transaction is done.
-    case WalDeltaData::Type::TRANSACTION_END:
-      return true;
-
-    // These operations aren't transactional and they are encoded only using
-    // a single delta, so they each individually mark the end of their
-    // 'transaction'.
-    case WalDeltaData::Type::LABEL_INDEX_CREATE:
-    case WalDeltaData::Type::LABEL_INDEX_DROP:
-    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
-    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
-    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
-    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
-    case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
-    case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
-      return true;
-  }
-}
-
 // Function used to either read or skip the current WAL delta data. The WAL
 // delta header must be read before calling this function. If the delta data is
 // read then the data returned is valid, if the delta data is skipped then the
@@ -385,6 +354,10 @@ WalInfo ReadWalInfo(const std::filesystem::path &path) {
     if (!maybe_uuid) throw RecoveryFailure("Invalid WAL data!");
     info.uuid = std::move(*maybe_uuid);
 
+    auto maybe_epoch_id = wal.ReadString();
+    if (!maybe_epoch_id) throw RecoveryFailure("Invalid WAL data!");
+    info.epoch_id = std::move(*maybe_epoch_id);
+
     auto maybe_seq_num = wal.ReadUint();
     if (!maybe_seq_num) throw RecoveryFailure("Invalid WAL data!");
     info.seq_num = *maybe_seq_num;
@@ -664,7 +637,7 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper,
 
 RecoveryInfo LoadWal(const std::filesystem::path &path,
                      RecoveredIndicesAndConstraints *indices_constraints,
-                     std::optional<uint64_t> snapshot_timestamp,
+                     const std::optional<uint64_t> last_loaded_timestamp,
                      utils::SkipList<Vertex> *vertices,
                      utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
                      std::atomic<uint64_t> *edge_count, Config::Items items) {
@@ -681,7 +654,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
   auto info = ReadWalInfo(path);
 
   // Check timestamp.
-  if (snapshot_timestamp && info.to_timestamp <= *snapshot_timestamp)
+  if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp)
     return ret;
 
   // Recover deltas.
@@ -693,7 +666,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
     // Read WAL delta header to find out the delta timestamp.
     auto timestamp = ReadWalDeltaHeader(&wal);
 
-    if (!snapshot_timestamp || timestamp > *snapshot_timestamp) {
+    if (!last_loaded_timestamp || timestamp > *last_loaded_timestamp) {
       // This delta should be loaded.
       auto delta = ReadWalDeltaData(&wal);
       switch (delta.type) {
@@ -969,14 +942,17 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
 }
 
 WalFile::WalFile(const std::filesystem::path &wal_directory,
-                 const std::string &uuid, Config::Items items,
-                 NameIdMapper *name_id_mapper, uint64_t seq_num)
+                 const std::string_view uuid, const std::string_view epoch_id,
+                 Config::Items items, NameIdMapper *name_id_mapper,
+                 uint64_t seq_num, utils::FileRetainer *file_retainer)
     : items_(items),
       name_id_mapper_(name_id_mapper),
       path_(wal_directory / MakeWalName()),
       from_timestamp_(0),
       to_timestamp_(0),
-      count_(0) {
+      count_(0),
+      seq_num_(seq_num),
+      file_retainer_(file_retainer) {
   // Ensure that the storage directory exists.
   utils::EnsureDirOrDie(wal_directory);
 
@@ -996,6 +972,7 @@ WalFile::WalFile(const std::filesystem::path &wal_directory,
   offset_metadata = wal_.GetPosition();
   wal_.WriteMarker(Marker::SECTION_METADATA);
   wal_.WriteString(uuid);
+  wal_.WriteString(epoch_id);
   wal_.WriteUint(seq_num);
 
   // Write final offsets.
@@ -1009,20 +986,43 @@ WalFile::WalFile(const std::filesystem::path &wal_directory,
   wal_.Sync();
 }
 
-WalFile::~WalFile() {
-  if (count_ != 0) {
-    // Finalize file.
-    wal_.Finalize();
+WalFile::WalFile(std::filesystem::path current_wal_path, Config::Items items,
+                 NameIdMapper *name_id_mapper, uint64_t seq_num,
+                 uint64_t from_timestamp, uint64_t to_timestamp, uint64_t count,
+                 utils::FileRetainer *file_retainer)
+    : items_(items),
+      name_id_mapper_(name_id_mapper),
+      path_(std::move(current_wal_path)),
+      from_timestamp_(from_timestamp),
+      to_timestamp_(to_timestamp),
+      count_(count),
+      seq_num_(seq_num),
+      file_retainer_(file_retainer) {
+  wal_.OpenExisting(path_);
+}
 
+void WalFile::FinalizeWal() {
+  if (count_ != 0) {
+    wal_.Finalize();
     // Rename file.
     std::filesystem::path new_path(path_);
     new_path.replace_filename(
         RemakeWalName(path_.filename(), from_timestamp_, to_timestamp_));
-    // If the rename fails it isn't a crucial situation. The renaming is done
-    // only to make the directory structure of the WAL files easier to read
-    // manually.
-    utils::RenamePath(path_, new_path);
-  } else {
+
+    utils::CopyFile(path_, new_path);
+    wal_.Close();
+    file_retainer_->DeleteFile(path_);
+    path_ = std::move(new_path);
+  }
+}
+
+void WalFile::DeleteWal() {
+  wal_.Close();
+  file_retainer_->DeleteFile(path_);
+}
+
+WalFile::~WalFile() {
+  if (count_ == 0) {
     // Remove empty WAL file.
     utils::DeleteFile(path_);
   }
@@ -1055,7 +1055,9 @@ void WalFile::AppendOperation(StorageGlobalOperation operation, LabelId label,
 
 void WalFile::Sync() { wal_.Sync(); }
 
-uint64_t WalFile::GetSize() { return wal_.GetPosition(); }
+uint64_t WalFile::GetSize() { return wal_.GetSize(); }
+
+uint64_t WalFile::SequenceNumber() const { return seq_num_; }
 
 void WalFile::UpdateStats(uint64_t timestamp) {
   if (count_ == 0) from_timestamp_ = timestamp;
@@ -1063,4 +1065,14 @@ void WalFile::UpdateStats(uint64_t timestamp) {
   count_ += 1;
 }
 
+void WalFile::DisableFlushing() { wal_.DisableFlushing(); }
+
+void WalFile::EnableFlushing() { wal_.EnableFlushing(); }
+
+void WalFile::TryFlushing() { wal_.TryFlushing(); }
+
+std::pair<const uint8_t *, size_t> WalFile::CurrentFileBuffer() const {
+  return wal_.CurrentFileBuffer();
+}
+
 }  // namespace storage::durability

src/storage/v2/durability/wal.hpp

@@ -14,6 +14,7 @@
 #include "storage/v2/name_id_mapper.hpp"
 #include "storage/v2/property_value.hpp"
 #include "storage/v2/vertex.hpp"
+#include "utils/file_locker.hpp"
 #include "utils/skip_list.hpp"
 
 namespace storage::durability {
@@ -24,6 +25,7 @@ struct WalInfo {
   uint64_t offset_deltas;
 
   std::string uuid;
+  std::string epoch_id;
   uint64_t seq_num;
   uint64_t from_timestamp;
   uint64_t to_timestamp;
@@ -106,6 +108,39 @@ enum class StorageGlobalOperation {
   UNIQUE_CONSTRAINT_DROP,
 };
 
+constexpr bool IsWalDeltaDataTypeTransactionEnd(const WalDeltaData::Type type) {
+  switch (type) {
+    // These delta actions are all found inside transactions so they don't
+    // indicate a transaction end.
+    case WalDeltaData::Type::VERTEX_CREATE:
+    case WalDeltaData::Type::VERTEX_DELETE:
+    case WalDeltaData::Type::VERTEX_ADD_LABEL:
+    case WalDeltaData::Type::VERTEX_REMOVE_LABEL:
+    case WalDeltaData::Type::EDGE_CREATE:
+    case WalDeltaData::Type::EDGE_DELETE:
+    case WalDeltaData::Type::VERTEX_SET_PROPERTY:
+    case WalDeltaData::Type::EDGE_SET_PROPERTY:
+      return false;
+
+    // This delta explicitly indicates that a transaction is done.
+    case WalDeltaData::Type::TRANSACTION_END:
+      return true;
+
+    // These operations aren't transactional and they are encoded only using
+    // a single delta, so they each individually mark the end of their
+    // 'transaction'.
+    case WalDeltaData::Type::LABEL_INDEX_CREATE:
+    case WalDeltaData::Type::LABEL_INDEX_DROP:
+    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE:
+    case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP:
+    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE:
+    case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP:
+    case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE:
+    case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP:
+      return true;
+  }
+}
+
 /// Function used to read information about the WAL file.
 /// @throw RecoveryFailure
 WalInfo ReadWalInfo(const std::filesystem::path &path);
@@ -149,7 +184,7 @@ void EncodeOperation(BaseEncoder *encoder, NameIdMapper *name_id_mapper,
 /// @throw RecoveryFailure
 RecoveryInfo LoadWal(const std::filesystem::path &path,
                      RecoveredIndicesAndConstraints *indices_constraints,
-                     std::optional<uint64_t> snapshot_timestamp,
+                     std::optional<uint64_t> last_loaded_timestamp,
                      utils::SkipList<Vertex> *vertices,
                      utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
                      std::atomic<uint64_t> *edge_count, Config::Items items);
@@ -157,8 +192,14 @@ RecoveryInfo LoadWal(const std::filesystem::path &path,
 /// WalFile class used to append deltas and operations to the WAL file.
 class WalFile {
  public:
-  WalFile(const std::filesystem::path &wal_directory, const std::string &uuid,
-          Config::Items items, NameIdMapper *name_id_mapper, uint64_t seq_num);
+  WalFile(const std::filesystem::path &wal_directory, std::string_view uuid,
+          std::string_view epoch_id, Config::Items items,
+          NameIdMapper *name_id_mapper, uint64_t seq_num,
+          utils::FileRetainer *file_retainer);
+  WalFile(std::filesystem::path current_wal_path, Config::Items items,
+          NameIdMapper *name_id_mapper, uint64_t seq_num,
+          uint64_t from_timestamp, uint64_t to_timestamp, uint64_t count,
+          utils::FileRetainer *file_retainer);
 
   WalFile(const WalFile &) = delete;
   WalFile(WalFile &&) = delete;
@@ -181,6 +222,29 @@ class WalFile {
 
   uint64_t GetSize();
 
+  uint64_t SequenceNumber() const;
+
+  auto FromTimestamp() const { return from_timestamp_; }
+
+  auto ToTimestamp() const { return to_timestamp_; }
+
+  auto Count() const { return count_; }
+
+  // Disable flushing of the internal buffer.
+  void DisableFlushing();
+  // Enable flushing of the internal buffer.
+  void EnableFlushing();
+  // Try flushing the internal buffer.
+  void TryFlushing();
+  // Get the internal buffer with its size.
+  std::pair<const uint8_t *, size_t> CurrentFileBuffer() const;
+
+  // Get the path of the current WAL file.
+  const auto &Path() const { return path_; }
+
+  void FinalizeWal();
+  void DeleteWal();
+
  private:
   void UpdateStats(uint64_t timestamp);
 
@@ -191,6 +255,9 @@ class WalFile {
   uint64_t from_timestamp_;
   uint64_t to_timestamp_;
   uint64_t count_;
+  uint64_t seq_num_;
+
+  utils::FileRetainer *file_retainer_;
 };
 
 }  // namespace storage::durability

src/storage/v2/property_value.hpp

@@ -25,7 +25,15 @@ class PropertyValueException : public utils::BasicException {
 class PropertyValue {
  public:
   /// A value type, each type corresponds to exactly one C++ type.
-  enum class Type : unsigned char { Null, Bool, Int, Double, String, List, Map };
+  enum class Type : uint8_t {
+    Null = 0,
+    Bool = 1,
+    Int = 2,
+    Double = 3,
+    String = 4,
+    List = 5,
+    Map = 6,
+  };
 
   static bool AreComparableTypes(Type a, Type b) {
     return (a == b) || (a == Type::Int && b == Type::Double) ||

src/storage/v2/replication/.gitignore

@@ -0,0 +1,2 @@
+# autogenerated files
+rpc.hpp

src/storage/v2/replication/config.hpp

@@ -0,0 +1,27 @@
+#pragma once
+#include <optional>
+#include <string>
+
+namespace storage::replication {
+struct ReplicationClientConfig {
+  std::optional<double> timeout;
+
+  struct SSL {
+    std::string key_file = "";
+    std::string cert_file = "";
+  };
+
+  std::optional<SSL> ssl;
+};
+
+struct ReplicationServerConfig {
+  struct SSL {
+    std::string key_file;
+    std::string cert_file;
+    std::string ca_file;
+    bool verify_peer;
+  };
+
+  std::optional<SSL> ssl;
+};
+}  // namespace storage::replication

src/storage/v2/replication/enums.hpp

@@ -0,0 +1,13 @@
+#pragma once
+#include <cstdint>
+
+namespace storage::replication {
+enum class ReplicationMode : std::uint8_t { SYNC, ASYNC };
+
+enum class ReplicaState : std::uint8_t {
+  READY,
+  REPLICATING,
+  RECOVERY,
+  INVALID
+};
+}  // namespace storage::replication

src/storage/v2/replication/replication_client.cpp

@@ -0,0 +1,621 @@
+#include "storage/v2/replication/replication_client.hpp"
+
+#include <algorithm>
+#include <type_traits>
+
+#include "storage/v2/durability/durability.hpp"
+#include "storage/v2/replication/config.hpp"
+#include "storage/v2/replication/enums.hpp"
+#include "utils/file_locker.hpp"
+
+namespace storage {
+
+namespace {
+template <typename>
+[[maybe_unused]] inline constexpr bool always_false_v = false;
+}  // namespace
+
+////// ReplicationClient //////
+Storage::ReplicationClient::ReplicationClient(
+    std::string name, Storage *storage, const io::network::Endpoint &endpoint,
+    const replication::ReplicationMode mode,
+    const replication::ReplicationClientConfig &config)
+    : name_(std::move(name)), storage_(storage), mode_(mode) {
+  if (config.ssl) {
+    rpc_context_.emplace(config.ssl->key_file, config.ssl->cert_file);
+  } else {
+    rpc_context_.emplace();
+  }
+
+  rpc_client_.emplace(endpoint, &*rpc_context_);
+  TryInitializeClient();
+
+  if (config.timeout && replica_state_ != replication::ReplicaState::INVALID) {
+    timeout_.emplace(*config.timeout);
+    timeout_dispatcher_.emplace();
+  }
+}
+
+/// @throws rpc::RpcFailedException
+void Storage::ReplicationClient::InitializeClient() {
+  uint64_t current_commit_timestamp{kTimestampInitialId};
+  auto stream{
+      rpc_client_->Stream<HeartbeatRpc>(storage_->last_commit_timestamp_)};
+  replication::Encoder encoder{stream.GetBuilder()};
+  // Write epoch id
+  {
+    // We need to lock so the epoch id isn't overwritten
+    std::unique_lock engine_guard{storage_->engine_lock_};
+    encoder.WriteString(storage_->epoch_id_);
+  }
+  const auto response = stream.AwaitResponse();
+  if (!response.success) {
+    LOG(ERROR)
+        << "Replica " << name_
+        << " is ahead of this instance. The branching point is on commit "
+        << response.current_commit_timestamp;
+    return;
+  }
+  current_commit_timestamp = response.current_commit_timestamp;
+  DLOG(INFO) << "Current timestamp on replica: " << current_commit_timestamp;
+  DLOG(INFO) << "Current MAIN timestamp: "
+             << storage_->last_commit_timestamp_.load();
+  if (current_commit_timestamp == storage_->last_commit_timestamp_.load()) {
+    DLOG(INFO) << "Replica up to date";
+    std::unique_lock client_guard{client_lock_};
+    replica_state_.store(replication::ReplicaState::READY);
+  } else {
+    DLOG(INFO) << "Replica is behind";
+    {
+      std::unique_lock client_guard{client_lock_};
+      replica_state_.store(replication::ReplicaState::RECOVERY);
+    }
+    thread_pool_.AddTask(
+        [=, this] { this->RecoverReplica(current_commit_timestamp); });
+  }
+}
+
+void Storage::ReplicationClient::TryInitializeClient() {
+  try {
+    InitializeClient();
+  } catch (const rpc::RpcFailedException &) {
+    std::unique_lock client_guarde{client_lock_};
+    replica_state_.store(replication::ReplicaState::INVALID);
+    LOG(ERROR) << "Failed to connect to replica " << name_ << " at "
+               << rpc_client_->Endpoint();
+  }
+}
+
+void Storage::ReplicationClient::HandleRpcFailure() {
+  LOG(ERROR) << "Couldn't replicate data to " << name_;
+  thread_pool_.AddTask([this] {
+    rpc_client_->Abort();
+    this->TryInitializeClient();
+  });
+}
+
+SnapshotRes Storage::ReplicationClient::TransferSnapshot(
+    const std::filesystem::path &path) {
+  auto stream{rpc_client_->Stream<SnapshotRpc>()};
+  replication::Encoder encoder(stream.GetBuilder());
+  encoder.WriteFile(path);
+  return stream.AwaitResponse();
+}
+
+WalFilesRes Storage::ReplicationClient::TransferWalFiles(
+    const std::vector<std::filesystem::path> &wal_files) {
+  CHECK(!wal_files.empty()) << "Wal files list is empty!";
+  auto stream{rpc_client_->Stream<WalFilesRpc>(wal_files.size())};
+  replication::Encoder encoder(stream.GetBuilder());
+  for (const auto &wal : wal_files) {
+    DLOG(INFO) << "Sending wal file: " << wal;
+    encoder.WriteFile(wal);
+  }
+
+  return stream.AwaitResponse();
+}
+
+OnlySnapshotRes Storage::ReplicationClient::TransferOnlySnapshot(
+    const uint64_t snapshot_timestamp) {
+  auto stream{rpc_client_->Stream<OnlySnapshotRpc>(snapshot_timestamp)};
+  replication::Encoder encoder{stream.GetBuilder()};
+  encoder.WriteString(storage_->epoch_id_);
+  return stream.AwaitResponse();
+}
+
+void Storage::ReplicationClient::StartTransactionReplication(
+    const uint64_t current_wal_seq_num) {
+  std::unique_lock guard(client_lock_);
+  const auto status = replica_state_.load();
+  switch (status) {
+    case replication::ReplicaState::RECOVERY:
+      DLOG(INFO) << "Replica " << name_ << " is behind MAIN instance";
+      return;
+    case replication::ReplicaState::REPLICATING:
+      DLOG(INFO) << "Replica " << name_ << " missed a transaction";
+      // We missed a transaction because we're still replicating
+      // the previous transaction so we need to go to RECOVERY
+      // state to catch up with the missing transaction
+      // We cannot queue the recovery process here because
+      // an error can happen while we're replicating the previous
+      // transaction after which the client should go to
+      // INVALID state before starting the recovery process
+      replica_state_.store(replication::ReplicaState::RECOVERY);
+      return;
+    case replication::ReplicaState::INVALID:
+      HandleRpcFailure();
+      return;
+    case replication::ReplicaState::READY:
+      CHECK(!replica_stream_);
+      try {
+        replica_stream_.emplace(
+            ReplicaStream{this, storage_->last_commit_timestamp_.load(),
+                          current_wal_seq_num});
+        replica_state_.store(replication::ReplicaState::REPLICATING);
+      } catch (const rpc::RpcFailedException &) {
+        replica_state_.store(replication::ReplicaState::INVALID);
+        HandleRpcFailure();
+      }
+      return;
+  }
+}
+
+void Storage::ReplicationClient::IfStreamingTransaction(
+    const std::function<void(ReplicaStream &handler)> &callback) {
+  // We can only check the state because it guarantees to be only
+  // valid during a single transaction replication (if the assumption
+  // that this and other transaction replication functions can only be
+  // called from a one thread stands)
+  if (replica_state_ != replication::ReplicaState::REPLICATING) {
+    return;
+  }
+
+  try {
+    callback(*replica_stream_);
+  } catch (const rpc::RpcFailedException &) {
+    {
+      std::unique_lock client_guard{client_lock_};
+      replica_state_.store(replication::ReplicaState::INVALID);
+    }
+    HandleRpcFailure();
+  }
+}
+
+void Storage::ReplicationClient::FinalizeTransactionReplication() {
+  // We can only check the state because it guarantees to be only
+  // valid during a single transaction replication (if the assumption
+  // that this and other transaction replication functions can only be
+  // called from a one thread stands)
+  if (replica_state_ != replication::ReplicaState::REPLICATING) {
+    return;
+  }
+
+  if (mode_ == replication::ReplicationMode::ASYNC) {
+    thread_pool_.AddTask(
+        [this] { this->FinalizeTransactionReplicationInternal(); });
+  } else if (timeout_) {
+    CHECK(mode_ == replication::ReplicationMode::SYNC)
+        << "Only SYNC replica can have a timeout.";
+    CHECK(timeout_dispatcher_) << "Timeout thread is missing";
+    timeout_dispatcher_->WaitForTaskToFinish();
+
+    timeout_dispatcher_->active = true;
+    thread_pool_.AddTask([&, this] {
+      this->FinalizeTransactionReplicationInternal();
+      std::unique_lock main_guard(timeout_dispatcher_->main_lock);
+      // TimerThread can finish waiting for timeout
+      timeout_dispatcher_->active = false;
+      // Notify the main thread
+      timeout_dispatcher_->main_cv.notify_one();
+    });
+
+    timeout_dispatcher_->StartTimeoutTask(*timeout_);
+
+    // Wait until one of the threads notifies us that they finished executing
+    // Both threads should first set the active flag to false
+    {
+      std::unique_lock main_guard(timeout_dispatcher_->main_lock);
+      timeout_dispatcher_->main_cv.wait(
+          main_guard, [&] { return !timeout_dispatcher_->active.load(); });
+    }
+
+    // TODO (antonio2368): Document and/or polish SEMI-SYNC to ASYNC fallback.
+    if (replica_state_ == replication::ReplicaState::REPLICATING) {
+      mode_ = replication::ReplicationMode::ASYNC;
+      timeout_.reset();
+      // This can only happen if we timeouted so we are sure that
+      // Timeout task finished
+      // We need to delete timeout dispatcher AFTER the replication
+      // finished because it tries to acquire the timeout lock
+      // and acces the `active` variable`
+      thread_pool_.AddTask([this] { timeout_dispatcher_.reset(); });
+    }
+  } else {
+    FinalizeTransactionReplicationInternal();
+  }
+}
+
+void Storage::ReplicationClient::FinalizeTransactionReplicationInternal() {
+  CHECK(replica_stream_) << "Missing stream for transaction deltas";
+  try {
+    auto response = replica_stream_->Finalize();
+    replica_stream_.reset();
+    std::unique_lock client_guard(client_lock_);
+    if (!response.success ||
+        replica_state_ == replication::ReplicaState::RECOVERY) {
+      replica_state_.store(replication::ReplicaState::RECOVERY);
+      thread_pool_.AddTask([&, this] {
+        this->RecoverReplica(response.current_commit_timestamp);
+      });
+    } else {
+      replica_state_.store(replication::ReplicaState::READY);
+    }
+  } catch (const rpc::RpcFailedException &) {
+    replica_stream_.reset();
+    {
+      std::unique_lock client_guard(client_lock_);
+      replica_state_.store(replication::ReplicaState::INVALID);
+    }
+    HandleRpcFailure();
+  }
+}
+
+void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
+  while (true) {
+    auto file_locker = storage_->file_retainer_.AddLocker();
+
+    const auto steps = GetRecoverySteps(replica_commit, &file_locker);
+    for (const auto &recovery_step : steps) {
+      try {
+        std::visit(
+            [&, this]<typename T>(T &&arg) {
+              using StepType = std::remove_cvref_t<T>;
+              if constexpr (std::is_same_v<StepType, RecoverySnapshot>) {
+                DLOG(INFO) << "Sending the latest snapshot file: " << arg;
+                auto response = TransferSnapshot(arg);
+                replica_commit = response.current_commit_timestamp;
+                DLOG(INFO) << "Current timestamp on replica: "
+                           << replica_commit;
+              } else if constexpr (std::is_same_v<StepType, RecoveryWals>) {
+                DLOG(INFO) << "Sending the latest wal files";
+                auto response = TransferWalFiles(arg);
+                replica_commit = response.current_commit_timestamp;
+                DLOG(INFO) << "Current timestamp on replica: "
+                           << replica_commit;
+              } else if constexpr (std::is_same_v<StepType,
+                                                  RecoveryCurrentWal>) {
+                std::unique_lock transaction_guard(storage_->engine_lock_);
+                if (storage_->wal_file_ &&
+                    storage_->wal_file_->SequenceNumber() ==
+                        arg.current_wal_seq_num) {
+                  storage_->wal_file_->DisableFlushing();
+                  transaction_guard.unlock();
+                  DLOG(INFO) << "Sending current wal file";
+                  replica_commit = ReplicateCurrentWal();
+                  DLOG(INFO)
+                      << "Current timestamp on replica: " << replica_commit;
+                  storage_->wal_file_->EnableFlushing();
+                }
+              } else if constexpr (std::is_same_v<StepType,
+                                                  RecoveryFinalSnapshot>) {
+                DLOG(INFO) << "Snapshot timestamp is the latest";
+                auto response = TransferOnlySnapshot(arg.snapshot_timestamp);
+                if (response.success) {
+                  replica_commit = response.current_commit_timestamp;
+                }
+              } else {
+                static_assert(always_false_v<T>,
+                              "Missing type from variant visitor");
+              }
+            },
+            recovery_step);
+      } catch (const rpc::RpcFailedException &) {
+        {
+          std::unique_lock client_guard{client_lock_};
+          replica_state_.store(replication::ReplicaState::INVALID);
+        }
+        HandleRpcFailure();
+      }
+    }
+
+    // To avoid the situation where we read a correct commit timestamp in
+    // one thread, and after that another thread commits a different a
+    // transaction and THEN we set the state to READY in the first thread,
+    // we set this lock before checking the timestamp.
+    // We will detect that the state is invalid during the next commit,
+    // because AppendDeltasRpc sends the last commit timestamp which
+    // replica checks if it's the same last commit timestamp it received
+    // and we will go to recovery.
+    // By adding this lock, we can avoid that, and go to RECOVERY immediately.
+    std::unique_lock client_guard{client_lock_};
+    if (storage_->last_commit_timestamp_.load() == replica_commit) {
+      replica_state_.store(replication::ReplicaState::READY);
+      return;
+    }
+  }
+}
+
+uint64_t Storage::ReplicationClient::ReplicateCurrentWal() {
+  auto stream = TransferCurrentWalFile();
+  stream.AppendFilename(storage_->wal_file_->Path().filename());
+  utils::InputFile file;
+  CHECK(file.Open(storage_->wal_file_->Path()))
+      << "Failed to open current WAL file!";
+  const auto [buffer, buffer_size] = storage_->wal_file_->CurrentFileBuffer();
+  stream.AppendSize(file.GetSize() + buffer_size);
+  stream.AppendFileData(&file);
+  stream.AppendBufferData(buffer, buffer_size);
+  auto response = stream.Finalize();
+  return response.current_commit_timestamp;
+}
+
+/// This method tries to find the optimal path for recoverying a single replica.
+/// Based on the last commit transfered to replica it tries to update the
+/// replica using durability files - WALs and Snapshots. WAL files are much
+/// smaller in size as they contain only the Deltas (changes) made during the
+/// transactions while Snapshots contain all the data. For that reason we prefer
+/// WALs as much as possible. As the WAL file that is currently being updated
+/// can change during the process we ignore it as much as possible. Also, it
+/// uses the transaction lock so lokcing it can be really expensive. After we
+/// fetch the list of finalized WALs, we try to find the longest chain of
+/// sequential WALs, starting from the latest one, that will update the recovery
+/// with the all missed updates. If the WAL chain cannot be created, replica is
+/// behind by a lot, so we use the regular recovery process, we send the latest
+/// snapshot and all the necessary WAL files, starting from the newest WAL that
+/// contains a timestamp before the snapshot. If we registered the existence of
+/// the current WAL, we add the sequence number we read from it to the recovery
+/// process. After all the other steps are finished, if the current WAL contains
+/// the same sequence number, it's the same WAL we read while fetching the
+/// recovery steps, so we can safely send it to the replica. There's also one
+/// edge case, if MAIN instance restarted and the snapshot contained the last
+/// change (creation of that snapshot) the latest timestamp is contained in it.
+/// As no changes were made to the data, we only need to send the timestamp of
+/// the snapshot so replica can set its last timestamp to that value.
+std::vector<Storage::ReplicationClient::RecoveryStep>
+Storage::ReplicationClient::GetRecoverySteps(
+    const uint64_t replica_commit,
+    utils::FileRetainer::FileLocker *file_locker) {
+  // First check if we can recover using the current wal file only
+  // otherwise save the seq_num of the current wal file
+  // This lock is also necessary to force the missed transaction to finish.
+  std::optional<uint64_t> current_wal_seq_num;
+  if (std::unique_lock transtacion_guard(storage_->engine_lock_);
+      storage_->wal_file_) {
+    current_wal_seq_num.emplace(storage_->wal_file_->SequenceNumber());
+  }
+
+  auto locker_acc = file_locker->Access();
+  auto wal_files = durability::GetWalFiles(
+      storage_->wal_directory_, storage_->uuid_, current_wal_seq_num);
+  CHECK(wal_files) << "Wal files could not be loaded";
+
+  auto snapshot_files = durability::GetSnapshotFiles(
+      storage_->snapshot_directory_, storage_->uuid_);
+  std::optional<durability::SnapshotDurabilityInfo> latest_snapshot;
+  if (!snapshot_files.empty()) {
+    std::sort(snapshot_files.begin(), snapshot_files.end());
+    latest_snapshot.emplace(std::move(snapshot_files.back()));
+  }
+
+  std::vector<RecoveryStep> recovery_steps;
+
+  // No finalized WAL files were found. This means the difference is contained
+  // inside the current WAL or the snapshot was loaded back without any WALs
+  // after.
+  if (wal_files->empty()) {
+    if (current_wal_seq_num) {
+      recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num});
+    } else {
+      CHECK(latest_snapshot);
+      locker_acc.AddFile(latest_snapshot->path);
+      recovery_steps.emplace_back(
+          RecoveryFinalSnapshot{latest_snapshot->start_timestamp});
+    }
+    return recovery_steps;
+  }
+
+  // Find the longest chain of WALs for recovery.
+  // The chain consists ONLY of sequential WALs.
+  auto rwal_it = wal_files->rbegin();
+
+  // if the last finalized WAL is before the replica commit
+  // then we can recovery only from current WAL or from snapshot
+  // if the main just recovered
+  if (rwal_it->to_timestamp <= replica_commit) {
+    if (current_wal_seq_num) {
+      recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num});
+    } else {
+      CHECK(latest_snapshot);
+      locker_acc.AddFile(latest_snapshot->path);
+      recovery_steps.emplace_back(
+          RecoveryFinalSnapshot{latest_snapshot->start_timestamp});
+    }
+    return recovery_steps;
+  }
+
+  uint64_t previous_seq_num{rwal_it->seq_num};
+  for (; rwal_it != wal_files->rend(); ++rwal_it) {
+    // If the difference between two consecutive wal files is not 0 or 1
+    // we have a missing WAL in our chain
+    if (previous_seq_num - rwal_it->seq_num > 1) {
+      break;
+    }
+
+    // Find first WAL that contains up to replica commit, i.e. WAL
+    // that is before the replica commit or conatins the replica commit
+    // as the last committed transaction OR we managed to find the first WAL
+    // file.
+    if (replica_commit >= rwal_it->from_timestamp || rwal_it->seq_num == 0) {
+      if (replica_commit >= rwal_it->to_timestamp) {
+        // We want the WAL after because the replica already contains all the
+        // commits from this WAL
+        --rwal_it;
+      }
+      std::vector<std::filesystem::path> wal_chain;
+      auto distance_from_first = std::distance(rwal_it, wal_files->rend() - 1);
+      // We have managed to create WAL chain
+      // We need to lock these files and add them to the chain
+      for (auto result_wal_it = wal_files->begin() + distance_from_first;
+           result_wal_it != wal_files->end(); ++result_wal_it) {
+        locker_acc.AddFile(result_wal_it->path);
+        wal_chain.push_back(std::move(result_wal_it->path));
+      }
+
+      recovery_steps.emplace_back(std::in_place_type_t<RecoveryWals>{},
+                                  std::move(wal_chain));
+
+      if (current_wal_seq_num) {
+        recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num});
+      }
+      return recovery_steps;
+    }
+
+    previous_seq_num = rwal_it->seq_num;
+  }
+
+  CHECK(latest_snapshot) << "Invalid durability state, missing snapshot";
+  // We didn't manage to find a WAL chain, we need to send the latest snapshot
+  // with its WALs
+  locker_acc.AddFile(latest_snapshot->path);
+  recovery_steps.emplace_back(std::in_place_type_t<RecoverySnapshot>{},
+                              std::move(latest_snapshot->path));
+
+  std::vector<std::filesystem::path> recovery_wal_files;
+  auto wal_it = wal_files->begin();
+  for (; wal_it != wal_files->end(); ++wal_it) {
+    // Assuming recovery process is correct the snashpot should
+    // always retain a single WAL that contains a transaction
+    // before its creation
+    if (latest_snapshot->start_timestamp < wal_it->to_timestamp) {
+      if (latest_snapshot->start_timestamp < wal_it->from_timestamp) {
+        CHECK(wal_it != wal_files->begin()) << "Invalid durability files state";
+        --wal_it;
+      }
+      break;
+    }
+  }
+
+  for (; wal_it != wal_files->end(); ++wal_it) {
+    locker_acc.AddFile(wal_it->path);
+    recovery_wal_files.push_back(std::move(wal_it->path));
+  }
+
+  // We only have a WAL before the snapshot
+  if (recovery_wal_files.empty()) {
+    locker_acc.AddFile(wal_files->back().path);
+    recovery_wal_files.push_back(std::move(wal_files->back().path));
+  }
+
+  recovery_steps.emplace_back(std::in_place_type_t<RecoveryWals>{},
+                              std::move(recovery_wal_files));
+
+  if (current_wal_seq_num) {
+    recovery_steps.emplace_back(RecoveryCurrentWal{*current_wal_seq_num});
+  }
+
+  return recovery_steps;
+}
+
+////// TimeoutDispatcher //////
+void Storage::ReplicationClient::TimeoutDispatcher::WaitForTaskToFinish() {
+  // Wait for the previous timeout task to finish
+  std::unique_lock main_guard(main_lock);
+  main_cv.wait(main_guard, [&] { return finished; });
+}
+
+void Storage::ReplicationClient::TimeoutDispatcher::StartTimeoutTask(
+    const double timeout) {
+  timeout_pool.AddTask([timeout, this] {
+    finished = false;
+    using std::chrono::steady_clock;
+    const auto timeout_duration =
+        std::chrono::duration_cast<steady_clock::duration>(
+            std::chrono::duration<double>(timeout));
+    const auto end_time = steady_clock::now() + timeout_duration;
+    while (active && (steady_clock::now() < end_time)) {
+      std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+
+    std::unique_lock main_guard(main_lock);
+    finished = true;
+    active = false;
+    main_cv.notify_one();
+  });
+}
+////// ReplicaStream //////
+Storage::ReplicationClient::ReplicaStream::ReplicaStream(
+    ReplicationClient *self, const uint64_t previous_commit_timestamp,
+    const uint64_t current_seq_num)
+    : self_(self),
+      stream_(self_->rpc_client_->Stream<AppendDeltasRpc>(
+          previous_commit_timestamp, current_seq_num)) {
+  replication::Encoder encoder{stream_.GetBuilder()};
+  encoder.WriteString(self_->storage_->epoch_id_);
+}
+
+void Storage::ReplicationClient::ReplicaStream::AppendDelta(
+    const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeDelta(&encoder, &self_->storage_->name_id_mapper_,
+              self_->storage_->config_.items, delta, vertex,
+              final_commit_timestamp);
+}
+
+void Storage::ReplicationClient::ReplicaStream::AppendDelta(
+    const Delta &delta, const Edge &edge, uint64_t final_commit_timestamp) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, delta, edge,
+              final_commit_timestamp);
+}
+
+void Storage::ReplicationClient::ReplicaStream::AppendTransactionEnd(
+    uint64_t final_commit_timestamp) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeTransactionEnd(&encoder, final_commit_timestamp);
+}
+
+void Storage::ReplicationClient::ReplicaStream::AppendOperation(
+    durability::StorageGlobalOperation operation, LabelId label,
+    const std::set<PropertyId> &properties, uint64_t timestamp) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  EncodeOperation(&encoder, &self_->storage_->name_id_mapper_, operation, label,
+                  properties, timestamp);
+}
+
+AppendDeltasRes Storage::ReplicationClient::ReplicaStream::Finalize() {
+  return stream_.AwaitResponse();
+}
+
+////// CurrentWalHandler //////
+Storage::ReplicationClient::CurrentWalHandler::CurrentWalHandler(
+    ReplicationClient *self)
+    : self_(self), stream_(self_->rpc_client_->Stream<CurrentWalRpc>()) {}
+
+void Storage::ReplicationClient::CurrentWalHandler::AppendFilename(
+    const std::string &filename) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  encoder.WriteString(filename);
+}
+
+void Storage::ReplicationClient::CurrentWalHandler::AppendSize(
+    const size_t size) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  encoder.WriteUint(size);
+}
+
+void Storage::ReplicationClient::CurrentWalHandler::AppendFileData(
+    utils::InputFile *file) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  encoder.WriteFileData(file);
+}
+
+void Storage::ReplicationClient::CurrentWalHandler::AppendBufferData(
+    const uint8_t *buffer, const size_t buffer_size) {
+  replication::Encoder encoder(stream_.GetBuilder());
+  encoder.WriteBuffer(buffer, buffer_size);
+}
+
+CurrentWalRes Storage::ReplicationClient::CurrentWalHandler::Finalize() {
+  return stream_.AwaitResponse();
+}
+}  // namespace storage

src/storage/v2/replication/replication_client.hpp

@@ -0,0 +1,215 @@
+#pragma once
+
+#include <atomic>
+#include <chrono>
+#include <thread>
+#include <variant>
+
+#include "rpc/client.hpp"
+#include "storage/v2/config.hpp"
+#include "storage/v2/delta.hpp"
+#include "storage/v2/durability/wal.hpp"
+#include "storage/v2/id_types.hpp"
+#include "storage/v2/mvcc.hpp"
+#include "storage/v2/name_id_mapper.hpp"
+#include "storage/v2/property_value.hpp"
+#include "storage/v2/replication/config.hpp"
+#include "storage/v2/replication/enums.hpp"
+#include "storage/v2/replication/rpc.hpp"
+#include "storage/v2/replication/serialization.hpp"
+#include "storage/v2/storage.hpp"
+#include "utils/file.hpp"
+#include "utils/file_locker.hpp"
+#include "utils/spin_lock.hpp"
+#include "utils/synchronized.hpp"
+#include "utils/thread_pool.hpp"
+
+namespace storage {
+
+class Storage::ReplicationClient {
+ public:
+  ReplicationClient(std::string name, Storage *storage,
+                    const io::network::Endpoint &endpoint,
+                    replication::ReplicationMode mode,
+                    const replication::ReplicationClientConfig &config = {});
+
+  // Handler used for transfering the current transaction.
+  class ReplicaStream {
+   private:
+    friend class ReplicationClient;
+    explicit ReplicaStream(ReplicationClient *self,
+                           uint64_t previous_commit_timestamp,
+                           uint64_t current_seq_num);
+
+   public:
+    /// @throw rpc::RpcFailedException
+    void AppendDelta(const Delta &delta, const Vertex &vertex,
+                     uint64_t final_commit_timestamp);
+
+    /// @throw rpc::RpcFailedException
+    void AppendDelta(const Delta &delta, const Edge &edge,
+                     uint64_t final_commit_timestamp);
+
+    /// @throw rpc::RpcFailedException
+    void AppendTransactionEnd(uint64_t final_commit_timestamp);
+
+    /// @throw rpc::RpcFailedException
+    void AppendOperation(durability::StorageGlobalOperation operation,
+                         LabelId label, const std::set<PropertyId> &properties,
+                         uint64_t timestamp);
+
+   private:
+    /// @throw rpc::RpcFailedException
+    AppendDeltasRes Finalize();
+
+    ReplicationClient *self_;
+    rpc::Client::StreamHandler<AppendDeltasRpc> stream_;
+  };
+
+  // Handler for transfering the current WAL file whose data is
+  // contained in the internal buffer and the file.
+  class CurrentWalHandler {
+   private:
+    friend class ReplicationClient;
+    explicit CurrentWalHandler(ReplicationClient *self);
+
+   public:
+    void AppendFilename(const std::string &filename);
+
+    void AppendSize(size_t size);
+
+    void AppendFileData(utils::InputFile *file);
+
+    void AppendBufferData(const uint8_t *buffer, size_t buffer_size);
+
+    /// @throw rpc::RpcFailedException
+    CurrentWalRes Finalize();
+
+   private:
+    ReplicationClient *self_;
+    rpc::Client::StreamHandler<CurrentWalRpc> stream_;
+  };
+
+  void StartTransactionReplication(uint64_t current_wal_seq_num);
+
+  // Replication clients can be removed at any point
+  // so to avoid any complexity of checking if the client was removed whenever
+  // we want to send part of transaction and to avoid adding some GC logic this
+  // function will run a callback if, after previously callling
+  // StartTransactionReplication, stream is created.
+  void IfStreamingTransaction(
+      const std::function<void(ReplicaStream &handler)> &callback);
+
+  void FinalizeTransactionReplication();
+
+  // Transfer the snapshot file.
+  // @param path Path of the snapshot file.
+  SnapshotRes TransferSnapshot(const std::filesystem::path &path);
+
+  // Transfer the timestamp of the snapshot if it's the only difference
+  // between main and replica
+  OnlySnapshotRes TransferOnlySnapshot(uint64_t snapshot_timestamp);
+
+  CurrentWalHandler TransferCurrentWalFile() { return CurrentWalHandler{this}; }
+
+  // Transfer the WAL files
+  WalFilesRes TransferWalFiles(
+      const std::vector<std::filesystem::path> &wal_files);
+
+  const auto &Name() const { return name_; }
+
+  auto State() const { return replica_state_.load(); }
+
+  auto Mode() const { return mode_; }
+
+  auto Timeout() const { return timeout_; }
+
+  const auto &Endpoint() const { return rpc_client_->Endpoint(); }
+
+ private:
+  void FinalizeTransactionReplicationInternal();
+
+  void RecoverReplica(uint64_t replica_commit);
+
+  uint64_t ReplicateCurrentWal();
+
+  using RecoveryWals = std::vector<std::filesystem::path>;
+  struct RecoveryCurrentWal {
+    uint64_t current_wal_seq_num;
+
+    explicit RecoveryCurrentWal(const uint64_t current_wal_seq_num)
+        : current_wal_seq_num(current_wal_seq_num) {}
+  };
+  using RecoverySnapshot = std::filesystem::path;
+  struct RecoveryFinalSnapshot {
+    uint64_t snapshot_timestamp;
+
+    explicit RecoveryFinalSnapshot(const uint64_t snapshot_timestamp)
+        : snapshot_timestamp(snapshot_timestamp) {}
+  };
+  using RecoveryStep = std::variant<RecoverySnapshot, RecoveryWals,
+                                    RecoveryCurrentWal, RecoveryFinalSnapshot>;
+
+  std::vector<RecoveryStep> GetRecoverySteps(
+      uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker);
+
+  void InitializeClient();
+
+  void TryInitializeClient();
+
+  void HandleRpcFailure();
+
+  std::string name_;
+
+  Storage *storage_;
+
+  std::optional<communication::ClientContext> rpc_context_;
+  std::optional<rpc::Client> rpc_client_;
+
+  std::optional<ReplicaStream> replica_stream_;
+  replication::ReplicationMode mode_{replication::ReplicationMode::SYNC};
+
+  // Dispatcher class for timeout tasks
+  struct TimeoutDispatcher {
+    explicit TimeoutDispatcher(){};
+
+    void WaitForTaskToFinish();
+
+    void StartTimeoutTask(double timeout);
+
+    // If the Timeout task should continue waiting
+    std::atomic<bool> active{false};
+
+    std::mutex main_lock;
+    std::condition_variable main_cv;
+
+   private:
+    // if the Timeout task finished executing
+    bool finished{true};
+
+    utils::ThreadPool timeout_pool{1};
+  };
+
+  std::optional<double> timeout_;
+  std::optional<TimeoutDispatcher> timeout_dispatcher_;
+
+  utils::SpinLock client_lock_;
+  // This thread pool is used for background tasks so we don't
+  // block the main storage thread
+  // We use only 1 thread for 2 reasons:
+  //  - background tasks ALWAYS contain some kind of RPC communication.
+  //    We can't have multiple RPC communication from a same client
+  //    because that's not logically valid (e.g. you cannot send a snapshot
+  //    and WAL at a same time because WAL will arrive earlier and be applied
+  //    before the snapshot which is not correct)
+  //  - the implementation is simplified as we have a total control of what
+  //    this pool is executing. Also, we can simply queue multiple tasks
+  //    and be sure of the execution order.
+  //    Not having mulitple possible threads in the same client allows us
+  //    to ignore concurrency problems inside the client.
+  utils::ThreadPool thread_pool_{1};
+  std::atomic<replication::ReplicaState> replica_state_{
+      replication::ReplicaState::INVALID};
+};
+
+}  // namespace storage

src/storage/v2/replication/replication_server.cpp

@@ -0,0 +1,733 @@
+#include "storage/v2/replication/replication_server.hpp"
+
+#include "storage/v2/durability/durability.hpp"
+#include "storage/v2/durability/snapshot.hpp"
+#include "storage/v2/replication/config.hpp"
+#include "storage/v2/transaction.hpp"
+#include "utils/exceptions.hpp"
+
+namespace storage {
+Storage::ReplicationServer::ReplicationServer(
+    Storage *storage, io::network::Endpoint endpoint,
+    const replication::ReplicationServerConfig &config)
+    : storage_(storage) {
+  // Create RPC server.
+  if (config.ssl) {
+    rpc_server_context_.emplace(config.ssl->key_file, config.ssl->cert_file,
+                                config.ssl->ca_file, config.ssl->verify_peer);
+  } else {
+    rpc_server_context_.emplace();
+  }
+  // NOTE: The replication server must have a single thread for processing
+  // because there is no need for more processing threads - each replica can
+  // have only a single main server. Also, the single-threaded guarantee
+  // simplifies the rest of the implementation.
+  rpc_server_.emplace(std::move(endpoint), &*rpc_server_context_,
+                      /* workers_count = */ 1);
+
+  rpc_server_->Register<HeartbeatRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received HeartbeatRpc";
+        this->HeartbeatHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<AppendDeltasRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received AppendDeltasRpc:";
+        this->AppendDeltasHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<SnapshotRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received SnapshotRpc";
+        this->SnapshotHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<OnlySnapshotRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received OnlySnapshotRpc";
+        this->OnlySnapshotHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<WalFilesRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received WalFilesRpc";
+        this->WalFilesHandler(req_reader, res_builder);
+      });
+  rpc_server_->Register<CurrentWalRpc>(
+      [this](auto *req_reader, auto *res_builder) {
+        DLOG(INFO) << "Received CurrentWalRpc";
+        this->CurrentWalHandler(req_reader, res_builder);
+      });
+  rpc_server_->Start();
+}
+
+void Storage::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader,
+                                                  slk::Builder *res_builder) {
+  HeartbeatReq req;
+  slk::Load(&req, req_reader);
+  replication::Decoder decoder{req_reader};
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid value read form HeartbeatRpc!";
+  if (storage_->last_commit_timestamp_ == kTimestampInitialId) {
+    // The replica has no commits
+    // use the main's epoch id
+    storage_->epoch_id_ = std::move(*maybe_epoch_id);
+  } else if (*maybe_epoch_id != storage_->epoch_id_) {
+    auto &epoch_history = storage_->epoch_history_;
+    const auto result =
+        std::find_if(epoch_history.rbegin(), epoch_history.rend(),
+                     [&](const auto &epoch_info) {
+                       return epoch_info.first == *maybe_epoch_id;
+                     });
+    auto branching_point = kTimestampInitialId;
+    if (result == epoch_history.rend()) {
+      // we couldn't find the epoch_id inside the history so if it has
+      // the same or larger commit timestamp, some old replica became a main
+      // This isn't always the case, there is one case where an old main
+      // becomes a replica then main again and it should have a commit timestamp
+      // larger than the one on replica.
+      if (req.main_commit_timestamp >= storage_->last_commit_timestamp_) {
+        epoch_history.emplace_back(std::move(storage_->epoch_id_),
+                                   storage_->last_commit_timestamp_);
+        storage_->epoch_id_ = std::move(*maybe_epoch_id);
+        HeartbeatRes res{true, storage_->last_commit_timestamp_.load()};
+        slk::Save(res, res_builder);
+        return;
+      }
+    } else {
+      branching_point = result->second;
+    }
+    HeartbeatRes res{false, branching_point};
+    slk::Save(res, res_builder);
+    return;
+  }
+  HeartbeatRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::AppendDeltasHandler(
+    slk::Reader *req_reader, slk::Builder *res_builder) {
+  AppendDeltasReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid replication message";
+
+  // Different epoch ids should not be possible in AppendDeltas
+  // because Recovery and Heartbeat handlers should resolve
+  // any issues with timestamp and epoch id
+  CHECK(*maybe_epoch_id == storage_->epoch_id_)
+      << "Received Deltas from transaction with incompatible"
+         " epoch id";
+
+  const auto read_delta =
+      [&]() -> std::pair<uint64_t, durability::WalDeltaData> {
+    try {
+      auto timestamp = ReadWalDeltaHeader(&decoder);
+      DLOG(INFO) << "       Timestamp " << timestamp;
+      auto delta = ReadWalDeltaData(&decoder);
+      return {timestamp, delta};
+    } catch (const slk::SlkReaderException &) {
+      throw utils::BasicException("Missing data!");
+    } catch (const durability::RecoveryFailure &) {
+      throw utils::BasicException("Invalid data!");
+    }
+  };
+
+  if (req.previous_commit_timestamp !=
+      storage_->last_commit_timestamp_.load()) {
+    // Empty the stream
+    bool transaction_complete = false;
+    while (!transaction_complete) {
+      DLOG(INFO) << "Skipping delta";
+      const auto [timestamp, delta] = read_delta();
+      transaction_complete =
+          durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
+    }
+
+    AppendDeltasRes res{false, storage_->last_commit_timestamp_.load()};
+    slk::Save(res, res_builder);
+    return;
+  }
+
+  if (storage_->wal_file_) {
+    if (req.seq_num > storage_->wal_file_->SequenceNumber() ||
+        *maybe_epoch_id != storage_->epoch_id_) {
+      storage_->wal_file_->FinalizeWal();
+      storage_->wal_file_.reset();
+      storage_->wal_seq_num_ = req.seq_num;
+    } else {
+      CHECK(storage_->wal_file_->SequenceNumber() == req.seq_num)
+          << "Invalid sequence number of current wal file";
+      storage_->wal_seq_num_ = req.seq_num + 1;
+    }
+  } else {
+    storage_->wal_seq_num_ = req.seq_num;
+  }
+
+  auto edge_acc = storage_->edges_.access();
+  auto vertex_acc = storage_->vertices_.access();
+
+  std::optional<std::pair<uint64_t, storage::Storage::Accessor>>
+      commit_timestamp_and_accessor;
+  auto get_transaction =
+      [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
+        if (!commit_timestamp_and_accessor) {
+          commit_timestamp_and_accessor.emplace(commit_timestamp,
+                                                storage_->Access());
+        } else if (commit_timestamp_and_accessor->first != commit_timestamp) {
+          throw utils::BasicException("Received more than one transaction!");
+        }
+        return &commit_timestamp_and_accessor->second;
+      };
+
+  bool transaction_complete = false;
+  for (uint64_t i = 0; !transaction_complete; ++i) {
+    DLOG(INFO) << "  Delta " << i;
+    const auto [timestamp, delta] = read_delta();
+
+    switch (delta.type) {
+      case durability::WalDeltaData::Type::VERTEX_CREATE: {
+        DLOG(INFO) << "       Create vertex "
+                   << delta.vertex_create_delete.gid.AsUint();
+        auto transaction = get_transaction(timestamp);
+        transaction->CreateVertex(delta.vertex_create_delete.gid);
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_DELETE: {
+        DLOG(INFO) << "       Delete vertex "
+                   << delta.vertex_create_delete.gid.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = transaction->DeleteVertex(&*vertex);
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_add_remove_label.gid.AsUint()
+                   << " add label " << delta.vertex_add_remove_label.label;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = vertex->AddLabel(
+            transaction->NameToLabel(delta.vertex_add_remove_label.label));
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_add_remove_label.gid.AsUint()
+                   << " remove label " << delta.vertex_add_remove_label.label;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid,
+                                              storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret = vertex->RemoveLabel(
+            transaction->NameToLabel(delta.vertex_add_remove_label.label));
+        if (ret.HasError() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
+        DLOG(INFO) << "       Vertex "
+                   << delta.vertex_edge_set_property.gid.AsUint()
+                   << " set property "
+                   << delta.vertex_edge_set_property.property << " to "
+                   << delta.vertex_edge_set_property.value;
+        auto transaction = get_transaction(timestamp);
+        auto vertex = transaction->FindVertex(
+            delta.vertex_edge_set_property.gid, storage::View::NEW);
+        if (!vertex) throw utils::BasicException("Invalid transaction!");
+        auto ret =
+            vertex->SetProperty(transaction->NameToProperty(
+                                    delta.vertex_edge_set_property.property),
+                                delta.vertex_edge_set_property.value);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_CREATE: {
+        DLOG(INFO) << "       Create edge "
+                   << delta.edge_create_delete.gid.AsUint() << " of type "
+                   << delta.edge_create_delete.edge_type << " from vertex "
+                   << delta.edge_create_delete.from_vertex.AsUint()
+                   << " to vertex "
+                   << delta.edge_create_delete.to_vertex.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto from_vertex = transaction->FindVertex(
+            delta.edge_create_delete.from_vertex, storage::View::NEW);
+        if (!from_vertex) throw utils::BasicException("Invalid transaction!");
+        auto to_vertex = transaction->FindVertex(
+            delta.edge_create_delete.to_vertex, storage::View::NEW);
+        if (!to_vertex) throw utils::BasicException("Invalid transaction!");
+        auto edge = transaction->CreateEdge(
+            &*from_vertex, &*to_vertex,
+            transaction->NameToEdgeType(delta.edge_create_delete.edge_type),
+            delta.edge_create_delete.gid);
+        if (edge.HasError())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_DELETE: {
+        DLOG(INFO) << "       Delete edge "
+                   << delta.edge_create_delete.gid.AsUint() << " of type "
+                   << delta.edge_create_delete.edge_type << " from vertex "
+                   << delta.edge_create_delete.from_vertex.AsUint()
+                   << " to vertex "
+                   << delta.edge_create_delete.to_vertex.AsUint();
+        auto transaction = get_transaction(timestamp);
+        auto from_vertex = transaction->FindVertex(
+            delta.edge_create_delete.from_vertex, storage::View::NEW);
+        if (!from_vertex) throw utils::BasicException("Invalid transaction!");
+        auto to_vertex = transaction->FindVertex(
+            delta.edge_create_delete.to_vertex, storage::View::NEW);
+        if (!to_vertex) throw utils::BasicException("Invalid transaction!");
+        auto edges = from_vertex->OutEdges(
+            storage::View::NEW,
+            {transaction->NameToEdgeType(delta.edge_create_delete.edge_type)},
+            &*to_vertex);
+        if (edges.HasError())
+          throw utils::BasicException("Invalid transaction!");
+        if (edges->size() != 1)
+          throw utils::BasicException("Invalid transaction!");
+        auto &edge = (*edges)[0];
+        auto ret = transaction->DeleteEdge(&edge);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
+        DLOG(INFO) << "       Edge "
+                   << delta.vertex_edge_set_property.gid.AsUint()
+                   << " set property "
+                   << delta.vertex_edge_set_property.property << " to "
+                   << delta.vertex_edge_set_property.value;
+
+        if (!storage_->config_.items.properties_on_edges)
+          throw utils::BasicException(
+              "Can't set properties on edges because properties on edges "
+              "are disabled!");
+
+        auto transaction = get_transaction(timestamp);
+
+        // The following block of code effectively implements `FindEdge` and
+        // yields an accessor that is only valid for managing the edge's
+        // properties.
+        auto edge = edge_acc.find(delta.vertex_edge_set_property.gid);
+        if (edge == edge_acc.end())
+          throw utils::BasicException("Invalid transaction!");
+        // The edge visibility check must be done here manually because we
+        // don't allow direct access to the edges through the public API.
+        {
+          bool is_visible = true;
+          Delta *delta = nullptr;
+          {
+            std::lock_guard<utils::SpinLock> guard(edge->lock);
+            is_visible = !edge->deleted;
+            delta = edge->delta;
+          }
+          ApplyDeltasForRead(&transaction->transaction_, delta, View::NEW,
+                             [&is_visible](const Delta &delta) {
+                               switch (delta.action) {
+                                 case Delta::Action::ADD_LABEL:
+                                 case Delta::Action::REMOVE_LABEL:
+                                 case Delta::Action::SET_PROPERTY:
+                                 case Delta::Action::ADD_IN_EDGE:
+                                 case Delta::Action::ADD_OUT_EDGE:
+                                 case Delta::Action::REMOVE_IN_EDGE:
+                                 case Delta::Action::REMOVE_OUT_EDGE:
+                                   break;
+                                 case Delta::Action::RECREATE_OBJECT: {
+                                   is_visible = true;
+                                   break;
+                                 }
+                                 case Delta::Action::DELETE_OBJECT: {
+                                   is_visible = false;
+                                   break;
+                                 }
+                               }
+                             });
+          if (!is_visible) throw utils::BasicException("Invalid transaction!");
+        }
+        EdgeRef edge_ref(&*edge);
+        // Here we create an edge accessor that we will use to get the
+        // properties of the edge. The accessor is created with an invalid
+        // type and invalid from/to pointers because we don't know them
+        // here, but that isn't an issue because we won't use that part of
+        // the API here.
+        auto ea = EdgeAccessor{edge_ref,
+                               EdgeTypeId::FromUint(0UL),
+                               nullptr,
+                               nullptr,
+                               &transaction->transaction_,
+                               &storage_->indices_,
+                               &storage_->constraints_,
+                               storage_->config_.items};
+
+        auto ret = ea.SetProperty(transaction->NameToProperty(
+                                      delta.vertex_edge_set_property.property),
+                                  delta.vertex_edge_set_property.value);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+
+      case durability::WalDeltaData::Type::TRANSACTION_END: {
+        DLOG(INFO) << "       Transaction end";
+        if (!commit_timestamp_and_accessor ||
+            commit_timestamp_and_accessor->first != timestamp)
+          throw utils::BasicException("Invalid data!");
+        auto ret = commit_timestamp_and_accessor->second.Commit(
+            commit_timestamp_and_accessor->first);
+        if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
+        commit_timestamp_and_accessor = std::nullopt;
+        break;
+      }
+
+      case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
+        DLOG(INFO) << "       Create label index on :"
+                   << delta.operation_label.label;
+        // Need to send the timestamp
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->CreateIndex(
+                storage_->NameToLabel(delta.operation_label.label), timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
+        DLOG(INFO) << "       Drop label index on :"
+                   << delta.operation_label.label;
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropIndex(
+                storage_->NameToLabel(delta.operation_label.label), timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
+        DLOG(INFO) << "       Create label+property index on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->CreateIndex(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
+        DLOG(INFO) << "       Drop label+property index on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropIndex(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
+        DLOG(INFO) << "       Create existence constraint on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        auto ret = storage_->CreateExistenceConstraint(
+            storage_->NameToLabel(delta.operation_label_property.label),
+            storage_->NameToProperty(delta.operation_label_property.property),
+            timestamp);
+        if (!ret.HasValue() || !ret.GetValue())
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
+        DLOG(INFO) << "       Drop existence constraint on :"
+                   << delta.operation_label_property.label << " ("
+                   << delta.operation_label_property.property << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        if (!storage_->DropExistenceConstraint(
+                storage_->NameToLabel(delta.operation_label_property.label),
+                storage_->NameToProperty(
+                    delta.operation_label_property.property),
+                timestamp))
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
+        std::stringstream ss;
+        utils::PrintIterable(ss, delta.operation_label_properties.properties);
+        DLOG(INFO) << "       Create unique constraint on :"
+                   << delta.operation_label_properties.label << " (" << ss.str()
+                   << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        std::set<PropertyId> properties;
+        for (const auto &prop : delta.operation_label_properties.properties) {
+          properties.emplace(storage_->NameToProperty(prop));
+        }
+        auto ret = storage_->CreateUniqueConstraint(
+            storage_->NameToLabel(delta.operation_label_properties.label),
+            properties, timestamp);
+        if (!ret.HasValue() ||
+            ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+      case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
+        std::stringstream ss;
+        utils::PrintIterable(ss, delta.operation_label_properties.properties);
+        DLOG(INFO) << "       Drop unique constraint on :"
+                   << delta.operation_label_properties.label << " (" << ss.str()
+                   << ")";
+        if (commit_timestamp_and_accessor)
+          throw utils::BasicException("Invalid transaction!");
+        std::set<PropertyId> properties;
+        for (const auto &prop : delta.operation_label_properties.properties) {
+          properties.emplace(storage_->NameToProperty(prop));
+        }
+        auto ret = storage_->DropUniqueConstraint(
+            storage_->NameToLabel(delta.operation_label_properties.label),
+            properties, timestamp);
+        if (ret != UniqueConstraints::DeletionStatus::SUCCESS)
+          throw utils::BasicException("Invalid transaction!");
+        break;
+      }
+    }
+    transaction_complete =
+        durability::IsWalDeltaDataTypeTransactionEnd(delta.type);
+  }
+
+  if (commit_timestamp_and_accessor)
+    throw utils::BasicException("Invalid data!");
+
+  AppendDeltasRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader,
+                                                 slk::Builder *res_builder) {
+  SnapshotReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->snapshot_directory_);
+
+  const auto maybe_snapshot_path =
+      decoder.ReadFile(storage_->snapshot_directory_);
+  CHECK(maybe_snapshot_path) << "Failed to load snapshot!";
+  DLOG(INFO) << "Received snapshot saved to " << *maybe_snapshot_path;
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  // Clear the database
+  storage_->vertices_.clear();
+  storage_->edges_.clear();
+
+  storage_->constraints_ = Constraints();
+  storage_->indices_.label_index = LabelIndex(
+      &storage_->indices_, &storage_->constraints_, storage_->config_.items);
+  storage_->indices_.label_property_index = LabelPropertyIndex(
+      &storage_->indices_, &storage_->constraints_, storage_->config_.items);
+  try {
+    DLOG(INFO) << "Loading snapshot";
+    auto recovered_snapshot = durability::LoadSnapshot(
+        *maybe_snapshot_path, &storage_->vertices_, &storage_->edges_,
+        &storage_->epoch_history_, &storage_->name_id_mapper_,
+        &storage_->edge_count_, storage_->config_.items);
+    DLOG(INFO) << "Snapshot loaded successfully";
+    // If this step is present it should always be the first step of
+    // the recovery so we use the UUID we read from snasphost
+    storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
+    storage_->epoch_id_ = std::move(recovered_snapshot.snapshot_info.epoch_id);
+    const auto &recovery_info = recovered_snapshot.recovery_info;
+    storage_->vertex_id_ = recovery_info.next_vertex_id;
+    storage_->edge_id_ = recovery_info.next_edge_id;
+    storage_->timestamp_ =
+        std::max(storage_->timestamp_, recovery_info.next_timestamp);
+
+    durability::RecoverIndicesAndConstraints(
+        recovered_snapshot.indices_constraints, &storage_->indices_,
+        &storage_->constraints_, &storage_->vertices_);
+  } catch (const durability::RecoveryFailure &e) {
+    LOG(FATAL) << "Couldn't load the snapshot because of: " << e.what();
+  }
+  storage_->last_commit_timestamp_ = storage_->timestamp_ - 1;
+  storage_guard.unlock();
+
+  SnapshotRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+
+  // Delete other durability files
+  auto snapshot_files = durability::GetSnapshotFiles(
+      storage_->snapshot_directory_, storage_->uuid_);
+  for (const auto &[path, uuid, _] : snapshot_files) {
+    if (path != *maybe_snapshot_path) {
+      storage_->file_retainer_.DeleteFile(path);
+    }
+  }
+
+  auto wal_files =
+      durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_);
+  if (wal_files) {
+    for (const auto &wal_file : *wal_files) {
+      storage_->file_retainer_.DeleteFile(wal_file.path);
+    }
+
+    storage_->wal_file_.reset();
+  }
+}
+
+void Storage::ReplicationServer::OnlySnapshotHandler(
+    slk::Reader *req_reader, slk::Builder *res_builder) {
+  OnlySnapshotReq req;
+  slk::Load(&req, req_reader);
+
+  CHECK(storage_->last_commit_timestamp_.load() < req.snapshot_timestamp)
+      << "Invalid snapshot timestamp, it should be less than the last"
+         "commited timestamp";
+
+  replication::Decoder decoder{req_reader};
+  auto maybe_epoch_id = decoder.ReadString();
+  CHECK(maybe_epoch_id) << "Invalid replication message";
+
+  if (*maybe_epoch_id != storage_->epoch_id_) {
+    storage_->epoch_history_.emplace_back(std::move(storage_->epoch_id_),
+                                          storage_->last_commit_timestamp_);
+    storage_->epoch_id_ = std::move(*maybe_epoch_id);
+  }
+
+  storage_->last_commit_timestamp_.store(req.snapshot_timestamp);
+
+  OnlySnapshotRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader,
+                                                 slk::Builder *res_builder) {
+  WalFilesReq req;
+  slk::Load(&req, req_reader);
+
+  const auto wal_file_number = req.file_number;
+  DLOG(INFO) << "Received WAL files: " << wal_file_number;
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->wal_directory_);
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  durability::RecoveredIndicesAndConstraints indices_constraints;
+  auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
+  if (wal_info.seq_num == 0) {
+    storage_->uuid_ = wal_info.uuid;
+  }
+
+  // Check the seq number of the first wal file to see if it's the
+  // finalized form of the current wal on replica
+  if (storage_->wal_file_) {
+    if (storage_->wal_file_->SequenceNumber() == wal_info.seq_num &&
+        storage_->wal_file_->Path() != path) {
+      storage_->wal_file_->DeleteWal();
+    }
+    storage_->wal_file_.reset();
+  }
+
+  for (auto i = 1; i < wal_file_number; ++i) {
+    LoadWal(&decoder, &indices_constraints);
+  }
+
+  durability::RecoverIndicesAndConstraints(
+      indices_constraints, &storage_->indices_, &storage_->constraints_,
+      &storage_->vertices_);
+  storage_guard.unlock();
+
+  WalFilesRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+void Storage::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader,
+                                                   slk::Builder *res_builder) {
+  CurrentWalReq req;
+  slk::Load(&req, req_reader);
+
+  replication::Decoder decoder(req_reader);
+
+  utils::EnsureDirOrDie(storage_->wal_directory_);
+
+  std::unique_lock<utils::RWLock> storage_guard(storage_->main_lock_);
+  durability::RecoveredIndicesAndConstraints indices_constraints;
+  auto [wal_info, path] = LoadWal(&decoder, &indices_constraints);
+  if (wal_info.seq_num == 0) {
+    storage_->uuid_ = wal_info.uuid;
+  }
+
+  if (storage_->wal_file_ &&
+      storage_->wal_file_->SequenceNumber() == wal_info.seq_num &&
+      storage_->wal_file_->Path() != path) {
+    // Delete the old wal file
+    storage_->file_retainer_.DeleteFile(storage_->wal_file_->Path());
+  }
+  CHECK(storage_->config_.durability.snapshot_wal_mode ==
+        Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL);
+  storage_->wal_file_.emplace(std::move(path), storage_->config_.items,
+                              &storage_->name_id_mapper_, wal_info.seq_num,
+                              wal_info.from_timestamp, wal_info.to_timestamp,
+                              wal_info.num_deltas, &storage_->file_retainer_);
+  durability::RecoverIndicesAndConstraints(
+      indices_constraints, &storage_->indices_, &storage_->constraints_,
+      &storage_->vertices_);
+  storage_guard.unlock();
+
+  CurrentWalRes res{true, storage_->last_commit_timestamp_.load()};
+  slk::Save(res, res_builder);
+}
+
+std::pair<durability::WalInfo, std::filesystem::path>
+Storage::ReplicationServer::LoadWal(
+    replication::Decoder *decoder,
+    durability::RecoveredIndicesAndConstraints *indices_constraints) {
+  auto maybe_wal_path = decoder->ReadFile(storage_->wal_directory_, "_MAIN");
+  CHECK(maybe_wal_path) << "Failed to load WAL!";
+  DLOG(INFO) << "Received WAL saved to " << *maybe_wal_path;
+  try {
+    auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
+    if (wal_info.epoch_id != storage_->epoch_id_) {
+      storage_->epoch_history_.emplace_back(wal_info.epoch_id,
+                                            storage_->last_commit_timestamp_);
+      storage_->epoch_id_ = std::move(wal_info.epoch_id);
+    }
+    auto info = durability::LoadWal(
+        *maybe_wal_path, indices_constraints, storage_->last_commit_timestamp_,
+        &storage_->vertices_, &storage_->edges_, &storage_->name_id_mapper_,
+        &storage_->edge_count_, storage_->config_.items);
+    storage_->vertex_id_ =
+        std::max(storage_->vertex_id_.load(), info.next_vertex_id);
+    storage_->edge_id_ = std::max(storage_->edge_id_.load(), info.next_edge_id);
+    storage_->timestamp_ = std::max(storage_->timestamp_, info.next_timestamp);
+    if (info.next_timestamp != 0) {
+      storage_->last_commit_timestamp_ = info.next_timestamp - 1;
+    }
+    DLOG(INFO) << *maybe_wal_path << " loaded successfully";
+    return {std::move(wal_info), std::move(*maybe_wal_path)};
+  } catch (const durability::RecoveryFailure &e) {
+    LOG(FATAL) << "Couldn't recover WAL deltas from " << *maybe_wal_path
+               << " because of: " << e.what();
+  }
+}
+
+Storage::ReplicationServer::~ReplicationServer() {
+  if (rpc_server_) {
+    rpc_server_->Shutdown();
+    rpc_server_->AwaitShutdown();
+  }
+}
+}  // namespace storage

src/storage/v2/replication/replication_server.hpp

@@ -0,0 +1,41 @@
+#pragma once
+
+#include "storage/v2/storage.hpp"
+
+namespace storage {
+
+class Storage::ReplicationServer {
+ public:
+  explicit ReplicationServer(
+      Storage *storage, io::network::Endpoint endpoint,
+      const replication::ReplicationServerConfig &config);
+  ReplicationServer(const ReplicationServer &) = delete;
+  ReplicationServer(ReplicationServer &&) = delete;
+  ReplicationServer &operator=(const ReplicationServer &) = delete;
+  ReplicationServer &operator=(ReplicationServer &&) = delete;
+
+  ~ReplicationServer();
+
+ private:
+  // RPC handlers
+  void HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  // RPC for replicating only the commit of the last snapshot as that is the
+  // only difference between the replica and main (all of the data is
+  // already replicated through previous WAL\Snapshot files)
+  void OnlySnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+  void CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder);
+
+  std::pair<durability::WalInfo, std::filesystem::path> LoadWal(
+      replication::Decoder *decoder,
+      durability::RecoveredIndicesAndConstraints *indices_constraints);
+
+  std::optional<communication::ServerContext> rpc_server_context_;
+  std::optional<rpc::Server> rpc_server_;
+
+  Storage *storage_;
+};
+
+}  // namespace storage

src/storage/v2/replication/rpc.lcp

@@ -0,0 +1,55 @@
+#>cpp
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+
+#include "rpc/messages.hpp"
+#include "slk/serialization.hpp"
+#include "slk/streams.hpp"
+cpp<#
+;; TODO(antonio2368): Change namespace to `storage::replication` once LCP is
+;; updated to support such namespaces.
+(lcp:namespace storage)
+
+(lcp:define-rpc append-deltas
+  ;; The actual deltas are sent as additional data using the RPC client's
+  ;; streaming API for additional data.
+  (:request
+    ((previous-commit-timestamp :uint64_t)
+     (seq-num :uint64_t)))
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:define-rpc heartbeat
+  (:request ((main-commit-timestamp :uint64_t)))
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:define-rpc snapshot
+  (:request ())
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:define-rpc only-snapshot
+  (:request ((snapshot-timestamp :uint64_t)))
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:define-rpc wal-files
+  (:request ((file-number :uint64_t)))
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:define-rpc current-wal
+  (:request ())
+  (:response
+    ((success :bool)
+     (current-commit-timestamp :uint64_t))))
+
+(lcp:pop-namespace) ;; storage

src/storage/v2/replication/serialization.cpp

@@ -0,0 +1,156 @@
+#include "storage/v2/replication/serialization.hpp"
+
+namespace storage::replication {
+////// Encoder //////
+void Encoder::WriteMarker(durability::Marker marker) {
+  slk::Save(marker, builder_);
+}
+
+void Encoder::WriteBool(bool value) {
+  WriteMarker(durability::Marker::TYPE_BOOL);
+  slk::Save(value, builder_);
+}
+
+void Encoder::WriteUint(uint64_t value) {
+  WriteMarker(durability::Marker::TYPE_INT);
+  slk::Save(value, builder_);
+}
+
+void Encoder::WriteDouble(double value) {
+  WriteMarker(durability::Marker::TYPE_DOUBLE);
+  slk::Save(value, builder_);
+}
+
+void Encoder::WriteString(const std::string_view &value) {
+  WriteMarker(durability::Marker::TYPE_STRING);
+  slk::Save(value, builder_);
+}
+
+void Encoder::WritePropertyValue(const PropertyValue &value) {
+  WriteMarker(durability::Marker::TYPE_PROPERTY_VALUE);
+  slk::Save(value, builder_);
+}
+
+void Encoder::WriteBuffer(const uint8_t *buffer, const size_t buffer_size) {
+  builder_->Save(buffer, buffer_size);
+}
+
+void Encoder::WriteFileData(utils::InputFile *file) {
+  auto file_size = file->GetSize();
+  uint8_t buffer[utils::kFileBufferSize];
+  while (file_size > 0) {
+    const auto chunk_size = std::min(file_size, utils::kFileBufferSize);
+    file->Read(buffer, chunk_size);
+    WriteBuffer(buffer, chunk_size);
+    file_size -= chunk_size;
+  }
+}
+
+void Encoder::WriteFile(const std::filesystem::path &path) {
+  utils::InputFile file;
+  CHECK(file.Open(path)) << "Failed to open file " << path;
+  CHECK(path.has_filename()) << "Path does not have a filename!";
+  const auto &filename = path.filename().generic_string();
+  WriteString(filename);
+  auto file_size = file.GetSize();
+  WriteUint(file_size);
+  WriteFileData(&file);
+  file.Close();
+}
+
+////// Decoder //////
+std::optional<durability::Marker> Decoder::ReadMarker() {
+  durability::Marker marker;
+  slk::Load(&marker, reader_);
+  return marker;
+}
+
+std::optional<bool> Decoder::ReadBool() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_BOOL)
+    return std::nullopt;
+  bool value;
+  slk::Load(&value, reader_);
+  return value;
+}
+
+std::optional<uint64_t> Decoder::ReadUint() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_INT)
+    return std::nullopt;
+  uint64_t value;
+  slk::Load(&value, reader_);
+  return value;
+}
+
+std::optional<double> Decoder::ReadDouble() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_DOUBLE)
+    return std::nullopt;
+  double value;
+  slk::Load(&value, reader_);
+  return value;
+}
+
+std::optional<std::string> Decoder::ReadString() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_STRING)
+    return std::nullopt;
+  std::string value;
+  slk::Load(&value, reader_);
+  return std::move(value);
+}
+
+std::optional<PropertyValue> Decoder::ReadPropertyValue() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_PROPERTY_VALUE)
+    return std::nullopt;
+  PropertyValue value;
+  slk::Load(&value, reader_);
+  return std::move(value);
+}
+
+bool Decoder::SkipString() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_STRING)
+    return false;
+  std::string value;
+  slk::Load(&value, reader_);
+  return true;
+}
+
+bool Decoder::SkipPropertyValue() {
+  if (const auto marker = ReadMarker();
+      !marker || marker != durability::Marker::TYPE_PROPERTY_VALUE)
+    return false;
+  PropertyValue value;
+  slk::Load(&value, reader_);
+  return true;
+}
+
+std::optional<std::filesystem::path> Decoder::ReadFile(
+    const std::filesystem::path &directory, const std::string &suffix) {
+  CHECK(std::filesystem::exists(directory) &&
+        std::filesystem::is_directory(directory))
+      << "Sent path for streamed files should be a valid directory!";
+  utils::OutputFile file;
+  const auto maybe_filename = ReadString();
+  CHECK(maybe_filename) << "Filename missing for the file";
+  const auto filename = *maybe_filename + suffix;
+  auto path = directory / filename;
+
+  file.Open(path, utils::OutputFile::Mode::OVERWRITE_EXISTING);
+  std::optional<size_t> maybe_file_size = ReadUint();
+  CHECK(maybe_file_size) << "File size missing";
+  auto file_size = *maybe_file_size;
+  uint8_t buffer[utils::kFileBufferSize];
+  while (file_size > 0) {
+    const auto chunk_size = std::min(file_size, utils::kFileBufferSize);
+    reader_->Load(buffer, chunk_size);
+    file.Write(buffer, chunk_size);
+    file_size -= chunk_size;
+  }
+  file.Close();
+  return std::move(path);
+}
+}  // namespace storage::replication

src/storage/v2/replication/serialization.hpp

@@ -0,0 +1,70 @@
+#pragma once
+
+#include <filesystem>
+
+#include "slk/streams.hpp"
+#include "storage/v2/durability/serialization.hpp"
+#include "storage/v2/replication/slk.hpp"
+#include "utils/cast.hpp"
+#include "utils/file.hpp"
+
+namespace storage::replication {
+
+class Encoder final : public durability::BaseEncoder {
+ public:
+  explicit Encoder(slk::Builder *builder) : builder_(builder) {}
+
+  void WriteMarker(durability::Marker marker) override;
+
+  void WriteBool(bool value) override;
+
+  void WriteUint(uint64_t value) override;
+
+  void WriteDouble(double value) override;
+
+  void WriteString(const std::string_view &value) override;
+
+  void WritePropertyValue(const PropertyValue &value) override;
+
+  void WriteBuffer(const uint8_t *buffer, size_t buffer_size);
+
+  void WriteFileData(utils::InputFile *file);
+
+  void WriteFile(const std::filesystem::path &path);
+
+ private:
+  slk::Builder *builder_;
+};
+
+class Decoder final : public durability::BaseDecoder {
+ public:
+  explicit Decoder(slk::Reader *reader) : reader_(reader) {}
+
+  std::optional<durability::Marker> ReadMarker() override;
+
+  std::optional<bool> ReadBool() override;
+
+  std::optional<uint64_t> ReadUint() override;
+
+  std::optional<double> ReadDouble() override;
+
+  std::optional<std::string> ReadString() override;
+
+  std::optional<PropertyValue> ReadPropertyValue() override;
+
+  bool SkipString() override;
+
+  bool SkipPropertyValue() override;
+
+  /// Read the file and save it inside the specified directory.
+  /// @param directory Directory which will contain the read file.
+  /// @param suffix Suffix to be added to the received file's filename.
+  /// @return If the read was successful, path to the read file.
+  std::optional<std::filesystem::path> ReadFile(
+      const std::filesystem::path &directory, const std::string &suffix = "");
+
+ private:
+  slk::Reader *reader_;
+};
+
+}  // namespace storage::replication

src/storage/v2/replication/slk.cpp

@@ -0,0 +1,161 @@
+#include "storage/v2/replication/slk.hpp"
+
+#include <type_traits>
+
+#include "utils/cast.hpp"
+
+namespace slk {
+
+void Save(const storage::Gid &gid, slk::Builder *builder) {
+  slk::Save(gid.AsUint(), builder);
+}
+
+void Load(storage::Gid *gid, slk::Reader *reader) {
+  uint64_t value;
+  slk::Load(&value, reader);
+  *gid = storage::Gid::FromUint(value);
+}
+
+void Save(const storage::PropertyValue::Type &type, slk::Builder *builder) {
+  slk::Save(utils::UnderlyingCast(type), builder);
+}
+
+void Load(storage::PropertyValue::Type *type, slk::Reader *reader) {
+  using PVTypeUnderlyingType =
+      std::underlying_type_t<storage::PropertyValue::Type>;
+  PVTypeUnderlyingType value;
+  slk::Load(&value, reader);
+  bool valid;
+  switch (value) {
+    case utils::UnderlyingCast(storage::PropertyValue::Type::Null):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::Bool):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::Int):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::Double):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::String):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::List):
+    case utils::UnderlyingCast(storage::PropertyValue::Type::Map):
+      valid = true;
+      break;
+    default:
+      valid = false;
+      break;
+  }
+  if (!valid)
+    throw slk::SlkDecodeException(
+        "Trying to load unknown storage::PropertyValue!");
+  *type = static_cast<storage::PropertyValue::Type>(value);
+}
+
+void Save(const storage::PropertyValue &value, slk::Builder *builder) {
+  switch (value.type()) {
+    case storage::PropertyValue::Type::Null:
+      slk::Save(storage::PropertyValue::Type::Null, builder);
+      return;
+    case storage::PropertyValue::Type::Bool:
+      slk::Save(storage::PropertyValue::Type::Bool, builder);
+      slk::Save(value.ValueBool(), builder);
+      return;
+    case storage::PropertyValue::Type::Int:
+      slk::Save(storage::PropertyValue::Type::Int, builder);
+      slk::Save(value.ValueInt(), builder);
+      return;
+    case storage::PropertyValue::Type::Double:
+      slk::Save(storage::PropertyValue::Type::Double, builder);
+      slk::Save(value.ValueDouble(), builder);
+      return;
+    case storage::PropertyValue::Type::String:
+      slk::Save(storage::PropertyValue::Type::String, builder);
+      slk::Save(value.ValueString(), builder);
+      return;
+    case storage::PropertyValue::Type::List: {
+      slk::Save(storage::PropertyValue::Type::List, builder);
+      const auto &values = value.ValueList();
+      size_t size = values.size();
+      slk::Save(size, builder);
+      for (const auto &v : values) {
+        slk::Save(v, builder);
+      }
+      return;
+    }
+    case storage::PropertyValue::Type::Map: {
+      slk::Save(storage::PropertyValue::Type::Map, builder);
+      const auto &map = value.ValueMap();
+      size_t size = map.size();
+      slk::Save(size, builder);
+      for (const auto &kv : map) {
+        slk::Save(kv, builder);
+      }
+      return;
+    }
+  }
+}
+
+void Load(storage::PropertyValue *value, slk::Reader *reader) {
+  storage::PropertyValue::Type type;
+  slk::Load(&type, reader);
+  switch (type) {
+    case storage::PropertyValue::Type::Null:
+      *value = storage::PropertyValue();
+      return;
+    case storage::PropertyValue::Type::Bool: {
+      bool v;
+      slk::Load(&v, reader);
+      *value = storage::PropertyValue(v);
+      return;
+    }
+    case storage::PropertyValue::Type::Int: {
+      int64_t v;
+      slk::Load(&v, reader);
+      *value = storage::PropertyValue(v);
+      return;
+    }
+    case storage::PropertyValue::Type::Double: {
+      double v;
+      slk::Load(&v, reader);
+      *value = storage::PropertyValue(v);
+      return;
+    }
+    case storage::PropertyValue::Type::String: {
+      std::string v;
+      slk::Load(&v, reader);
+      *value = storage::PropertyValue(std::move(v));
+      return;
+    }
+    case storage::PropertyValue::Type::List: {
+      size_t size;
+      slk::Load(&size, reader);
+      std::vector<storage::PropertyValue> list(size);
+      for (size_t i = 0; i < size; ++i) {
+        slk::Load(&list[i], reader);
+      }
+      *value = storage::PropertyValue(std::move(list));
+      return;
+    }
+    case storage::PropertyValue::Type::Map: {
+      size_t size;
+      slk::Load(&size, reader);
+      std::map<std::string, storage::PropertyValue> map;
+      for (size_t i = 0; i < size; ++i) {
+        std::pair<std::string, storage::PropertyValue> kv;
+        slk::Load(&kv, reader);
+        map.insert(kv);
+      }
+      *value = storage::PropertyValue(std::move(map));
+      return;
+    }
+  }
+}
+
+void Save(const storage::durability::Marker &marker, slk::Builder *builder) {
+  slk::Save(utils::UnderlyingCast(marker), builder);
+}
+
+void Load(storage::durability::Marker *marker, slk::Reader *reader) {
+  using PVTypeUnderlyingType =
+      std::underlying_type_t<storage::PropertyValue::Type>;
+  PVTypeUnderlyingType value;
+  slk::Load(&value, reader);
+  *marker = static_cast<storage::durability::Marker>(value);
+}
+
+}  // namespace slk

src/storage/v2/replication/slk.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "slk/serialization.hpp"
+#include "storage/v2/durability/marker.hpp"
+#include "storage/v2/id_types.hpp"
+#include "storage/v2/property_value.hpp"
+
+namespace slk {
+
+void Save(const storage::Gid &gid, slk::Builder *builder);
+void Load(storage::Gid *gid, slk::Reader *reader);
+
+void Save(const storage::PropertyValue &value, slk::Builder *builder);
+void Load(storage::PropertyValue *value, slk::Reader *reader);
+
+void Save(const storage::durability::Marker &marker, slk::Builder *builder);
+void Load(storage::durability::Marker *marker, slk::Reader *reader);
+
+}  // namespace slk

src/storage/v2/storage.cpp

@@ -1,20 +1,41 @@
 #include "storage/v2/storage.hpp"
 
 #include <algorithm>
+#include <atomic>
 #include <memory>
+#include <mutex>
+#include <variant>
 
 #include <gflags/gflags.h>
 #include <glog/logging.h>
 
+#include "io/network/endpoint.hpp"
 #include "storage/v2/durability/durability.hpp"
+#include "storage/v2/durability/metadata.hpp"
 #include "storage/v2/durability/paths.hpp"
 #include "storage/v2/durability/snapshot.hpp"
+#include "storage/v2/durability/wal.hpp"
+#include "storage/v2/indices.hpp"
 #include "storage/v2/mvcc.hpp"
+#include "storage/v2/replication/config.hpp"
+#include "utils/file.hpp"
+#include "utils/rw_lock.hpp"
+#include "utils/spin_lock.hpp"
 #include "utils/stat.hpp"
 #include "utils/uuid.hpp"
 
+#ifdef MG_ENTERPRISE
+#include "storage/v2/replication/replication_client.hpp"
+#include "storage/v2/replication/replication_server.hpp"
+#include "storage/v2/replication/rpc.hpp"
+#endif
+
 namespace storage {
 
+namespace {
+constexpr uint16_t kEpochHistoryRetention = 1000;
+}  // namespace
+
 auto AdvanceToVisibleVertex(utils::SkipList<Vertex>::Iterator it,
                             utils::SkipList<Vertex>::Iterator end,
                             std::optional<VertexAccessor> *vertex,
@@ -303,7 +324,8 @@ Storage::Storage(Config config)
                      durability::kWalDirectory),
       lock_file_path_(config_.durability.storage_directory /
                       durability::kLockFile),
-      uuid_(utils::GenerateUUID()) {
+      uuid_(utils::GenerateUUID()),
+      epoch_id_(utils::GenerateUUID()) {
   if (config_.durability.snapshot_wal_mode !=
           Config::Durability::SnapshotWalMode::DISABLED ||
       config_.durability.snapshot_on_exit ||
@@ -335,13 +357,20 @@ Storage::Storage(Config config)
   }
   if (config_.durability.recover_on_startup) {
     auto info = durability::RecoverData(
-        snapshot_directory_, wal_directory_, &uuid_, &vertices_, &edges_,
-        &edge_count_, &name_id_mapper_, &indices_, &constraints_, config_.items,
-        &wal_seq_num_);
+        snapshot_directory_, wal_directory_, &uuid_, &epoch_id_,
+        &epoch_history_, &vertices_, &edges_, &edge_count_, &name_id_mapper_,
+        &indices_, &constraints_, config_.items, &wal_seq_num_);
     if (info) {
       vertex_id_ = info->next_vertex_id;
       edge_id_ = info->next_edge_id;
       timestamp_ = std::max(timestamp_, info->next_timestamp);
+#if MG_ENTERPRISE
+      // After we finished the recovery, the info->next_timestamp will
+      // basically be
+      // `std::max(latest_snapshot.start_timestamp + 1, latest_wal.to_timestamp
+      // + 1)` So the last commited transaction is one before that.
+      last_commit_timestamp_ = timestamp_ - 1;
+#endif
     }
   } else if (config_.durability.snapshot_wal_mode !=
                  Config::Durability::SnapshotWalMode::DISABLED ||
@@ -379,23 +408,8 @@ Storage::Storage(Config config)
   }
   if (config_.durability.snapshot_wal_mode !=
       Config::Durability::SnapshotWalMode::DISABLED) {
-    snapshot_runner_.Run(
-        "Snapshot", config_.durability.snapshot_interval, [this] {
-          // Take master RW lock (for reading).
-          std::shared_lock<utils::RWLock> storage_guard(main_lock_);
-
-          // Create the transaction used to create the snapshot.
-          auto transaction = CreateTransaction();
-
-          // Create snapshot.
-          durability::CreateSnapshot(
-              &transaction, snapshot_directory_, wal_directory_,
-              config_.durability.snapshot_retention_count, &vertices_, &edges_,
-              &name_id_mapper_, &indices_, &constraints_, config_.items, uuid_);
-
-          // Finalize snapshot transaction.
-          commit_log_.MarkFinished(transaction.start_timestamp);
-        });
+    snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval,
+                         [this] { this->CreateSnapshot(); });
   }
   if (config_.gc.type == Config::Gc::Type::PERIODIC) {
     gc_runner_.Run("Storage GC", config_.gc.interval,
@@ -407,26 +421,23 @@ Storage::~Storage() {
   if (config_.gc.type == Config::Gc::Type::PERIODIC) {
     gc_runner_.Stop();
   }
-  wal_file_ = std::nullopt;
+#ifdef MG_ENTERPRISE
+  {
+    // Clear replication data
+    replication_server_.reset();
+    replication_clients_.WithLock([&](auto &clients) { clients.clear(); });
+  }
+#endif
+  if (wal_file_) {
+    wal_file_->FinalizeWal();
+    wal_file_ = std::nullopt;
+  }
   if (config_.durability.snapshot_wal_mode !=
       Config::Durability::SnapshotWalMode::DISABLED) {
     snapshot_runner_.Stop();
   }
   if (config_.durability.snapshot_on_exit) {
-    // Take master RW lock (for reading).
-    std::shared_lock<utils::RWLock> storage_guard(main_lock_);
-
-    // Create the transaction used to create the snapshot.
-    auto transaction = CreateTransaction();
-
-    // Create snapshot.
-    durability::CreateSnapshot(
-        &transaction, snapshot_directory_, wal_directory_,
-        config_.durability.snapshot_retention_count, &vertices_, &edges_,
-        &name_id_mapper_, &indices_, &constraints_, config_.items, uuid_);
-
-    // Finalize snapshot transaction.
-    commit_log_.MarkFinished(transaction.start_timestamp);
+    CreateSnapshot();
   }
 }
 
@@ -467,6 +478,29 @@ VertexAccessor Storage::Accessor::CreateVertex() {
                         &storage_->constraints_, config_);
 }
 
+#ifdef MG_ENTERPRISE
+VertexAccessor Storage::Accessor::CreateVertex(storage::Gid gid) {
+  // NOTE: When we update the next `vertex_id_` here we perform a RMW
+  // (read-modify-write) operation that ISN'T atomic! But, that isn't an issue
+  // because this function is only called from the replication delta applier
+  // that runs single-threadedly and while this instance is set-up to apply
+  // threads (it is the replica), it is guaranteed that no other writes are
+  // possible.
+  storage_->vertex_id_.store(
+      std::max(storage_->vertex_id_.load(std::memory_order_acquire),
+               gid.AsUint() + 1),
+      std::memory_order_release);
+  auto acc = storage_->vertices_.access();
+  auto delta = CreateDeleteObjectDelta(&transaction_);
+  auto [it, inserted] = acc.insert(Vertex{gid, delta});
+  CHECK(inserted) << "The vertex must be inserted here!";
+  CHECK(it != acc.end()) << "Invalid Vertex accessor!";
+  delta->prev.Set(&*it);
+  return VertexAccessor(&*it, &transaction_, &storage_->indices_,
+                        &storage_->constraints_, config_);
+}
+#endif
+
 std::optional<VertexAccessor> Storage::Accessor::FindVertex(Gid gid,
                                                             View view) {
   auto acc = storage_->vertices_.access();
@@ -625,6 +659,84 @@ Result<EdgeAccessor> Storage::Accessor::CreateEdge(VertexAccessor *from,
                       &storage_->indices_, &storage_->constraints_, config_);
 }
 
+#ifdef MG_ENTERPRISE
+Result<EdgeAccessor> Storage::Accessor::CreateEdge(VertexAccessor *from,
+                                                   VertexAccessor *to,
+                                                   EdgeTypeId edge_type,
+                                                   storage::Gid gid) {
+  CHECK(from->transaction_ == to->transaction_)
+      << "VertexAccessors must be from the same transaction when creating "
+         "an edge!";
+  CHECK(from->transaction_ == &transaction_)
+      << "VertexAccessors must be from the same transaction in when "
+         "creating an edge!";
+
+  auto from_vertex = from->vertex_;
+  auto to_vertex = to->vertex_;
+
+  // Obtain the locks by `gid` order to avoid lock cycles.
+  std::unique_lock<utils::SpinLock> guard_from(from_vertex->lock,
+                                               std::defer_lock);
+  std::unique_lock<utils::SpinLock> guard_to(to_vertex->lock, std::defer_lock);
+  if (from_vertex->gid < to_vertex->gid) {
+    guard_from.lock();
+    guard_to.lock();
+  } else if (from_vertex->gid > to_vertex->gid) {
+    guard_to.lock();
+    guard_from.lock();
+  } else {
+    // The vertices are the same vertex, only lock one.
+    guard_from.lock();
+  }
+
+  if (!PrepareForWrite(&transaction_, from_vertex))
+    return Error::SERIALIZATION_ERROR;
+  if (from_vertex->deleted) return Error::DELETED_OBJECT;
+
+  if (to_vertex != from_vertex) {
+    if (!PrepareForWrite(&transaction_, to_vertex))
+      return Error::SERIALIZATION_ERROR;
+    if (to_vertex->deleted) return Error::DELETED_OBJECT;
+  }
+
+  // NOTE: When we update the next `edge_id_` here we perform a RMW
+  // (read-modify-write) operation that ISN'T atomic! But, that isn't an issue
+  // because this function is only called from the replication delta applier
+  // that runs single-threadedly and while this instance is set-up to apply
+  // threads (it is the replica), it is guaranteed that no other writes are
+  // possible.
+  storage_->edge_id_.store(
+      std::max(storage_->edge_id_.load(std::memory_order_acquire),
+               gid.AsUint() + 1),
+      std::memory_order_release);
+
+  EdgeRef edge(gid);
+  if (config_.properties_on_edges) {
+    auto acc = storage_->edges_.access();
+    auto delta = CreateDeleteObjectDelta(&transaction_);
+    auto [it, inserted] = acc.insert(Edge(gid, delta));
+    CHECK(inserted) << "The edge must be inserted here!";
+    CHECK(it != acc.end()) << "Invalid Edge accessor!";
+    edge = EdgeRef(&*it);
+    delta->prev.Set(&*it);
+  }
+
+  CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(),
+                     edge_type, to_vertex, edge);
+  from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge);
+
+  CreateAndLinkDelta(&transaction_, to_vertex, Delta::RemoveInEdgeTag(),
+                     edge_type, from_vertex, edge);
+  to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge);
+
+  // Increment edge count.
+  storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
+
+  return EdgeAccessor(edge, edge_type, from_vertex, to_vertex, &transaction_,
+                      &storage_->indices_, &storage_->constraints_, config_);
+}
+#endif
+
 Result<bool> Storage::Accessor::DeleteEdge(EdgeAccessor *edge) {
   CHECK(edge->transaction_ == &transaction_)
       << "EdgeAccessor must be from the same transaction as the storage "
@@ -743,7 +855,8 @@ EdgeTypeId Storage::Accessor::NameToEdgeType(const std::string_view &name) {
 
 void Storage::Accessor::AdvanceCommand() { ++transaction_.command_id; }
 
-utils::BasicResult<ConstraintViolation, void> Storage::Accessor::Commit() {
+utils::BasicResult<ConstraintViolation, void> Storage::Accessor::Commit(
+    const std::optional<uint64_t> desired_commit_timestamp) {
   CHECK(is_transaction_active_) << "The transaction is already terminated!";
   CHECK(!transaction_.must_abort) << "The transaction can't be committed!";
 
@@ -780,7 +893,7 @@ utils::BasicResult<ConstraintViolation, void> Storage::Accessor::Commit() {
 
     {
       std::unique_lock<utils::SpinLock> engine_guard(storage_->engine_lock_);
-      commit_timestamp = storage_->timestamp_++;
+      commit_timestamp = storage_->CommitTimestamp(desired_commit_timestamp);
 
       // Before committing and validating vertices against unique constraints,
       // we have to update unique constraints with the vertices that are going
@@ -820,7 +933,17 @@ utils::BasicResult<ConstraintViolation, void> Storage::Accessor::Commit() {
         // written before actually committing the transaction (before setting
         // the commit timestamp) so that no other transaction can see the
         // modifications before they are written to disk.
+#ifdef MG_ENTERPRISE
+        // Replica can log only the write transaction received from Main
+        // so the Wal files are consistent
+        if (storage_->replication_role_ == ReplicationRole::MAIN ||
+            desired_commit_timestamp.has_value()) {
+          storage_->AppendToWal(transaction_, commit_timestamp);
+        }
+#else
+
         storage_->AppendToWal(transaction_, commit_timestamp);
+#endif
 
         // Take committed_transactions lock while holding the engine lock to
         // make sure that committed transactions are sorted by the commit
@@ -833,6 +956,15 @@ utils::BasicResult<ConstraintViolation, void> Storage::Accessor::Commit() {
                   << "Invalid database state!";
               transaction_.commit_timestamp->store(commit_timestamp,
                                                    std::memory_order_release);
+#ifdef MG_ENTERPRISE
+              // Replica can only update the last commit timestamp with
+              // the commits received from main.
+              if (storage_->replication_role_ == ReplicationRole::MAIN ||
+                  desired_commit_timestamp.has_value()) {
+                // Update the last commit timestamp
+                storage_->last_commit_timestamp_.store(commit_timestamp);
+              }
+#endif
               // Release engine lock because we don't have to hold it anymore
               // and emplace back could take a long time.
               engine_guard.unlock();
@@ -1060,52 +1192,66 @@ EdgeTypeId Storage::NameToEdgeType(const std::string_view &name) {
   return EdgeTypeId::FromUint(name_id_mapper_.NameToId(name));
 }
 
-bool Storage::CreateIndex(LabelId label) {
+bool Storage::CreateIndex(
+    LabelId label, const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   if (!indices_.label_index.CreateIndex(label, vertices_.access()))
     return false;
-  // Here it is safe to use `timestamp_` as the final commit timestamp of this
-  // operation even though this operation isn't transactional. The `timestamp_`
-  // variable holds the next timestamp that will be used. Because the above
-  // `storage_guard` ensures that no transactions are currently active, the
-  // value of `timestamp_` is guaranteed to be used as a start timestamp for the
-  // next regular transaction after this operation. This prevents collisions of
-  // commit timestamps between non-transactional operations and transactional
-  // operations.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::LABEL_INDEX_CREATE, label, {},
-              timestamp_);
+              commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
-bool Storage::CreateIndex(LabelId label, PropertyId property) {
+bool Storage::CreateIndex(
+    LabelId label, PropertyId property,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   if (!indices_.label_property_index.CreateIndex(label, property,
                                                  vertices_.access()))
     return false;
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_CREATE,
-              label, {property}, timestamp_);
+              label, {property}, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
-bool Storage::DropIndex(LabelId label) {
+bool Storage::DropIndex(
+    LabelId label, const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   if (!indices_.label_index.DropIndex(label)) return false;
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::LABEL_INDEX_DROP, label, {},
-              timestamp_);
+              commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
-bool Storage::DropIndex(LabelId label, PropertyId property) {
+bool Storage::DropIndex(
+    LabelId label, PropertyId property,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   if (!indices_.label_property_index.DropIndex(label, property)) return false;
   // For a description why using `timestamp_` is correct, see
   // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::LABEL_PROPERTY_INDEX_DROP,
-              label, {property}, timestamp_);
+              label, {property}, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
@@ -1116,32 +1262,43 @@ IndicesInfo Storage::ListAllIndices() const {
 }
 
 utils::BasicResult<ConstraintViolation, bool>
-Storage::CreateExistenceConstraint(LabelId label, PropertyId property) {
+Storage::CreateExistenceConstraint(
+    LabelId label, PropertyId property,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   auto ret = ::storage::CreateExistenceConstraint(&constraints_, label,
                                                   property, vertices_.access());
   if (ret.HasError() || !ret.GetValue()) return ret;
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_CREATE,
-              label, {property}, timestamp_);
+              label, {property}, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
-bool Storage::DropExistenceConstraint(LabelId label, PropertyId property) {
+bool Storage::DropExistenceConstraint(
+    LabelId label, PropertyId property,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   if (!::storage::DropExistenceConstraint(&constraints_, label, property))
     return false;
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::EXISTENCE_CONSTRAINT_DROP,
-              label, {property}, timestamp_);
+              label, {property}, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return true;
 }
 
 utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus>
-Storage::CreateUniqueConstraint(LabelId label,
-                                const std::set<PropertyId> &properties) {
+Storage::CreateUniqueConstraint(
+    LabelId label, const std::set<PropertyId> &properties,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   auto ret = constraints_.unique_constraints.CreateConstraint(
       label, properties, vertices_.access());
@@ -1149,24 +1306,31 @@ Storage::CreateUniqueConstraint(LabelId label,
       ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS) {
     return ret;
   }
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_CREATE,
-              label, properties, timestamp_);
+              label, properties, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return UniqueConstraints::CreationStatus::SUCCESS;
 }
 
 UniqueConstraints::DeletionStatus Storage::DropUniqueConstraint(
-    LabelId label, const std::set<PropertyId> &properties) {
+    LabelId label, const std::set<PropertyId> &properties,
+    const std::optional<uint64_t> desired_commit_timestamp) {
   std::unique_lock<utils::RWLock> storage_guard(main_lock_);
   auto ret = constraints_.unique_constraints.DropConstraint(label, properties);
   if (ret != UniqueConstraints::DeletionStatus::SUCCESS) {
     return ret;
   }
-  // For a description why using `timestamp_` is correct, see
-  // `CreateIndex(LabelId label)`.
+  const auto commit_timestamp = CommitTimestamp(desired_commit_timestamp);
   AppendToWal(durability::StorageGlobalOperation::UNIQUE_CONSTRAINT_DROP, label,
-              properties, timestamp_);
+              properties, commit_timestamp);
+  commit_log_.MarkFinished(commit_timestamp);
+#ifdef MG_ENTERPRISE
+  last_commit_timestamp_ = commit_timestamp;
+#endif
   return UniqueConstraints::DeletionStatus::SUCCESS;
 }
 
@@ -1223,7 +1387,21 @@ Transaction Storage::CreateTransaction() {
   {
     std::lock_guard<utils::SpinLock> guard(engine_lock_);
     transaction_id = transaction_id_++;
+#ifdef MG_ENTERPRISE
+    // Replica should have only read queries and the write queries
+    // can come from main instance with any past timestamp.
+    // To preserve snapshot isolation we set the start timestamp
+    // of any query on replica to the last commited transaction
+    // which is timestamp_ as only commit of transaction with writes
+    // can change the value of it.
+    if (replication_role_ == ReplicationRole::REPLICA) {
+      start_timestamp = timestamp_;
+    } else {
+      start_timestamp = timestamp_++;
+    }
+#else
     start_timestamp = timestamp_++;
+#endif
   }
   return {transaction_id, start_timestamp};
 }
@@ -1349,7 +1527,7 @@ void Storage::CollectGarbage() {
             break;
           }
           case PreviousPtr::Type::DELTA: {
-            if (prev.delta->timestamp->load(std::memory_order_release) ==
+            if (prev.delta->timestamp->load(std::memory_order_acquire) ==
                 commit_timestamp) {
               // The delta that is newer than this one is also a delta from this
               // transaction. We skip the current delta and will remove it as a
@@ -1463,8 +1641,8 @@ bool Storage::InitializeWalFile() {
       Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL)
     return false;
   if (!wal_file_) {
-    wal_file_.emplace(wal_directory_, uuid_, config_.items, &name_id_mapper_,
-                      wal_seq_num_++);
+    wal_file_.emplace(wal_directory_, uuid_, epoch_id_, config_.items,
+                      &name_id_mapper_, wal_seq_num_++, &file_retainer_);
   }
   return true;
 }
@@ -1478,8 +1656,15 @@ void Storage::FinalizeWalFile() {
   }
   if (wal_file_->GetSize() / 1024 >=
       config_.durability.wal_file_size_kibibytes) {
+    wal_file_->FinalizeWal();
     wal_file_ = std::nullopt;
     wal_unsynced_transactions_ = 0;
+  } else {
+    // Try writing the internal buffer if possible, if not
+    // the data should be written as soon as it's possible
+    // (triggered by the new transaction commit, or some
+    // reading thread EnabledFlushing)
+    wal_file_->TryFlushing();
   }
 }
 
@@ -1490,6 +1675,17 @@ void Storage::AppendToWal(const Transaction &transaction,
   // A single transaction will always be contained in a single WAL file.
   auto current_commit_timestamp =
       transaction.commit_timestamp->load(std::memory_order_acquire);
+
+#ifdef MG_ENTERPRISE
+  if (replication_role_.load() == ReplicationRole::MAIN) {
+    replication_clients_.WithLock([&](auto &clients) {
+      for (auto &client : clients) {
+        client->StartTransactionReplication(wal_file_->SequenceNumber());
+      }
+    });
+  }
+#endif
+
   // Helper lambda that traverses the delta chain on order to find the first
   // delta that should be processed and then appends all discovered deltas.
   auto find_and_apply_deltas = [&](const auto *delta, const auto &parent,
@@ -1505,6 +1701,15 @@ void Storage::AppendToWal(const Transaction &transaction,
     while (true) {
       if (filter(delta->action)) {
         wal_file_->AppendDelta(*delta, parent, final_commit_timestamp);
+#ifdef MG_ENTERPRISE
+        replication_clients_.WithLock([&](auto &clients) {
+          for (auto &client : clients) {
+            client->IfStreamingTransaction([&](auto &stream) {
+              stream.AppendDelta(*delta, parent, final_commit_timestamp);
+            });
+          }
+        });
+#endif
       }
       auto prev = delta->prev.Get();
       if (prev.type != PreviousPtr::Type::DELTA) break;
@@ -1637,6 +1842,17 @@ void Storage::AppendToWal(const Transaction &transaction,
   wal_file_->AppendTransactionEnd(final_commit_timestamp);
 
   FinalizeWalFile();
+
+#ifdef MG_ENTERPRISE
+  replication_clients_.WithLock([&](auto &clients) {
+    for (auto &client : clients) {
+      client->IfStreamingTransaction([&](auto &stream) {
+        stream.AppendTransactionEnd(final_commit_timestamp);
+      });
+      client->FinalizeTransactionReplication();
+    }
+  });
+#endif
 }
 
 void Storage::AppendToWal(durability::StorageGlobalOperation operation,
@@ -1645,7 +1861,193 @@ void Storage::AppendToWal(durability::StorageGlobalOperation operation,
   if (!InitializeWalFile()) return;
   wal_file_->AppendOperation(operation, label, properties,
                              final_commit_timestamp);
+#ifdef MG_ENTERPRISE
+  {
+    if (replication_role_.load() == ReplicationRole::MAIN) {
+      replication_clients_.WithLock([&](auto &clients) {
+        for (auto &client : clients) {
+          client->StartTransactionReplication(wal_file_->SequenceNumber());
+          client->IfStreamingTransaction([&](auto &stream) {
+            stream.AppendOperation(operation, label, properties,
+                                   final_commit_timestamp);
+          });
+          client->FinalizeTransactionReplication();
+        }
+      });
+    }
+  }
+#endif
   FinalizeWalFile();
 }
 
+void Storage::CreateSnapshot() {
+#ifdef MG_ENTERPRISE
+  if (replication_role_.load() != ReplicationRole::MAIN) {
+    LOG(WARNING) << "Snapshots are disabled for replicas!";
+    return;
+  }
+#endif
+
+  // Take master RW lock (for reading).
+  std::shared_lock<utils::RWLock> storage_guard(main_lock_);
+
+  // Create the transaction used to create the snapshot.
+  auto transaction = CreateTransaction();
+
+  // Create snapshot.
+  durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_,
+                             config_.durability.snapshot_retention_count,
+                             &vertices_, &edges_, &name_id_mapper_, &indices_,
+                             &constraints_, config_.items, uuid_, epoch_id_,
+                             epoch_history_, &file_retainer_);
+
+  // Finalize snapshot transaction.
+  commit_log_.MarkFinished(transaction.start_timestamp);
+}
+
+uint64_t Storage::CommitTimestamp(
+    const std::optional<uint64_t> desired_commit_timestamp) {
+#ifdef MG_ENTERPRISE
+  if (!desired_commit_timestamp) {
+    return timestamp_++;
+  } else {
+    const auto commit_timestamp = *desired_commit_timestamp;
+    timestamp_ = std::max(timestamp_, *desired_commit_timestamp + 1);
+    return commit_timestamp;
+  }
+#else
+  return timestamp_++;
+#endif
+}
+
+#ifdef MG_ENTERPRISE
+bool Storage::SetReplicaRole(
+    io::network::Endpoint endpoint,
+    const replication::ReplicationServerConfig &config) {
+  // We don't want to restart the server if we're already a REPLICA
+  if (replication_role_ == ReplicationRole::REPLICA) {
+    return false;
+  }
+
+  replication_server_ =
+      std::make_unique<ReplicationServer>(this, std::move(endpoint), config);
+
+  replication_role_.store(ReplicationRole::REPLICA);
+  return true;
+}
+
+bool Storage::SetMainReplicationRole() {
+  // We don't want to generate new epoch_id and do the
+  // cleanup if we're already a MAIN
+  if (replication_role_ == ReplicationRole::MAIN) {
+    return false;
+  }
+
+  // Main instance does not need replication server
+  // This should be always called first so we finalize everything
+  replication_server_.reset(nullptr);
+
+  {
+    std::unique_lock engine_guard{engine_lock_};
+    if (wal_file_) {
+      wal_file_->FinalizeWal();
+      wal_file_.reset();
+    }
+
+    // Generate new epoch id and save the last one to the history.
+    if (epoch_history_.size() == kEpochHistoryRetention) {
+      epoch_history_.pop_front();
+    }
+    epoch_history_.emplace_back(std::move(epoch_id_), last_commit_timestamp_);
+    epoch_id_ = utils::GenerateUUID();
+  }
+
+  replication_role_.store(ReplicationRole::MAIN);
+  return true;
+}
+
+utils::BasicResult<Storage::RegisterReplicaError> Storage::RegisterReplica(
+    std::string name, io::network::Endpoint endpoint,
+    const replication::ReplicationMode replication_mode,
+    const replication::ReplicationClientConfig &config) {
+  CHECK(replication_role_.load() == ReplicationRole::MAIN)
+      << "Only main instance can register a replica!";
+
+  const bool name_exists = replication_clients_.WithLock([&](auto &clients) {
+    return std::any_of(clients.begin(), clients.end(),
+                       [&](auto &client) { return client->Name() == name; });
+  });
+
+  if (name_exists) {
+    return RegisterReplicaError::NAME_EXISTS;
+  }
+
+  CHECK(replication_mode == replication::ReplicationMode::SYNC ||
+        !config.timeout)
+      << "Only SYNC mode can have a timeout set";
+
+  auto client = std::make_unique<ReplicationClient>(
+      std::move(name), this, endpoint, replication_mode, config);
+  if (client->State() == replication::ReplicaState::INVALID) {
+    return RegisterReplicaError::CONNECTION_FAILED;
+  }
+
+  return replication_clients_.WithLock(
+      [&](auto &clients) -> utils::BasicResult<Storage::RegisterReplicaError> {
+        // Another thread could have added a client with same name while
+        // we were connecting to this client.
+        if (std::any_of(clients.begin(), clients.end(),
+                        [&](auto &other_client) {
+                          return client->Name() == other_client->Name();
+                        })) {
+          return RegisterReplicaError::NAME_EXISTS;
+        }
+
+        clients.push_back(std::move(client));
+        return {};
+      });
+}
+
+bool Storage::UnregisterReplica(const std::string_view name) {
+  CHECK(replication_role_.load() == ReplicationRole::MAIN)
+      << "Only main instance can unregister a replica!";
+  return replication_clients_.WithLock([&](auto &clients) {
+    return std::erase_if(
+        clients, [&](const auto &client) { return client->Name() == name; });
+  });
+}
+
+std::optional<replication::ReplicaState> Storage::GetReplicaState(
+    const std::string_view name) {
+  return replication_clients_.WithLock(
+      [&](auto &clients) -> std::optional<replication::ReplicaState> {
+        const auto client_it = std::find_if(
+            clients.cbegin(), clients.cend(),
+            [name](auto &client) { return client->Name() == name; });
+        if (client_it == clients.cend()) {
+          return std::nullopt;
+        }
+        return (*client_it)->State();
+      });
+}
+
+ReplicationRole Storage::GetReplicationRole() const {
+  return replication_role_;
+}
+
+std::vector<Storage::ReplicaInfo> Storage::ReplicasInfo() {
+  return replication_clients_.WithLock([](auto &clients) {
+    std::vector<Storage::ReplicaInfo> replica_info;
+    replica_info.reserve(clients.size());
+    std::transform(clients.begin(), clients.end(),
+                   std::back_inserter(replica_info),
+                   [](const auto &client) -> ReplicaInfo {
+                     return {client->Name(), client->Mode(), client->Timeout(),
+                             client->Endpoint(), client->State()};
+                   });
+    return replica_info;
+  });
+}
+#endif
+
 }  // namespace storage

src/storage/v2/storage.hpp

@@ -1,12 +1,15 @@
 #pragma once
 
+#include <atomic>
 #include <filesystem>
 #include <optional>
 #include <shared_mutex>
 
+#include "io/network/endpoint.hpp"
 #include "storage/v2/commit_log.hpp"
 #include "storage/v2/config.hpp"
 #include "storage/v2/constraints.hpp"
+#include "storage/v2/durability/metadata.hpp"
 #include "storage/v2/durability/wal.hpp"
 #include "storage/v2/edge.hpp"
 #include "storage/v2/edge_accessor.hpp"
@@ -17,10 +20,20 @@
 #include "storage/v2/transaction.hpp"
 #include "storage/v2/vertex.hpp"
 #include "storage/v2/vertex_accessor.hpp"
+#include "utils/file_locker.hpp"
 #include "utils/rw_lock.hpp"
 #include "utils/scheduler.hpp"
 #include "utils/skip_list.hpp"
 #include "utils/synchronized.hpp"
+#include "utils/uuid.hpp"
+
+#ifdef MG_ENTERPRISE
+#include "rpc/server.hpp"
+#include "storage/v2/replication/config.hpp"
+#include "storage/v2/replication/enums.hpp"
+#include "storage/v2/replication/rpc.hpp"
+#include "storage/v2/replication/serialization.hpp"
+#endif
 
 namespace storage {
 
@@ -159,6 +172,10 @@ struct StorageInfo {
   uint64_t disk_usage;
 };
 
+#ifdef MG_ENTERPRISE
+enum class ReplicationRole : uint8_t { MAIN, REPLICA };
+#endif
+
 class Storage final {
  public:
   /// @throw std::system_error
@@ -299,12 +316,22 @@ class Storage final {
     /// transaction violate an existence or unique constraint. In that case the
     /// transaction is automatically aborted. Otherwise, void is returned.
     /// @throw std::bad_alloc
-    utils::BasicResult<ConstraintViolation, void> Commit();
+    utils::BasicResult<ConstraintViolation, void> Commit(
+        std::optional<uint64_t> desired_commit_timestamp = {});
 
     /// @throw std::bad_alloc
     void Abort();
 
    private:
+#ifdef MG_ENTERPRISE
+    /// @throw std::bad_alloc
+    VertexAccessor CreateVertex(storage::Gid gid);
+
+    /// @throw std::bad_alloc
+    Result<EdgeAccessor> CreateEdge(VertexAccessor *from, VertexAccessor *to,
+                                    EdgeTypeId edge_type, storage::Gid gid);
+#endif
+
     Storage *storage_;
     std::shared_lock<utils::RWLock> storage_guard_;
     Transaction transaction_;
@@ -328,14 +355,18 @@ class Storage final {
   EdgeTypeId NameToEdgeType(const std::string_view &name);
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label);
+  bool CreateIndex(LabelId label,
+                   std::optional<uint64_t> desired_commit_timestamp = {});
 
   /// @throw std::bad_alloc
-  bool CreateIndex(LabelId label, PropertyId property);
+  bool CreateIndex(LabelId label, PropertyId property,
+                   std::optional<uint64_t> desired_commit_timestamp = {});
 
-  bool DropIndex(LabelId label);
+  bool DropIndex(LabelId label,
+                 std::optional<uint64_t> desired_commit_timestamp = {});
 
-  bool DropIndex(LabelId label, PropertyId property);
+  bool DropIndex(LabelId label, PropertyId property,
+                 std::optional<uint64_t> desired_commit_timestamp = {});
 
   IndicesInfo ListAllIndices() const;
 
@@ -346,11 +377,14 @@ class Storage final {
   /// @throw std::bad_alloc
   /// @throw std::length_error
   utils::BasicResult<ConstraintViolation, bool> CreateExistenceConstraint(
-      LabelId label, PropertyId property);
+      LabelId label, PropertyId property,
+      std::optional<uint64_t> desired_commit_timestamp = {});
 
   /// Removes an existence constraint. Returns true if the constraint was
   /// removed, and false if it doesn't exist.
-  bool DropExistenceConstraint(LabelId label, PropertyId property);
+  bool DropExistenceConstraint(
+      LabelId label, PropertyId property,
+      std::optional<uint64_t> desired_commit_timestamp = {});
 
   /// Creates a unique constraint. In the case of two vertices violating the
   /// constraint, it returns `ConstraintViolation`. Otherwise returns a
@@ -363,7 +397,8 @@ class Storage final {
   ///
   /// @throw std::bad_alloc
   utils::BasicResult<ConstraintViolation, UniqueConstraints::CreationStatus>
-  CreateUniqueConstraint(LabelId label, const std::set<PropertyId> &properties);
+  CreateUniqueConstraint(LabelId label, const std::set<PropertyId> &properties,
+                         std::optional<uint64_t> desired_commit_timestamp = {});
 
   /// Removes a unique constraint. Returns `UniqueConstraints::DeletionStatus`
   /// enum with the following possibilities:
@@ -373,12 +408,47 @@ class Storage final {
   ///     * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the
   //        limit of maximum number of properties.
   UniqueConstraints::DeletionStatus DropUniqueConstraint(
-      LabelId label, const std::set<PropertyId> &properties);
+      LabelId label, const std::set<PropertyId> &properties,
+      std::optional<uint64_t> desired_commit_timestamp = {});
 
   ConstraintsInfo ListAllConstraints() const;
 
   StorageInfo GetInfo() const;
 
+#if MG_ENTERPRISE
+
+  bool SetReplicaRole(io::network::Endpoint endpoint,
+                      const replication::ReplicationServerConfig &config = {});
+
+  bool SetMainReplicationRole();
+
+  enum class RegisterReplicaError : uint8_t { NAME_EXISTS, CONNECTION_FAILED };
+
+  /// @pre The instance should have a MAIN role
+  /// @pre Timeout can only be set for SYNC replication
+  utils::BasicResult<RegisterReplicaError, void> RegisterReplica(
+      std::string name, io::network::Endpoint endpoint,
+      replication::ReplicationMode replication_mode,
+      const replication::ReplicationClientConfig &config = {});
+  /// @pre The instance should have a MAIN role
+  bool UnregisterReplica(std::string_view name);
+
+  std::optional<replication::ReplicaState> GetReplicaState(
+      std::string_view name);
+
+  ReplicationRole GetReplicationRole() const;
+
+  struct ReplicaInfo {
+    std::string name;
+    replication::ReplicationMode mode;
+    std::optional<double> timeout;
+    io::network::Endpoint endpoint;
+    replication::ReplicaState state;
+  };
+
+  std::vector<ReplicaInfo> ReplicasInfo();
+#endif
+
  private:
   Transaction CreateTransaction();
 
@@ -395,6 +465,14 @@ class Storage final {
                    const std::set<PropertyId> &properties,
                    uint64_t final_commit_timestamp);
 
+  void CreateSnapshot();
+
+  uint64_t CommitTimestamp(
+      std::optional<uint64_t> desired_commit_timestamp = {});
+
+#ifdef MG_ENTERPRISE
+#endif
+
   // Main storage lock.
   //
   // Accessors take a shared lock when starting, so it is possible to block
@@ -465,8 +543,57 @@ class Storage final {
   // Sequence number used to keep track of the chain of WALs.
   uint64_t wal_seq_num_{0};
 
+  // UUID to distinguish different main instance runs for replication process
+  // on SAME storage.
+  // Multiple instances can have same storage UUID and be MAIN at the same time.
+  // We cannot compare commit timestamps of those instances if one of them
+  // becomes the replica of the other so we use epoch_id_ as additional
+  // discriminating property.
+  // Example of this:
+  // We have 2 instances of the same storage, S1 and S2.
+  // S1 and S2 are MAIN and accept their own commits and write them to the WAL.
+  // At the moment when S1 commited a transaction with timestamp 20, and S2
+  // a different transaction with timestamp 15, we change S2's role to REPLICA
+  // and register it on S1.
+  // Without using the epoch_id, we don't know that S1 and S2 have completely
+  // different transactions, we think that the S2 is behind only by 5 commits.
+  std::string epoch_id_;
+  // History of the previous epoch ids.
+  // Each value consists of the epoch id along the last commit belonging to that
+  // epoch.
+  std::deque<std::pair<std::string, uint64_t>> epoch_history_;
+
   std::optional<durability::WalFile> wal_file_;
   uint64_t wal_unsynced_transactions_{0};
+
+  utils::FileRetainer file_retainer_;
+
+  // Replication
+#ifdef MG_ENTERPRISE
+  // Last commited timestamp
+  std::atomic<uint64_t> last_commit_timestamp_{kTimestampInitialId};
+
+  class ReplicationServer;
+  std::unique_ptr<ReplicationServer> replication_server_{nullptr};
+
+  class ReplicationClient;
+  // We create ReplicationClient using unique_ptr so we can move
+  // newly created client into the vector.
+  // We cannot move the client directly because it contains ThreadPool
+  // which cannot be moved. Also, the move is necessary because
+  // we don't want to create the client directly inside the vector
+  // because that would require the lock on the list putting all
+  // commits (they iterate list of clients) to halt.
+  // This way we can initialize client in main thread which means
+  // that we can immediately notify the user if the initialization
+  // failed.
+  using ReplicationClientList =
+      utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>,
+                          utils::SpinLock>;
+  ReplicationClientList replication_clients_;
+
+  std::atomic<ReplicationRole> replication_role_{ReplicationRole::MAIN};
+#endif
 };
 
 }  // namespace storage

src/utils/CMakeLists.txt

@@ -1,8 +1,10 @@
 set(utils_src_files
     file.cpp
+    file_locker.cpp
     memory.cpp
     signals.cpp
     thread.cpp
+    thread_pool.cpp
     uuid.cpp)
 
 add_library(mg-utils STATIC ${utils_src_files})

src/utils/file.cpp

@@ -7,6 +7,8 @@
 
 #include <cstring>
 #include <fstream>
+#include <mutex>
+#include <shared_mutex>
 #include <type_traits>
 
 #include <glog/logging.h>
@@ -289,9 +291,9 @@ OutputFile::~OutputFile() {
 OutputFile::OutputFile(OutputFile &&other) noexcept
     : fd_(other.fd_),
       written_since_last_sync_(other.written_since_last_sync_),
-      path_(std::move(other.path_)),
-      buffer_position_(other.buffer_position_) {
+      path_(std::move(other.path_)) {
   memcpy(buffer_, other.buffer_, kFileBufferSize);
+  buffer_position_.store(other.buffer_position_.load());
   other.fd_ = -1;
   other.written_since_last_sync_ = 0;
   other.buffer_position_ = 0;
@@ -303,7 +305,7 @@ OutputFile &OutputFile::operator=(OutputFile &&other) noexcept {
   fd_ = other.fd_;
   written_since_last_sync_ = other.written_since_last_sync_;
   path_ = std::move(other.path_);
-  buffer_position_ = other.buffer_position_;
+  buffer_position_ = other.buffer_position_.load();
   memcpy(buffer_, other.buffer_, kFileBufferSize);
 
   other.fd_ = -1;
@@ -350,13 +352,21 @@ const std::filesystem::path &OutputFile::path() const { return path_; }
 void OutputFile::Write(const uint8_t *data, size_t size) {
   while (size > 0) {
     FlushBuffer(false);
-    auto buffer_left = kFileBufferSize - buffer_position_;
-    auto to_write = size < buffer_left ? size : buffer_left;
-    memcpy(buffer_ + buffer_position_, data, to_write);
-    size -= to_write;
-    data += to_write;
-    buffer_position_ += to_write;
-    written_since_last_sync_ += to_write;
+    {
+      // Reading thread can call EnableFlushing which triggers
+      // TryFlushing.
+      // We can't use a single shared lock for the entire Write
+      // because FlushBuffer acquires the unique_lock.
+      std::shared_lock flush_guard(flush_lock_);
+      const size_t buffer_position = buffer_position_.load();
+      auto buffer_left = kFileBufferSize - buffer_position;
+      auto to_write = size < buffer_left ? size : buffer_left;
+      memcpy(buffer_ + buffer_position, data, to_write);
+      size -= to_write;
+      data += to_write;
+      buffer_position_.fetch_add(to_write);
+      written_since_last_sync_ += to_write;
+    }
   }
 }
 
@@ -367,13 +377,7 @@ void OutputFile::Write(const std::string_view &data) {
   Write(data.data(), data.size());
 }
 
-size_t OutputFile::GetPosition() {
-  return SetPosition(Position::RELATIVE_TO_CURRENT, 0);
-}
-
-size_t OutputFile::SetPosition(Position position, ssize_t offset) {
-  FlushBuffer(true);
-
+size_t OutputFile::SeekFile(const Position position, const ssize_t offset) {
   int whence;
   switch (position) {
     case Position::SET:
@@ -398,6 +402,15 @@ size_t OutputFile::SetPosition(Position position, ssize_t offset) {
   }
 }
 
+size_t OutputFile::GetPosition() {
+  return SetPosition(Position::RELATIVE_TO_CURRENT, 0);
+}
+
+size_t OutputFile::SetPosition(Position position, ssize_t offset) {
+  FlushBuffer(true);
+  return SeekFile(position, offset);
+}
+
 bool OutputFile::AcquireLock() {
   CHECK(IsOpen()) << "Trying to acquire a write lock on an unopened file!";
   int ret = -1;
@@ -497,14 +510,20 @@ void OutputFile::Close() noexcept {
 void OutputFile::FlushBuffer(bool force_flush) {
   CHECK(IsOpen());
 
-  if (!force_flush && buffer_position_ < kFileBufferSize) return;
+  if (!force_flush && buffer_position_.load() < kFileBufferSize) return;
 
+  std::unique_lock flush_guard(flush_lock_);
+  FlushBufferInternal();
+}
+
+void OutputFile::FlushBufferInternal() {
   CHECK(buffer_position_ <= kFileBufferSize)
       << "While trying to write to " << path_
       << " more file was written to the buffer than the buffer has space!";
 
   auto *buffer = buffer_;
-  while (buffer_position_ > 0) {
+  auto buffer_position = buffer_position_.load();
+  while (buffer_position > 0) {
     auto written = write(fd_, buffer, buffer_position_);
     if (written == -1 && errno == EINTR) {
       continue;
@@ -517,9 +536,40 @@ void OutputFile::FlushBuffer(bool force_flush) {
         << " bytes of data were lost from this call and possibly "
         << written_since_last_sync_ << " bytes were lost from previous calls.";
 
-    buffer_position_ -= written;
+    buffer_position -= written;
     buffer += written;
   }
+
+  buffer_position_.store(buffer_position);
+}
+
+void OutputFile::DisableFlushing() { flush_lock_.lock_shared(); }
+
+void OutputFile::EnableFlushing() {
+  flush_lock_.unlock_shared();
+  TryFlushing();
+}
+
+std::pair<const uint8_t *, size_t> OutputFile::CurrentBuffer() const {
+  return {buffer_, buffer_position_.load()};
+}
+
+size_t OutputFile::GetSize() {
+  // There's an alternative way of fetching the files size using fstat.
+  // lseek should be faster for smaller number of clients while fstat
+  // should have an advantage for high number of clients.
+  // The reason for this is the way those functions implement the
+  // support for multi-threading. While lseek uses locks, fstat is lockfree.
+  // For now, lseek should be good enough. If at any point this proves to
+  // be a bottleneck, fstat should be considered.
+  return SeekFile(Position::RELATIVE_TO_END, 0) + buffer_position_.load();
+}
+
+void OutputFile::TryFlushing() {
+  if (std::unique_lock guard(flush_lock_, std::try_to_lock);
+      guard.owns_lock()) {
+    FlushBufferInternal();
+  }
 }
 
 }  // namespace utils

src/utils/file.hpp

@@ -6,12 +6,15 @@
  */
 #pragma once
 
+#include <atomic>
 #include <filesystem>
 #include <optional>
 #include <string>
 #include <string_view>
 #include <vector>
 
+#include "utils/rw_lock.hpp"
+
 namespace utils {
 
 /// Get the path of the current executable.
@@ -152,7 +155,11 @@ class InputFile {
 /// written to permanent storage.
 ///
 /// This class *isn't* thread safe. It is implemented as a wrapper around low
-/// level system calls used for file manipulation.
+/// level system calls used for file manipulation. It allows concurrent
+/// READING of the file that is being written. To read the file, disable the
+/// flushing of the internal buffer using `DisableFlushing`. Don't forget to
+/// enable flushing again after you're done with reading using the
+/// 'EnableFlushing' method!
 class OutputFile {
  public:
   enum class Mode {
@@ -220,14 +227,37 @@ class OutputFile {
   /// file. On failure and misuse it crashes the program.
   void Close() noexcept;
 
+  /// Disable flushing of the internal buffer.
+  void DisableFlushing();
+
+  /// Enable flushing of the internal buffer.
+  /// Before the flushing is enabled, the internal buffer
+  /// is flushed.
+  void EnableFlushing();
+
+  /// Try flushing the internal buffer.
+  void TryFlushing();
+
+  /// Get the internal buffer with its current size.
+  std::pair<const uint8_t *, size_t> CurrentBuffer() const;
+
+  /// Get the size of the file.
+  size_t GetSize();
+
  private:
   void FlushBuffer(bool force_flush);
+  void FlushBufferInternal();
+
+  size_t SeekFile(Position position, ssize_t offset);
 
   int fd_{-1};
   size_t written_since_last_sync_{0};
   std::filesystem::path path_;
   uint8_t buffer_[kFileBufferSize];
-  size_t buffer_position_{0};
+  std::atomic<size_t> buffer_position_{0};
+
+  // Flushing buffer should be a higher priority
+  utils::RWLock flush_lock_{RWLock::Priority::WRITE};
 };
 
 }  // namespace utils

src/utils/file_locker.cpp

@@ -0,0 +1,100 @@
+#include "utils/file_locker.hpp"
+
+namespace utils {
+
+namespace {
+void DeleteFromSystem(const std::filesystem::path &path) {
+  if (!utils::DeleteFile(path)) {
+    LOG(WARNING) << "Couldn't delete file " << path << "!";
+  }
+}
+}  // namespace
+
+////// FileRetainer //////
+void FileRetainer::DeleteFile(const std::filesystem::path &path) {
+  if (active_accessors_.load()) {
+    files_for_deletion_.WithLock([&](auto &files) { files.emplace(path); });
+    return;
+  }
+  std::unique_lock guard(main_lock_);
+  DeleteOrAddToQueue(path);
+}
+
+FileRetainer::FileLocker FileRetainer::AddLocker() {
+  const size_t current_locker_id = next_locker_id_.fetch_add(1);
+  lockers_.WithLock([&](auto &lockers) {
+    lockers.emplace(current_locker_id, std::set<std::filesystem::path>{});
+  });
+  return FileLocker{this, current_locker_id};
+}
+
+FileRetainer::~FileRetainer() {
+  CHECK(files_for_deletion_->empty()) << "Files weren't properly deleted";
+}
+
+[[nodiscard]] bool FileRetainer::FileLocked(const std::filesystem::path &path) {
+  return lockers_.WithLock([&](auto &lockers) {
+    for (const auto &[_, paths] : lockers) {
+      if (paths.count(path)) {
+        return true;
+      }
+    }
+    return false;
+  });
+}
+
+void FileRetainer::DeleteOrAddToQueue(const std::filesystem::path &path) {
+  if (FileLocked(path)) {
+    files_for_deletion_.WithLock([&](auto &files) { files.emplace(path); });
+  } else {
+    DeleteFromSystem(path);
+  }
+}
+
+void FileRetainer::CleanQueue() {
+  files_for_deletion_.WithLock([&](auto &files) {
+    for (auto it = files.cbegin(); it != files.cend();) {
+      if (!FileLocked(*it)) {
+        DeleteFromSystem(*it);
+        it = files.erase(it);
+      } else {
+        ++it;
+      }
+    }
+  });
+}
+
+////// FileLocker //////
+FileRetainer::FileLocker::~FileLocker() {
+  file_retainer_->lockers_.WithLock(
+      [this](auto &lockers) { lockers.erase(locker_id_); });
+  std::unique_lock guard(file_retainer_->main_lock_);
+  file_retainer_->CleanQueue();
+}
+
+FileRetainer::FileLockerAccessor FileRetainer::FileLocker::Access() {
+  return FileLockerAccessor{file_retainer_, locker_id_};
+}
+
+////// FileLockerAccessor //////
+FileRetainer::FileLockerAccessor::FileLockerAccessor(FileRetainer *retainer,
+                                                     size_t locker_id)
+    : file_retainer_{retainer},
+      retainer_guard_{retainer->main_lock_},
+      locker_id_{locker_id} {
+  file_retainer_->active_accessors_.fetch_add(1);
+}
+
+bool FileRetainer::FileLockerAccessor::AddFile(
+    const std::filesystem::path &path) {
+  if (!std::filesystem::exists(path)) return false;
+  file_retainer_->lockers_.WithLock(
+      [&](auto &lockers) { lockers[locker_id_].emplace(path); });
+  return true;
+}
+
+FileRetainer::FileLockerAccessor::~FileLockerAccessor() {
+  file_retainer_->active_accessors_.fetch_sub(1);
+}
+
+}  // namespace utils

src/utils/file_locker.hpp

@@ -0,0 +1,164 @@
+#pragma once
+#include <atomic>
+#include <deque>
+#include <functional>
+#include <map>
+#include <mutex>
+#include <set>
+#include <shared_mutex>
+
+#include "utils/file.hpp"
+#include "utils/rw_lock.hpp"
+#include "utils/spin_lock.hpp"
+#include "utils/synchronized.hpp"
+
+namespace utils {
+
+/**
+ * Helper class used for safer modifying and reading of files
+ * by preventing a deletion of a file until the file is not used in any of
+ * currently running threads.
+ * Also, while a single thread modyifies it's list of locked files, the deletion
+ * of ALL the files is delayed.
+ *
+ * Basic usage of FileRetainer consists of following parts:
+ * - Defining a global FileRetainer object which is used for locking and
+ *   deleting of the files.
+ * - Each thread that wants to lock a single or multiple files first creates a
+ *   FileLocker object.
+ * - Modifying a FileLocker is only possible through the FileLockerAccessor.
+ * - FileLockerAccessor prevents deletion of any file, so you can safely add
+ *   multiple files to the locker with no risk of having files deleted during
+ *   the process.
+ * - After a FileLocker or FileLockerAccessor is destroyed, FileRetainer scans
+ *   the list of the files that wait to be deleted, and deletes all the files
+ *   that are not inside any of currently present lockers.
+ *
+ * e.g.
+ * FileRetainer file_retainer;
+ * std::filesystem::path file1;
+ * std::filesystem::path file2;
+ *
+ * void Foo() {
+ *   // I want to lock a list of files
+ *   // Create a locker
+ *   auto locker = file_retainer.AddLocker();
+ *   {
+ *     // Create accessor to the locker so you can
+ *     // add the files which need to be locked.
+ *     // Accesor prevents deletion of any files
+ *     // so you safely add multiple files in atomic way
+ *     auto accessor = locker.Access();
+ *     accessor.AddFile(file1);
+ *     accessor.AddFile(file2);
+ *   }
+ *   // DO SOMETHING WITH THE FILES
+ * }
+ *
+ * void Bar() {
+ *   // I want to delete file1.
+ *   file_retiner.DeleteFile(file1);
+ * }
+ *
+ * int main() {
+ * // Run Foo() and Bar() in different threads.
+ * }
+ *
+ */
+class FileRetainer {
+ public:
+  struct FileLockerAccessor;
+
+  /**
+   * A single locker inside the FileRetainer that contains a list
+   * of files that are guarded from deletion.
+   */
+  struct FileLocker {
+    friend FileRetainer;
+    ~FileLocker();
+
+    /**
+     * Access the FileLocker so you can modify it.
+     */
+    FileLockerAccessor Access();
+
+    FileLocker(const FileLocker &) = delete;
+    FileLocker(FileLocker &&) = default;
+    FileLocker &operator=(const FileLocker &) = delete;
+    FileLocker &operator=(FileLocker &&) = default;
+
+   private:
+    explicit FileLocker(FileRetainer *retainer, size_t locker_id)
+        : file_retainer_{retainer}, locker_id_{locker_id} {}
+
+    FileRetainer *file_retainer_;
+    size_t locker_id_;
+  };
+
+  /**
+   * Accessor to the FileLocker.
+   * All the modification to the FileLocker are done
+   * using this struct.
+   */
+  struct FileLockerAccessor {
+    friend FileLocker;
+
+    /**
+     * Add a single file to the current locker.
+     */
+    bool AddFile(const std::filesystem::path &path);
+
+    FileLockerAccessor(const FileLockerAccessor &) = delete;
+    FileLockerAccessor(FileLockerAccessor &&) = default;
+    FileLockerAccessor &operator=(const FileLockerAccessor &) = delete;
+    FileLockerAccessor &operator=(FileLockerAccessor &&) = default;
+
+    ~FileLockerAccessor();
+
+   private:
+    explicit FileLockerAccessor(FileRetainer *retainer, size_t locker_id);
+
+    FileRetainer *file_retainer_;
+    std::shared_lock<utils::RWLock> retainer_guard_;
+    size_t locker_id_;
+  };
+
+  /**
+   * Delete a file.
+   * If the file is inside any of the lockers or some thread is modifying
+   * any of the lockers, the file will be deleted after all the locks are
+   * lifted.
+   */
+  void DeleteFile(const std::filesystem::path &path);
+
+  /**
+   * Create and return a new locker.
+   */
+  FileLocker AddLocker();
+
+  explicit FileRetainer() = default;
+  FileRetainer(const FileRetainer &) = delete;
+  FileRetainer(FileRetainer &&) = delete;
+  FileRetainer &operator=(const FileRetainer &) = delete;
+  FileRetainer &operator=(FileRetainer &&) = delete;
+
+  ~FileRetainer();
+
+ private:
+  [[nodiscard]] bool FileLocked(const std::filesystem::path &path);
+  void DeleteOrAddToQueue(const std::filesystem::path &path);
+  void CleanQueue();
+
+  utils::RWLock main_lock_{RWLock::Priority::WRITE};
+
+  std::atomic<size_t> active_accessors_{0};
+  std::atomic<size_t> next_locker_id_{0};
+  utils::Synchronized<std::map<size_t, std::set<std::filesystem::path>>,
+                      utils::SpinLock>
+      lockers_;
+
+  utils::Synchronized<std::set<std::filesystem::path>, utils::SpinLock>
+      files_for_deletion_;
+};
+
+}  // namespace utils

src/utils/result.hpp

@@ -7,7 +7,7 @@
 
 namespace utils {
 
-template <class TError, class TValue>
+template <class TError, class TValue = void>
 class [[nodiscard]] BasicResult final {
  public:
   BasicResult(const TValue &value) : value_(value) {}

src/utils/synchronized.hpp

@@ -79,7 +79,7 @@ class Synchronized {
   LockedPtr Lock() { return LockedPtr(&object_, &mutex_); }
 
   template <class TCallable>
-  auto WithLock(TCallable &&callable) {
+  decltype(auto) WithLock(TCallable &&callable) {
     return callable(*Lock());
   }
 

src/utils/thread_pool.cpp

@@ -0,0 +1,82 @@
+#include "utils/thread_pool.hpp"
+
+namespace utils {
+
+ThreadPool::ThreadPool(const size_t pool_size) {
+  for (size_t i = 0; i < pool_size; ++i) {
+    thread_pool_.emplace_back(([this] { this->ThreadLoop(); }));
+  }
+}
+
+void ThreadPool::AddTask(std::function<void()> new_task) {
+  task_queue_.WithLock([&](auto &queue) {
+    queue.emplace(std::make_unique<TaskSignature>(std::move(new_task)));
+    unfinished_tasks_num_.fetch_add(1);
+  });
+  std::unique_lock pool_guard(pool_lock_);
+  queue_cv_.notify_one();
+}
+
+void ThreadPool::Shutdown() {
+  terminate_pool_.store(true);
+  {
+    std::unique_lock pool_guard(pool_lock_);
+    queue_cv_.notify_all();
+  }
+
+  for (auto &thread : thread_pool_) {
+    if (thread.joinable()) {
+      thread.join();
+    }
+  }
+
+  thread_pool_.clear();
+  stopped_.store(true);
+}
+
+ThreadPool::~ThreadPool() {
+  if (!stopped_.load()) {
+    Shutdown();
+  }
+}
+
+std::unique_ptr<ThreadPool::TaskSignature> ThreadPool::PopTask() {
+  return task_queue_.WithLock(
+      [](auto &queue) -> std::unique_ptr<TaskSignature> {
+        if (queue.empty()) {
+          return nullptr;
+        }
+        auto front = std::move(queue.front());
+        queue.pop();
+        return front;
+      });
+}
+
+void ThreadPool::ThreadLoop() {
+  std::unique_ptr<TaskSignature> task = PopTask();
+  while (true) {
+    while (task) {
+      if (terminate_pool_.load()) {
+        return;
+      }
+      (*task)();
+      unfinished_tasks_num_.fetch_sub(1);
+      task = PopTask();
+    }
+
+    std::unique_lock guard(pool_lock_);
+    queue_cv_.wait(guard, [&] {
+      task = PopTask();
+      return task || terminate_pool_.load();
+    });
+    if (terminate_pool_.load()) {
+      return;
+    }
+  }
+}
+
+size_t ThreadPool::UnfinishedTasksNum() const {
+  return unfinished_tasks_num_.load();
+}
+
+}  // namespace utils

src/utils/thread_pool.hpp

@@ -0,0 +1,51 @@
+#pragma once
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <queue>
+#include <thread>
+
+#include "utils/spin_lock.hpp"
+#include "utils/synchronized.hpp"
+#include "utils/thread.hpp"
+
+namespace utils {
+
+class ThreadPool {
+  using TaskSignature = std::function<void()>;
+
+ public:
+  explicit ThreadPool(size_t pool_size);
+
+  void AddTask(std::function<void()> new_task);
+
+  void Shutdown();
+
+  ~ThreadPool();
+
+  ThreadPool(const ThreadPool &) = delete;
+  ThreadPool(ThreadPool &&) = delete;
+  ThreadPool &operator=(const ThreadPool &) = delete;
+  ThreadPool &operator=(ThreadPool &&) = delete;
+
+  size_t UnfinishedTasksNum() const;
+
+ private:
+  std::unique_ptr<TaskSignature> PopTask();
+
+  void ThreadLoop();
+
+  std::vector<std::thread> thread_pool_;
+
+  std::atomic<size_t> unfinished_tasks_num_{0};
+  std::atomic<bool> terminate_pool_{false};
+  std::atomic<bool> stopped_{false};
+  utils::Synchronized<std::queue<std::unique_ptr<TaskSignature>>,
+                      utils::SpinLock>
+      task_queue_;
+  std::mutex pool_lock_;
+  std::condition_variable queue_cv_;
+};
+
+}  // namespace utils

tests/concurrent/network_read_hang.cpp

@@ -72,7 +72,7 @@ TEST(Network, SocketReadHangOnConcurrentConnections) {
   // start clients
   std::vector<std::thread> clients;
   for (int i = 0; i < Nc; ++i)
-    clients.push_back(std::thread(client_run, i, interface, ep.port()));
+    clients.push_back(std::thread(client_run, i, interface, ep.port));
 
   // wait for 2s and stop clients
   std::this_thread::sleep_for(std::chrono::seconds(2));

tests/concurrent/network_server.cpp

@@ -30,7 +30,7 @@ TEST(Network, Server) {
   std::vector<std::thread> clients;
   for (int i = 0; i < N; ++i)
     clients.push_back(
-        std::thread(client_run, i, interface, ep.port(), data, 30000, SIZE));
+        std::thread(client_run, i, interface, ep.port, data, 30000, SIZE));
 
   // cleanup clients
   for (int i = 0; i < N; ++i) clients[i].join();

tests/concurrent/network_session_leak.cpp

@@ -33,8 +33,8 @@ TEST(Network, SessionLeak) {
   const auto &ep = server.endpoint();
   int testlen = 3000;
   for (int i = 0; i < N; ++i) {
-    clients.push_back(std::thread(client_run, i, interface, ep.port(), data,
-                                  testlen, testlen));
+    clients.push_back(
+        std::thread(client_run, i, interface, ep.port, data, testlen, testlen));
     std::this_thread::sleep_for(10ms);
   }
 

tests/integration/durability/runner.py

@@ -15,7 +15,9 @@ TESTS_DIR = os.path.join(SCRIPT_DIR, "tests")
 
 SNAPSHOT_FILE_NAME = "snapshot.bin"
 WAL_FILE_NAME = "wal.bin"
-DUMP_FILE_NAME = "expected.cypher"
+
+DUMP_SNAPSHOT_FILE_NAME = "expected_snapshot.cypher"
+DUMP_WAL_FILE_NAME = "expected_wal.cypher"
 
 
 def wait_for_server(port, delay=0.1):
@@ -38,7 +40,12 @@ def list_to_string(data):
     return ret
 
 
-def execute_test(memgraph_binary, dump_binary, test_directory, test_type):
+def execute_test(
+        memgraph_binary,
+        dump_binary,
+        test_directory,
+        test_type,
+        write_expected):
     assert test_type in ["SNAPSHOT", "WAL"], \
         "Test type should be either 'SNAPSHOT' or 'WAL'."
     print("\033[1;36m~~ Executing test {} ({}) ~~\033[0m"
@@ -82,15 +89,25 @@ def execute_test(memgraph_binary, dump_binary, test_directory, test_type):
     memgraph.terminate()
     assert memgraph.wait() == 0, "Memgraph process didn't exit cleanly!"
 
-    # Compare dump files
-    expected_dump_file = os.path.join(test_directory, DUMP_FILE_NAME)
-    assert os.path.exists(expected_dump_file), \
-        "Could not find expected dump path {}".format(expected_dump_file)
-    queries_got = sorted_content(dump_output_file.name)
-    queries_expected = sorted_content(expected_dump_file)
-    assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
-        "{}".format(list_to_string(queries_got),
-                    list_to_string(queries_expected))
+    dump_file_name = DUMP_SNAPSHOT_FILE_NAME if test_type == "SNAPSHOT" else DUMP_WAL_FILE_NAME
+
+    if write_expected:
+        with open(dump_output_file.name, 'r') as dump:
+            queries_got = dump.readlines()
+        # Write dump files
+        expected_dump_file = os.path.join(test_directory, dump_file_name)
+        with open(expected_dump_file, 'w') as expected:
+            expected.writelines(queries_got)
+    else:
+        # Compare dump files
+        expected_dump_file = os.path.join(test_directory, dump_file_name)
+        assert os.path.exists(expected_dump_file), \
+            "Could not find expected dump path {}".format(expected_dump_file)
+        queries_got = sorted_content(dump_output_file.name)
+        queries_expected = sorted_content(expected_dump_file)
+        assert queries_got == queries_expected, "Expected\n{}\nto be equal to\n" \
+            "{}".format(list_to_string(queries_got),
+                        list_to_string(queries_expected))
 
     print("\033[1;32m~~ Test successful ~~\033[0m\n")
 
@@ -112,9 +129,11 @@ def find_test_directories(directory):
                 continue
             snapshot_file = os.path.join(test_dir_path, SNAPSHOT_FILE_NAME)
             wal_file = os.path.join(test_dir_path, WAL_FILE_NAME)
-            dump_file = os.path.join(test_dir_path, DUMP_FILE_NAME)
-            if (os.path.isfile(snapshot_file) and os.path.isfile(dump_file) and
-                    os.path.isfile(wal_file)):
+            dump_snapshot_file = os.path.join(
+                test_dir_path, DUMP_SNAPSHOT_FILE_NAME)
+            dump_wal_file = os.path.join(test_dir_path, DUMP_WAL_FILE_NAME)
+            if (os.path.isfile(snapshot_file) and os.path.isfile(dump_snapshot_file)
+                    and os.path.isfile(wal_file) and os.path.isfile(dump_wal_file)):
                 test_dirs.append(test_dir_path)
             else:
                 raise Exception("Missing data in test directory '{}'"
@@ -129,13 +148,27 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--memgraph", default=memgraph_binary)
     parser.add_argument("--dump", default=dump_binary)
+    parser.add_argument(
+        '--write-expected',
+        action='store_true',
+        help='Overwrite the expected cypher with results from current run')
     args = parser.parse_args()
 
     test_directories = find_test_directories(TESTS_DIR)
     assert len(test_directories) > 0, "No tests have been found!"
 
     for test_directory in test_directories:
-        execute_test(args.memgraph, args.dump, test_directory, "SNAPSHOT")
-        execute_test(args.memgraph, args.dump, test_directory, "WAL")
+        execute_test(
+            args.memgraph,
+            args.dump,
+            test_directory,
+            "SNAPSHOT",
+            args.write_expected)
+        execute_test(
+            args.memgraph,
+            args.dump,
+            test_directory,
+            "WAL",
+            args.write_expected)
 
     sys.exit(0)

tests/integration/durability/tests/v12/test_all/expected.cypher

@@ -1,11 +0,0 @@
-CREATE INDEX ON :`label`;
-CREATE INDEX ON :`label2`(`prop2`);
-CREATE INDEX ON :`label2`(`prop`);
-CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `prop`: "joj", `ext`: 2});
-CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `prop`: "joj", `ext`: 2});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_constraints/expected.cypher

@@ -1,10 +0,0 @@
-CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`prop`);
-CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`prop1`);
-CREATE CONSTRAINT ON (u:`labellabel`) ASSERT EXISTS (u.`prop`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label` {__mg_id__: 0, `prop`: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 1, `prop`: false});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop1`: 1});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop1`: 2});
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_edges/expected.cypher

@@ -1,45 +0,0 @@
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__ {__mg_id__: 0});
-CREATE (:__mg_vertex__ {__mg_id__: 1});
-CREATE (:__mg_vertex__ {__mg_id__: 2});
-CREATE (:__mg_vertex__ {__mg_id__: 3});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 4});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 5});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 6});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 7});
-CREATE (:__mg_vertex__:`lab` {__mg_id__: 8});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 9});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 10});
-CREATE (:__mg_vertex__:`lab2` {__mg_id__: 11});
-CREATE (:__mg_vertex__ {__mg_id__: 12});
-CREATE (:__mg_vertex__ {__mg_id__: 13});
-CREATE (:__mg_vertex__ {__mg_id__: 14});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 4 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:`link`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 4 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 5 AND v.__mg_id__ = 5 CREATE (u)-[:`link2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 9 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 9 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 12 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 13 CREATE (u)-[:`link88`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 12 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 14 CREATE (u)-[:`selfedge`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 11 AND v.__mg_id__ = 11 CREATE (u)-[:`selfedge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`link3`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 12 CREATE (u)-[:`selfedge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 13 AND v.__mg_id__ = 13 CREATE (u)-[:`selfedge2`]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_edges_with_properties/expected.cypher

@@ -1,24 +0,0 @@
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__ {__mg_id__: 0});
-CREATE (:__mg_vertex__ {__mg_id__: 1});
-CREATE (:__mg_vertex__ {__mg_id__: 2});
-CREATE (:__mg_vertex__ {__mg_id__: 3});
-CREATE (:__mg_vertex__ {__mg_id__: 4});
-CREATE (:__mg_vertex__ {__mg_id__: 5});
-CREATE (:__mg_vertex__ {__mg_id__: 6});
-CREATE (:__mg_vertex__ {__mg_id__: 7});
-CREATE (:__mg_vertex__ {__mg_id__: 8});
-CREATE (:__mg_vertex__ {__mg_id__: 9});
-CREATE (:__mg_vertex__ {__mg_id__: 10});
-CREATE (:__mg_vertex__ {__mg_id__: 11});
-CREATE (:__mg_vertex__ {__mg_id__: 12});
-CREATE (:__mg_vertex__ {__mg_id__: 13});
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 1}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: false}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: 1, `prop2`: {`prop4`: 9}}]->(v);
-MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, ""]}]->(v);
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v12/test_indices/expected.cypher

@@ -1,16 +0,0 @@
-CREATE INDEX ON :`label2`;
-CREATE INDEX ON :`label1`;
-CREATE INDEX ON :`label3`;
-CREATE INDEX ON :`label`(`prop`);
-CREATE INDEX ON :`label2`(`prop`);
-CREATE INDEX ON :__mg_vertex__(__mg_id__);
-CREATE (:__mg_vertex__:`label` {__mg_id__: 0});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 2});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: 1});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 4, `prop`: 2});
-CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop`: 3});
-CREATE (:__mg_vertex__:`label2` {__mg_id__: 6, `prop2`: 1});
-CREATE (:__mg_vertex__:`label3` {__mg_id__: 7});
-DROP INDEX ON :__mg_vertex__(__mg_id__);
-MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;

tests/integration/durability/tests/v14/test_all/expected_snapshot.cypher

@@ -0,0 +1,16 @@
+CREATE INDEX ON :`label`;
+CREATE INDEX ON :`label2`(`prop2`);
+CREATE INDEX ON :`label2`(`prop`);
+CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
+CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE;
+CREATE INDEX ON :__mg_vertex__(__mg_id__);
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"});
+CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `ext`: 2, `prop`: "joj"});
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1});
+CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2});
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
+DROP INDEX ON :__mg_vertex__(__mg_id__);
+MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;