Compare commits
3 Commits
property_t
...
add-gnuplo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f81d4d092 | ||
|
|
0220e9b4f7 | ||
|
|
0ff389ffc4 |
@@ -30,8 +30,6 @@ enum class ErrorCode : uint8_t {
|
||||
SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
|
||||
SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
|
||||
SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
|
||||
// Distributed race conditions
|
||||
STALE_SHARD_MAP,
|
||||
|
||||
OBJECT_NOT_FOUND,
|
||||
};
|
||||
@@ -64,8 +62,6 @@ constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
|
||||
return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
|
||||
case ErrorCode::OBJECT_NOT_FOUND:
|
||||
return "OBJECT_NOT_FOUND";
|
||||
case ErrorCode::STALE_SHARD_MAP:
|
||||
return "STALE_SHARD_MAP";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -9,272 +9,21 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include <coordinator/coordinator.hpp>
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
// 1. try to begin any suggested splits
|
||||
// 2. mark all initialized RSMs as INITIALIZED in the ShardMap
|
||||
// 3. assign any valid underreplicated shards to the Heartbeat sender
|
||||
// 4. send any split requests that the Heartbeat sender should be applying
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
|
||||
spdlog::trace("Coordinator handling HeartbeatRequest");
|
||||
spdlog::info("Coordinator handling HeartbeatRequest");
|
||||
|
||||
HeartbeatResponse ret{};
|
||||
// add this storage engine to any under-replicated shards that it is not already a part of
|
||||
|
||||
bool initiated_split = false;
|
||||
auto initializing_rsms_for_shard_manager =
|
||||
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
|
||||
|
||||
// 1. try to begin any suggested splits
|
||||
for (const auto &suggested_split_info : heartbeat_request.suggested_splits) {
|
||||
const LabelId label_id = suggested_split_info.label_id;
|
||||
auto &label_space = shard_map_.label_spaces.at(label_id);
|
||||
|
||||
const PrimaryKey splitting_shard_low_key =
|
||||
storage::conversions::ConvertPropertyVector(suggested_split_info.splitting_shard_low_key);
|
||||
const PrimaryKey split_key = storage::conversions::ConvertPropertyVector(suggested_split_info.split_key);
|
||||
|
||||
auto &shard = label_space.shards.at(splitting_shard_low_key);
|
||||
const ShardId splitting_shard_id = std::make_pair(label_id, splitting_shard_low_key);
|
||||
|
||||
if (shard.pending_split.has_value() || shard.version != suggested_split_info.shard_version) {
|
||||
spdlog::debug("Coordinator skipping split, already splitting: {}, shard.version: {}, suggested shard_version: {}",
|
||||
shard.pending_split.has_value(), shard.version, suggested_split_info.shard_version);
|
||||
continue;
|
||||
}
|
||||
|
||||
MG_ASSERT(!label_space.shards.contains(split_key));
|
||||
|
||||
// begin the split process for this shard
|
||||
const auto new_uuid_lhs = shard_map_.GetHlc();
|
||||
const auto new_uuid_rhs = shard_map_.GetHlc();
|
||||
spdlog::debug(
|
||||
"Coordinator beginning new split process for shard {} after receiving a pending split. splitting into lhs: {} "
|
||||
"and rhs: {} at split key {}",
|
||||
shard.version.logical_id, new_uuid_lhs.logical_id, new_uuid_rhs.logical_id, split_key.back());
|
||||
|
||||
// bump current shard version and store pending split info
|
||||
shard.version = new_uuid_lhs;
|
||||
shard.pending_split = suggested_split_info;
|
||||
MG_ASSERT(!splitting_shards_.contains(splitting_shard_id));
|
||||
splitting_shards_.insert(splitting_shard_id);
|
||||
initiated_split = true;
|
||||
|
||||
// copy this shard and store it in the ShardMap
|
||||
ShardMetadata duplicated_shard{shard};
|
||||
duplicated_shard.version = new_uuid_rhs;
|
||||
duplicated_shard.pending_split.reset();
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
|
||||
for (auto &peer_metadata : duplicated_shard.peers) {
|
||||
peer_metadata.status = Status::PENDING_SPLIT;
|
||||
peer_metadata.split_from = peer_metadata.address.unique_id;
|
||||
|
||||
const auto new_uuid = shard_map_.NewShardUuid();
|
||||
|
||||
spdlog::debug("Coordinator allocating new rsm uuid: {} for new shard {} with low key {}", new_uuid, new_uuid_rhs,
|
||||
split_key.back());
|
||||
|
||||
// store new uuid for the right side of each shard
|
||||
rsm_split_from_.insert({new_uuid, splitting_shard_id});
|
||||
split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
|
||||
|
||||
peer_metadata.address.unique_id = new_uuid;
|
||||
}
|
||||
|
||||
// TODO(tyler) fix this good assertion
|
||||
// if (high_key) {
|
||||
// MG_ASSERT(converted_pk < *high_key, "Split point is beyond low key of the next shard");
|
||||
// }
|
||||
label_space.shards.insert({split_key, duplicated_shard});
|
||||
|
||||
// we will send the heartbeater its split request in stage 4 below
|
||||
}
|
||||
|
||||
// 2. mark all initialized RSMs as INITIALIZED in the ShardMap
|
||||
for (const auto &[initialized_rsm, shard_id] : heartbeat_request.initialized_rsms) {
|
||||
auto [label_id, low_key] = shard_id;
|
||||
auto &label_space = shard_map_.label_spaces.at(label_id);
|
||||
auto &shard = label_space.shards.at(low_key);
|
||||
|
||||
// if even a single shard has been initialized after a split, its raft log has
|
||||
// reached consensus and we can allow more splits again and remove the pending
|
||||
// split state.
|
||||
if (rsm_split_from_.contains(initialized_rsm)) {
|
||||
auto split_shard_id = rsm_split_from_.at(initialized_rsm);
|
||||
|
||||
auto [split_label_id, split_low_key] = split_shard_id;
|
||||
auto &split_shard = label_space.shards.at(split_low_key);
|
||||
|
||||
spdlog::debug("Coordinator clearing split for shard {}", split_shard.version);
|
||||
MG_ASSERT(split_shard.pending_split.has_value());
|
||||
MG_ASSERT(splitting_shards_.contains(split_shard_id));
|
||||
|
||||
split_shard.pending_split.reset();
|
||||
splitting_shards_.erase(split_shard_id);
|
||||
rsm_split_from_.erase(initialized_rsm);
|
||||
}
|
||||
|
||||
size_t initialized_count = 0;
|
||||
bool found = false;
|
||||
for (auto &peer : shard.peers) {
|
||||
if (peer.address.last_known_ip == heartbeat_request.from_storage_manager.last_known_ip &&
|
||||
peer.address.last_known_port == heartbeat_request.from_storage_manager.last_known_port) {
|
||||
// TODO(tyler) switch the conditional to match on rsm uuid and update the peer last_known_* address always
|
||||
const int low_key_int = low_key[0].ValueInt();
|
||||
// TODO(tyler) this is failing in simulation tests
|
||||
spdlog::debug("Coordinator marking rsm {} for shard {} as initialized", initialized_rsm, shard.version);
|
||||
MG_ASSERT(peer.address.unique_id == initialized_rsm,
|
||||
"expected Coordinator uuid {} to equal peer uuid {} for shard version {} with low key {}",
|
||||
peer.address.unique_id, initialized_rsm, shard.version, low_key_int);
|
||||
peer.status = Status::CONSENSUS_PARTICIPANT;
|
||||
ret.acknowledged_initialized_rsms.push_back(initialized_rsm);
|
||||
found = true;
|
||||
}
|
||||
|
||||
if (peer.status == Status::CONSENSUS_PARTICIPANT) {
|
||||
initialized_count++;
|
||||
}
|
||||
}
|
||||
MG_ASSERT(shard.peers.size() == 1);
|
||||
|
||||
MG_ASSERT(found, "did not find peer {} in shard version {}", heartbeat_request.from_storage_manager.unique_id,
|
||||
shard.version);
|
||||
|
||||
if (initialized_count >= label_space.replication_factor) {
|
||||
spdlog::debug("Coordinator clearing underreplicated shard with {} initialized peers", initialized_count);
|
||||
underreplicated_shards_.erase(shard_id);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. assign any valid underreplicated shards to the Heartbeat sender
|
||||
for (auto &underreplicated_shard_id : underreplicated_shards_) {
|
||||
auto [label_id, low_key] = underreplicated_shard_id;
|
||||
auto &label_space = shard_map_.label_spaces.at(label_id);
|
||||
auto &shard = label_space.shards.at(low_key);
|
||||
|
||||
bool needs_to_initialize = true;
|
||||
|
||||
// make sure we're not already a member
|
||||
for (const auto &peer : shard.peers) {
|
||||
if (peer.address.last_known_ip == heartbeat_request.from_storage_manager.last_known_ip &&
|
||||
peer.address.last_known_port == heartbeat_request.from_storage_manager.last_known_port) {
|
||||
needs_to_initialize = false;
|
||||
|
||||
if (peer.status == Status::INITIALIZING) {
|
||||
spdlog::trace(
|
||||
"Coordinator reminding ShardManager to initialize a shard that we have previously assigned it to");
|
||||
Address address = heartbeat_request.from_storage_manager;
|
||||
address.unique_id = peer.address.unique_id;
|
||||
|
||||
std::optional<PrimaryKey> high_key;
|
||||
auto next = std::next(label_space.shards.find(low_key));
|
||||
if (next != label_space.shards.end()) {
|
||||
high_key = next->first;
|
||||
}
|
||||
|
||||
ret.shards_to_initialize.push_back(ShardToInitialize{
|
||||
.new_shard_version = shard.version,
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = shard_map_.schemas[label_id],
|
||||
.config = Config{.split =
|
||||
Config::Split{
|
||||
.max_shard_vertex_size = label_space.split_threshold,
|
||||
}},
|
||||
.id_to_names = shard_map_.IdToNames(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!needs_to_initialize) {
|
||||
// we are already a member of this shard. continue to next underreplicated shard to consider
|
||||
continue;
|
||||
}
|
||||
|
||||
// we are not already a member, so we can be assigned to this shard
|
||||
Address address = heartbeat_request.from_storage_manager;
|
||||
address.unique_id = shard_map_.NewShardUuid();
|
||||
|
||||
std::optional<PrimaryKey> high_key;
|
||||
auto next = std::next(label_space.shards.find(low_key));
|
||||
if (next != label_space.shards.end()) {
|
||||
high_key = next->first;
|
||||
}
|
||||
|
||||
spdlog::debug("Coordinator assigning new rsm uuid {} to shard version {} with low key {} to shard worker {}",
|
||||
address.unique_id, shard.version, low_key.back(), address);
|
||||
|
||||
ret.shards_to_initialize.push_back(ShardToInitialize{
|
||||
.new_shard_version = shard.version,
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = shard_map_.schemas[label_id],
|
||||
.config = Config{.split =
|
||||
Config::Split{
|
||||
.max_shard_vertex_size = label_space.split_threshold,
|
||||
}},
|
||||
.id_to_names = shard_map_.IdToNames(),
|
||||
});
|
||||
|
||||
PeerMetadata peer_metadata = {
|
||||
.address = address,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.peers.emplace_back(peer_metadata);
|
||||
}
|
||||
|
||||
// 4. send any split requests that the Heartbeat sender should be applying
|
||||
for (const auto &[label_id, low_key] : splitting_shards_) {
|
||||
// see if this machine is a PENDING_SPLIT peer for any of the splitting shards
|
||||
const auto &label_space = shard_map_.label_spaces.at(label_id);
|
||||
const auto &splitting_shard = label_space.shards.at(low_key);
|
||||
MG_ASSERT(splitting_shard.pending_split.has_value());
|
||||
const auto &suggested_split_info = splitting_shard.pending_split.value();
|
||||
const auto split_key = storage::conversions::ConvertPropertyVector(suggested_split_info.split_key);
|
||||
const auto &new_shard = label_space.shards.at(split_key);
|
||||
|
||||
for (const auto &peer : new_shard.peers) {
|
||||
if (peer.status != Status::PENDING_SPLIT ||
|
||||
peer.address.last_known_ip != heartbeat_request.from_storage_manager.last_known_ip ||
|
||||
peer.address.last_known_port != heartbeat_request.from_storage_manager.last_known_port) {
|
||||
spdlog::debug("Coordinator not splitting peer: status is: {}, peer address: {} storage manager address: {}",
|
||||
peer.status, peer.address, heartbeat_request.from_storage_manager);
|
||||
// not splitting or not us
|
||||
continue;
|
||||
}
|
||||
|
||||
spdlog::trace("Coordinator expecting heartbeating peer to split, so it will reply with a ShardToSplit message");
|
||||
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping{};
|
||||
|
||||
// need to iterate over all peers again to build the full uuid_mapping to
|
||||
// send to this machine
|
||||
for (const auto &peer_metadata2 : new_shard.peers) {
|
||||
uuid_mapping.emplace(peer_metadata2.split_from, peer_metadata2.address.unique_id);
|
||||
}
|
||||
|
||||
ret.shards_to_split.push_back(ShardToSplit{
|
||||
.split_key = split_key,
|
||||
.old_shard_version = suggested_split_info.shard_version,
|
||||
.new_lhs_shard_version = splitting_shard.version,
|
||||
.new_rhs_shard_version = new_shard.version,
|
||||
.uuid_mapping = uuid_mapping,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (initiated_split) {
|
||||
MG_ASSERT(!ret.shards_to_split.empty(), "did not send a split request to the shard that suggested a split");
|
||||
}
|
||||
|
||||
return ret;
|
||||
return HeartbeatResponse{
|
||||
.shards_to_initialize = initializing_rsms_for_shard_manager,
|
||||
};
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
|
||||
@@ -288,7 +37,6 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
|
||||
.logical_id = ++highest_allocated_timestamp_,
|
||||
// TODO(tyler) probably pass some more context to the Coordinator here
|
||||
// so that we can use our wall clock and enforce monotonicity.
|
||||
// Check it to ensure it's +/- 1 day of the coordinator's io_.Now()
|
||||
// .coordinator_wall_clock = io_.Now(),
|
||||
};
|
||||
|
||||
@@ -313,6 +61,22 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
|
||||
return res;
|
||||
}
|
||||
|
||||
/// This splits the shard immediately beneath the provided
|
||||
/// split key, keeping the assigned peers identical for now,
|
||||
/// but letting them be gradually migrated over time.
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
|
||||
SplitShardResponse res{};
|
||||
|
||||
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
|
||||
res.success = false;
|
||||
} else {
|
||||
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
|
||||
split_shard_request.split_key);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/// This adds the provided storage engine to the standby storage engine pool,
|
||||
/// which can be used to rebalance storage over time.
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(
|
||||
@@ -338,16 +102,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initi
|
||||
|
||||
std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
|
||||
initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
|
||||
initialize_label_request.split_threshold, initialize_label_request.last_shard_map_version);
|
||||
initialize_label_request.last_shard_map_version);
|
||||
|
||||
if (new_label_id) {
|
||||
res.new_label_id = new_label_id.value();
|
||||
res.fresher_shard_map = std::nullopt;
|
||||
res.success = true;
|
||||
|
||||
auto min_key = SchemaToMinKey(initialize_label_request.schema);
|
||||
auto shard_id = std::make_pair(new_label_id.value(), min_key);
|
||||
underreplicated_shards_.insert(shard_id);
|
||||
} else {
|
||||
res.fresher_shard_map = shard_map_;
|
||||
res.success = false;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,14 +20,13 @@
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/simulator/simulator.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include <coordinator/hybrid_logical_clock.hpp>
|
||||
#include <coordinator/shard_map.hpp>
|
||||
#include <io/simulator/simulator.hpp>
|
||||
#include <io/time.hpp>
|
||||
#include <io/transport.hpp>
|
||||
#include <storage/v3/id_types.hpp>
|
||||
#include <storage/v3/schemas.hpp>
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
@@ -38,8 +37,6 @@ using memgraph::storage::v3::SchemaProperty;
|
||||
using SimT = memgraph::io::simulator::SimulatorTransport;
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
|
||||
using ShardId = std::pair<LabelId, PrimaryKey>;
|
||||
|
||||
struct HlcRequest {
|
||||
Hlc last_shard_map_version;
|
||||
};
|
||||
@@ -115,7 +112,6 @@ struct InitializeLabelRequest {
|
||||
std::string label_name;
|
||||
std::vector<SchemaProperty> schema;
|
||||
size_t replication_factor;
|
||||
uint64_t split_threshold;
|
||||
Hlc last_shard_map_version;
|
||||
};
|
||||
|
||||
@@ -125,29 +121,28 @@ struct InitializeLabelResponse {
|
||||
std::optional<ShardMap> fresher_shard_map;
|
||||
};
|
||||
|
||||
struct HeartbeatRequest {
|
||||
Address from_storage_manager;
|
||||
std::set<boost::uuids::uuid> initialized_rsms;
|
||||
};
|
||||
|
||||
struct HeartbeatResponse {
|
||||
std::vector<ShardToInitialize> shards_to_initialize;
|
||||
};
|
||||
|
||||
using CoordinatorWriteRequests =
|
||||
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, RegisterStorageEngineRequest, DeregisterStorageEngineRequest,
|
||||
InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
|
||||
using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, RegisterStorageEngineResponse,
|
||||
DeregisterStorageEngineResponse, InitializeLabelResponse,
|
||||
AllocatePropertyIdsResponse, HeartbeatResponse>;
|
||||
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
|
||||
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
|
||||
using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
|
||||
RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
|
||||
InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
|
||||
|
||||
using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
|
||||
using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
|
||||
|
||||
class Coordinator {
|
||||
public:
|
||||
explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {
|
||||
// Populate underreplicated_shards_
|
||||
for (const auto &[label_id, label_space] : shard_map_.label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.peers.size() < label_space.replication_factor) {
|
||||
ShardId shard_id = std::make_pair(label_id, low_key);
|
||||
underreplicated_shards_.insert(shard_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {}
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
|
||||
CoordinatorReadResponses Read(CoordinatorReadRequests requests) {
|
||||
@@ -165,11 +160,6 @@ class Coordinator {
|
||||
ShardMap shard_map_;
|
||||
uint64_t highest_allocated_timestamp_{0};
|
||||
|
||||
std::set<ShardId> underreplicated_shards_;
|
||||
std::map<boost::uuids::uuid, ShardId> rsm_split_from_;
|
||||
std::set<ShardId> splitting_shards_;
|
||||
std::map<Address, std::set<ShardId>> assigned_shards_;
|
||||
|
||||
/// Query engines need to periodically request batches of unique edge IDs.
|
||||
uint64_t highest_allocated_edge_id_{0};
|
||||
|
||||
@@ -185,6 +175,11 @@ class Coordinator {
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
|
||||
|
||||
/// This splits the shard immediately beneath the provided
|
||||
/// split key, keeping the assigned peers identical for now,
|
||||
/// but letting them be gradually migrated over time.
|
||||
CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
|
||||
|
||||
/// This adds the provided storage engine to the standby storage engine pool,
|
||||
/// which can be used to rebalance storage over time.
|
||||
static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -23,19 +23,17 @@ namespace memgraph::coordinator {
|
||||
using Time = memgraph::io::Time;
|
||||
|
||||
/// Hybrid-logical clock
|
||||
struct Hlc final {
|
||||
uint64_t logical_id{0};
|
||||
struct Hlc {
|
||||
uint64_t logical_id = 0;
|
||||
Time coordinator_wall_clock = Time::min();
|
||||
|
||||
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
|
||||
|
||||
bool operator==(const Hlc &other) const noexcept = default;
|
||||
bool operator<(const Hlc &other) const noexcept = default;
|
||||
bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
|
||||
bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
|
||||
bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
|
||||
|
||||
Hlc operator++() { return {.logical_id = ++logical_id, .coordinator_wall_clock = Time::min()}; }
|
||||
bool operator==(const Hlc &other) const = default;
|
||||
bool operator<(const Hlc &other) const = default;
|
||||
bool operator==(const uint64_t other) const { return logical_id == other; }
|
||||
bool operator<(const uint64_t other) const { return logical_id < other; }
|
||||
bool operator>=(const uint64_t other) const { return logical_id >= other; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
|
||||
auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -9,21 +9,15 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/temporal.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "utils/cast.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/string.hpp"
|
||||
@@ -74,9 +68,6 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
|
||||
}
|
||||
|
||||
ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
const uint64_t default_replication_factor = 1;
|
||||
const uint64_t default_split_threshold = 1'000'000;
|
||||
|
||||
ShardMap shard_map;
|
||||
const auto read_size = [&input_stream] {
|
||||
size_t size{0};
|
||||
@@ -93,17 +84,23 @@ ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
|
||||
const auto read_names = [&read_size, &read_word] {
|
||||
const auto number_of_names = read_size();
|
||||
spdlog::trace("ShardMap::Parse reading {} names", number_of_names);
|
||||
spdlog::trace("Reading {} names", number_of_names);
|
||||
std::vector<std::string> names;
|
||||
names.reserve(number_of_names);
|
||||
|
||||
for (auto name_index = 0; name_index < number_of_names; ++name_index) {
|
||||
names.push_back(read_word());
|
||||
spdlog::trace("ShardMap::Parse read '{}'", names.back());
|
||||
spdlog::trace("Read '{}'", names.back());
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
const auto read_line = [&input_stream] {
|
||||
std::string line;
|
||||
std::getline(input_stream, line);
|
||||
return line;
|
||||
};
|
||||
|
||||
const auto parse_type = [](const std::string &type) {
|
||||
static const auto type_map = std::unordered_map<std::string, common::SchemaType>{
|
||||
{"string", common::SchemaType::STRING}, {"int", common::SchemaType::INT}, {"bool", common::SchemaType::BOOL}};
|
||||
@@ -113,32 +110,45 @@ ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
return it->second;
|
||||
};
|
||||
|
||||
spdlog::trace("ShardMap::Parse reading properties");
|
||||
const auto parse_property_value = [](std::string text, const common::SchemaType type) {
|
||||
if (type == common::SchemaType::STRING) {
|
||||
return storage::v3::PropertyValue{std::move(text)};
|
||||
}
|
||||
if (type == common::SchemaType::INT) {
|
||||
size_t processed{0};
|
||||
int64_t value = std::stoll(text, &processed);
|
||||
MG_ASSERT(processed == text.size() || text[processed] == ' ', "Invalid integer format: '{}'", text);
|
||||
return storage::v3::PropertyValue{value};
|
||||
}
|
||||
LOG_FATAL("Not supported type: {}", utils::UnderlyingCast(type));
|
||||
};
|
||||
|
||||
spdlog::debug("Reading properties");
|
||||
const auto properties = read_names();
|
||||
MG_ASSERT(shard_map.AllocatePropertyIds(properties).size() == properties.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::trace("ShardMap::Parse Reading edge types");
|
||||
spdlog::debug("Reading edge types");
|
||||
const auto edge_types = read_names();
|
||||
MG_ASSERT(shard_map.AllocateEdgeTypeIds(edge_types).size() == edge_types.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::trace("ShardMap::Parse reading primary labels");
|
||||
spdlog::debug("Reading primary labels");
|
||||
const auto number_of_primary_labels = read_size();
|
||||
spdlog::trace("ShardMap::Parse reading {} primary labels", number_of_primary_labels);
|
||||
spdlog::debug("Reading {} primary labels", number_of_primary_labels);
|
||||
|
||||
for (auto label_index = 0; label_index < number_of_primary_labels; ++label_index) {
|
||||
const auto primary_label = read_word();
|
||||
spdlog::trace("ShardMap::Parse reading primary label named '{}'", primary_label);
|
||||
spdlog::debug("Reading primary label named '{}'", primary_label);
|
||||
const auto number_of_primary_properties = read_size();
|
||||
spdlog::trace("ShardMap::Parse reading {} primary properties", number_of_primary_properties);
|
||||
spdlog::debug("Reading {} primary properties", number_of_primary_properties);
|
||||
std::vector<std::string> pp_names;
|
||||
std::vector<common::SchemaType> pp_types;
|
||||
pp_names.reserve(number_of_primary_properties);
|
||||
pp_types.reserve(number_of_primary_properties);
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pp_names.push_back(read_word());
|
||||
spdlog::trace("ShardMap::Parse reading primary property named '{}'", pp_names.back());
|
||||
spdlog::debug("Reading primary property named '{}'", pp_names.back());
|
||||
pp_types.push_back(parse_type(read_word()));
|
||||
}
|
||||
auto pp_mapping = shard_map.AllocatePropertyIds(pp_names);
|
||||
@@ -149,10 +159,33 @@ ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
|
||||
}
|
||||
const auto hlc = shard_map.GetHlc();
|
||||
MG_ASSERT(
|
||||
shard_map.InitializeNewLabel(primary_label, schema, default_replication_factor, default_split_threshold, hlc)
|
||||
.has_value(),
|
||||
"Cannot initialize new label: {}", primary_label);
|
||||
MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
|
||||
"Cannot initialize new label: {}", primary_label);
|
||||
|
||||
const auto number_of_split_points = read_size();
|
||||
spdlog::debug("Reading {} split points", number_of_split_points);
|
||||
|
||||
[[maybe_unused]] const auto remainder_from_last_line = read_line();
|
||||
for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
|
||||
const auto line = read_line();
|
||||
spdlog::debug("Read split point '{}'", line);
|
||||
MG_ASSERT(line.front() == '[', "Invalid split file format!");
|
||||
MG_ASSERT(line.back() == ']', "Invalid split file format!");
|
||||
std::string_view line_view{line};
|
||||
line_view.remove_prefix(1);
|
||||
line_view.remove_suffix(1);
|
||||
static constexpr std::string_view kDelimiter{","};
|
||||
auto pk_values_as_text = utils::Split(line_view, kDelimiter);
|
||||
std::vector<PropertyValue> pk;
|
||||
pk.reserve(number_of_primary_properties);
|
||||
MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
|
||||
"Split point contains invalid number of values '{}'", line);
|
||||
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
|
||||
}
|
||||
shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
|
||||
}
|
||||
}
|
||||
|
||||
return shard_map;
|
||||
@@ -213,7 +246,9 @@ std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
|
||||
return id_to_names;
|
||||
}
|
||||
|
||||
boost::uuids::uuid Uint64ToUuid(uint64_t u) {
|
||||
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
|
||||
|
||||
boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
|
||||
return boost::uuids::uuid{0,
|
||||
0,
|
||||
0,
|
||||
@@ -222,28 +257,124 @@ boost::uuids::uuid Uint64ToUuid(uint64_t u) {
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
static_cast<unsigned char>(u >> 56U),
|
||||
static_cast<unsigned char>(u >> 48U),
|
||||
static_cast<unsigned char>(u >> 40U),
|
||||
static_cast<unsigned char>(u >> 32U),
|
||||
static_cast<unsigned char>(u >> 24U),
|
||||
static_cast<unsigned char>(u >> 16U),
|
||||
static_cast<unsigned char>(u >> 8U),
|
||||
static_cast<unsigned char>(u)};
|
||||
static_cast<unsigned char>(shard_id >> 56U),
|
||||
static_cast<unsigned char>(shard_id >> 48U),
|
||||
static_cast<unsigned char>(shard_id >> 40U),
|
||||
static_cast<unsigned char>(shard_id >> 32U),
|
||||
static_cast<unsigned char>(shard_id >> 24U),
|
||||
static_cast<unsigned char>(shard_id >> 16U),
|
||||
static_cast<unsigned char>(shard_id >> 8U),
|
||||
static_cast<unsigned char>(shard_id)};
|
||||
}
|
||||
|
||||
boost::uuids::uuid ShardMap::NewShardUuid() {
|
||||
uint64_t shard_id = GetHlc().logical_id;
|
||||
return Uint64ToUuid(shard_id);
|
||||
std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
std::set<boost::uuids::uuid> initialized) {
|
||||
std::vector<ShardToInitialize> ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (auto &[label_id, label_space] : label_spaces) {
|
||||
for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
|
||||
auto &[low_key, shard] = *it;
|
||||
std::optional<PrimaryKey> high_key;
|
||||
if (const auto next_it = std::next(it); next_it != label_space.shards.end()) {
|
||||
high_key = next_it->first;
|
||||
}
|
||||
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
|
||||
bool machine_contains_shard = false;
|
||||
|
||||
for (auto &aas : shard.peers) {
|
||||
if (initialized.contains(aas.address.unique_id)) {
|
||||
machine_contains_shard = true;
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
mutated = true;
|
||||
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
|
||||
aas.status = Status::CONSENSUS_PARTICIPANT;
|
||||
}
|
||||
} else {
|
||||
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
|
||||
aas.address.last_known_port == storage_manager.last_known_port;
|
||||
if (same_machine) {
|
||||
machine_contains_shard = true;
|
||||
spdlog::info("reminding shard manager that they should begin participating in shard");
|
||||
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = aas.address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
|
||||
// increment version for each new uuid for deterministic creation
|
||||
IncrementShardMapVersion();
|
||||
|
||||
Address address = storage_manager;
|
||||
|
||||
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
|
||||
address.unique_id = NewShardUuid(shard_map_version.logical_id);
|
||||
|
||||
spdlog::info("assigning shard manager to shard");
|
||||
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
|
||||
AddressAndStatus aas = {
|
||||
.address = address,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.peers.emplace_back(aas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
Hlc ShardMap::GetHlc() noexcept { return ++shard_map_version; }
|
||||
bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
|
||||
if (previous_shard_map_version != shard_map_version) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto &label_space = label_spaces.at(label_id);
|
||||
auto &shards_in_map = label_space.shards;
|
||||
|
||||
MG_ASSERT(!shards_in_map.empty());
|
||||
MG_ASSERT(!shards_in_map.contains(key));
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
// Finding the ShardMetadata that the new PrimaryKey should map to.
|
||||
auto prev = std::prev(shards_in_map.upper_bound(key));
|
||||
ShardMetadata duplicated_shard = prev->second;
|
||||
|
||||
// Apply the split
|
||||
shards_in_map[key] = duplicated_shard;
|
||||
|
||||
IncrementShardMapVersion();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, uint64_t split_threshold,
|
||||
Hlc last_shard_map_version) {
|
||||
size_t replication_factor, Hlc last_shard_map_version) {
|
||||
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
|
||||
MG_ASSERT(false, "failed to InitializeNewLabel");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -262,9 +393,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
|
||||
.schema = schema,
|
||||
.shards = shards,
|
||||
.replication_factor = replication_factor,
|
||||
.split_threshold = split_threshold,
|
||||
};
|
||||
|
||||
schemas[label_id] = std::move(schema);
|
||||
|
||||
label_spaces.emplace(label_id, label_space);
|
||||
@@ -428,13 +557,13 @@ bool ShardMap::ClusterInitialized() const {
|
||||
for (const auto &[label_id, label_space] : label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.peers.size() < label_space.replication_factor) {
|
||||
spdlog::trace("ShardMap::ClusterInitialized label_space below desired replication factor");
|
||||
spdlog::info("label_space below desired replication factor");
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto &peer_metadata : shard.peers) {
|
||||
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::trace("ShardMap::ClusterInitialized shard member not yet a CONSENSUS_PARTICIPANT");
|
||||
for (const auto &aas : shard.peers) {
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -444,29 +573,4 @@ bool ShardMap::ClusterInitialized() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t ShardMap::InitializedShards() const {
|
||||
size_t count = 0;
|
||||
|
||||
for (const auto &[label_id, label_space] : label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.peers.size() < label_space.replication_factor) {
|
||||
continue;
|
||||
}
|
||||
|
||||
bool all_initialized = true;
|
||||
for (const auto &peer_metadata : shard.peers) {
|
||||
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
all_initialized = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (all_initialized) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,7 +14,6 @@
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
@@ -24,7 +23,6 @@
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
@@ -49,59 +47,39 @@ using memgraph::storage::v3::SchemaProperty;
|
||||
enum class Status : uint8_t {
|
||||
CONSENSUS_PARTICIPANT,
|
||||
INITIALIZING,
|
||||
PENDING_SPLIT,
|
||||
// TODO(tyler) this will possibly have more states,
|
||||
// depending on the reconfiguration protocol that we
|
||||
// implement.
|
||||
};
|
||||
|
||||
struct PeerMetadata {
|
||||
struct AddressAndStatus {
|
||||
memgraph::io::Address address;
|
||||
Status status;
|
||||
boost::uuids::uuid split_from;
|
||||
|
||||
friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
|
||||
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
|
||||
in << "PeerMetadata { address: ";
|
||||
in << peer_metadata.address;
|
||||
|
||||
if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
|
||||
in << ", status: CONSENSUS_PARTICIPANT";
|
||||
} else if (peer_metadata.status == Status::INITIALIZING) {
|
||||
in << ", status: INITIALIZING";
|
||||
} else if (peer_metadata.status == Status::PENDING_SPLIT) {
|
||||
in << ", status: PENDING_SPLIT";
|
||||
friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
|
||||
in << "AddressAndStatus { address: ";
|
||||
in << address_and_status.address;
|
||||
if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
|
||||
in << ", status: CONSENSUS_PARTICIPANT }";
|
||||
} else {
|
||||
MG_ASSERT(false, "failed to update the operator<< implementation for Status");
|
||||
in << ", status: INITIALIZING }";
|
||||
}
|
||||
|
||||
in << ", split_from: " << peer_metadata.split_from << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
|
||||
friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
|
||||
return lhs.address == rhs.address;
|
||||
}
|
||||
};
|
||||
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
using ShardId = std::pair<LabelId, PrimaryKey>;
|
||||
|
||||
struct ShardMetadata {
|
||||
std::vector<PeerMetadata> peers;
|
||||
Hlc version;
|
||||
std::optional<msgs::SuggestedSplitInfo> pending_split;
|
||||
|
||||
bool Underreplicated(size_t replication_factor) { return peers.size() < replication_factor; }
|
||||
|
||||
bool ContainsPeer(io::PartialAddress machine) {
|
||||
for (const auto &peer : peers) {
|
||||
if (peer.address.last_known_ip == machine.ip && peer.address.last_known_port == machine.port) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
std::vector<AddressAndStatus> peers;
|
||||
uint64_t version;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
|
||||
using utils::print_helpers::operator<<;
|
||||
@@ -115,9 +93,7 @@ struct ShardMetadata {
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) {
|
||||
return lhs.version == rhs.version && lhs.peers == rhs.peers;
|
||||
};
|
||||
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
|
||||
|
||||
friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
|
||||
if (lhs.peers != rhs.peers) {
|
||||
@@ -136,7 +112,6 @@ using PropertyMap = std::map<PropertyName, PropertyId>;
|
||||
using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
|
||||
|
||||
struct ShardToInitialize {
|
||||
Hlc new_shard_version;
|
||||
boost::uuids::uuid uuid;
|
||||
LabelId label_id;
|
||||
PrimaryKey min_key;
|
||||
@@ -146,26 +121,6 @@ struct ShardToInitialize {
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
};
|
||||
|
||||
struct ShardToSplit {
|
||||
PrimaryKey split_key;
|
||||
Hlc old_shard_version;
|
||||
Hlc new_lhs_shard_version;
|
||||
Hlc new_rhs_shard_version;
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
|
||||
};
|
||||
|
||||
struct HeartbeatRequest {
|
||||
Address from_storage_manager;
|
||||
std::map<boost::uuids::uuid, ShardId> initialized_rsms;
|
||||
std::set<msgs::SuggestedSplitInfo> suggested_splits;
|
||||
};
|
||||
|
||||
struct HeartbeatResponse {
|
||||
std::vector<ShardToInitialize> shards_to_initialize;
|
||||
std::vector<ShardToSplit> shards_to_split;
|
||||
std::vector<boost::uuids::uuid> acknowledged_initialized_rsms;
|
||||
};
|
||||
|
||||
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
|
||||
|
||||
struct LabelSpace {
|
||||
@@ -173,7 +128,6 @@ struct LabelSpace {
|
||||
// Maps between the smallest primary key stored in the shard and the shard
|
||||
std::map<PrimaryKey, ShardMetadata> shards;
|
||||
size_t replication_factor;
|
||||
uint64_t split_threshold;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
|
||||
using utils::print_helpers::operator<<;
|
||||
@@ -209,15 +163,17 @@ struct ShardMap {
|
||||
// TODO(gabor) later we will want to update the wallclock time with
|
||||
// the given Io<impl>'s time as well
|
||||
Hlc IncrementShardMapVersion() noexcept;
|
||||
Hlc GetHlc() noexcept;
|
||||
Hlc GetHlc() const noexcept;
|
||||
|
||||
std::unordered_map<uint64_t, std::string> IdToNames();
|
||||
|
||||
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, uint64_t split_threshold,
|
||||
Hlc last_shard_map_version);
|
||||
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
|
||||
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
|
||||
|
||||
boost::uuids::uuid NewShardUuid();
|
||||
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
|
||||
|
||||
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, Hlc last_shard_map_version);
|
||||
|
||||
void AddServer(Address server_address);
|
||||
|
||||
@@ -243,9 +199,6 @@ struct ShardMap {
|
||||
/// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
|
||||
/// there is also an active leader for each shard.
|
||||
bool ClusterInitialized() const;
|
||||
|
||||
/// Returns the current count of all shards that are fully initialized.
|
||||
size_t InitializedShards() const;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
|
||||
@@ -120,10 +120,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
|
||||
case Error::STALE_SHARD_MAP:
|
||||
throw ExpressionRuntimeException(
|
||||
"Cluster performed a Shard split or merge that invalidated the transaction's metadata. This should have "
|
||||
"been handled in the RequestRouter.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -457,7 +457,6 @@ TypedValueT Properties(const TypedValueT *args, int64_t nargs, const FunctionCon
|
||||
case common::ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case common::ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
case common::ErrorCode::OBJECT_NOT_FOUND:
|
||||
case common::ErrorCode::STALE_SHARD_MAP:
|
||||
throw functions::FunctionRuntimeException("Unexpected error when getting properties.");
|
||||
}
|
||||
}
|
||||
@@ -533,7 +532,6 @@ inline size_t UnwrapDegreeResult(storage::v3::ShardResult<size_t> maybe_degree)
|
||||
case common::ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case common::ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
case common::ErrorCode::OBJECT_NOT_FOUND:
|
||||
case common::ErrorCode::STALE_SHARD_MAP:
|
||||
throw functions::FunctionRuntimeException("Unexpected error when getting node degree.");
|
||||
}
|
||||
}
|
||||
@@ -721,7 +719,6 @@ TypedValueT Labels(const TypedValueT *args, int64_t nargs, const FunctionContext
|
||||
case common::ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case common::ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
case common::ErrorCode::OBJECT_NOT_FOUND:
|
||||
case common::ErrorCode::STALE_SHARD_MAP:
|
||||
throw functions::FunctionRuntimeException("Unexpected error when getting labels.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,12 +14,7 @@
|
||||
namespace memgraph::io {
|
||||
// Signifies that a retriable operation was unable to
|
||||
// complete after a configured number of retries.
|
||||
struct RetriesExhausted {
|
||||
friend std::ostream &operator<<(std::ostream &in, const RetriesExhausted & /* retries_exhausted */) {
|
||||
in << "RetriesExhausted {}";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
struct RetriesExhausted {};
|
||||
|
||||
// Signifies that a request was unable to receive a response
|
||||
// within some configured timeout duration. It is important
|
||||
@@ -27,21 +22,5 @@ struct RetriesExhausted {
|
||||
// not signify that a request was not received or processed.
|
||||
// It may be the case that the request was fully processed
|
||||
// but that the response was not received.
|
||||
struct TimedOut {
|
||||
friend std::ostream &operator<<(std::ostream &in, const TimedOut & /* timed_out */) {
|
||||
in << "TimedOut {}";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
// This error signifies that a shard has been contacted that does
|
||||
// not match our expected shard version, and that we should retry
|
||||
// the operation so that we can ensure that we are talking to the
|
||||
// correct shard.
|
||||
struct ShardVersionMismatch {
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardVersionMismatch & /* shard_version_mismatch */) {
|
||||
in << "ShardVersionMismatch {}";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
struct TimedOut {};
|
||||
}; // namespace memgraph::io
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,7 +20,6 @@
|
||||
#include "io/local_transport/local_transport_handle.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
|
||||
namespace memgraph::io::local_transport {
|
||||
|
||||
@@ -31,19 +30,19 @@ class LocalTransport {
|
||||
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
|
||||
: local_transport_handle_(std::move(local_transport_handle)) {}
|
||||
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
|
||||
std::function<void()> fill_notifier, Duration timeout) {
|
||||
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
|
||||
to_address, from_address, std::move(request), timeout, fill_notifier);
|
||||
}
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
|
||||
return local_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
|
||||
}
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,7 +22,6 @@
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
|
||||
namespace memgraph::io::local_transport {
|
||||
|
||||
@@ -31,7 +30,7 @@ class LocalTransportHandle {
|
||||
mutable std::condition_variable cv_;
|
||||
bool should_shut_down_ = false;
|
||||
MessageHistogramCollector histograms_;
|
||||
RequestId request_id_counter_ = 1;
|
||||
RequestId request_id_counter_ = 0;
|
||||
|
||||
// the responses to requests that are being waited on
|
||||
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
|
||||
@@ -68,7 +67,7 @@ class LocalTransportHandle {
|
||||
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
|
||||
}
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
|
||||
std::unique_lock lock(mu_);
|
||||
|
||||
@@ -104,7 +103,7 @@ class LocalTransportHandle {
|
||||
return std::move(m_opt).value();
|
||||
}
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
auto type_info = TypeInfoFor(message);
|
||||
|
||||
@@ -139,7 +138,7 @@ class LocalTransportHandle {
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
|
||||
Duration timeout, std::function<void()> fill_notifier) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
|
||||
@@ -156,7 +155,7 @@ class LocalTransportHandle {
|
||||
const auto now = Now();
|
||||
const Time deadline = now + timeout;
|
||||
|
||||
RequestId request_id;
|
||||
RequestId request_id = 0;
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,7 +14,6 @@
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
#include "utils/type_info_ref.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
@@ -49,7 +48,7 @@ struct OpaqueMessage {
|
||||
/// Return is the full std::variant<Ts...> type that holds the
|
||||
/// full parameter pack without interfering with recursive
|
||||
/// narrowing expansion.
|
||||
template <typename Return, utils::Message Head, utils::Message... Rest>
|
||||
template <typename Return, Message Head, Message... Rest>
|
||||
std::optional<Return> Unpack(std::any &&a) {
|
||||
if (typeid(Head) == a.type()) {
|
||||
Head concrete = std::any_cast<Head>(std::move(a));
|
||||
@@ -68,12 +67,12 @@ struct OpaqueMessage {
|
||||
/// parameter pack for the types that they want to compare
|
||||
/// with the any and potentially include in the returned
|
||||
/// variant.
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
|
||||
return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
|
||||
}
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
|
||||
std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -13,8 +13,8 @@
|
||||
|
||||
#include <variant>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include <coordinator/coordinator.hpp>
|
||||
#include <io/rsm/raft.hpp>
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
|
||||
@@ -41,6 +41,6 @@ using CoordinatorMessages =
|
||||
using ShardMessages = std::variant<ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, AppendResponse,
|
||||
WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>;
|
||||
|
||||
using ShardManagerMessages = std::variant<msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>;
|
||||
using ShardManagerMessages = std::variant<WriteResponse<CoordinatorWriteResponses>>;
|
||||
|
||||
} // namespace memgraph::io::messages
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,23 +19,17 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <thread>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/rsm/shard_rsm.hpp"
|
||||
#include "io/simulator/simulator.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
|
||||
namespace memgraph::io::rsm {
|
||||
|
||||
template <typename Type>
|
||||
concept HasShouldSplit = std::is_member_function_pointer<decltype(&Type::ShouldSplit)>::value;
|
||||
|
||||
/// Timeout and replication tunables
|
||||
using namespace std::chrono_literals;
|
||||
static constexpr auto kMinimumElectionTimeout = 100ms;
|
||||
@@ -348,21 +342,15 @@ class Raft {
|
||||
}
|
||||
}
|
||||
|
||||
template <typename = void>
|
||||
requires HasShouldSplit<ReplicatedState> std::optional<msgs::SuggestedSplitInfo> ShouldSplit() {
|
||||
return replicated_state_.ShouldSplit();
|
||||
}
|
||||
|
||||
private :
|
||||
// Raft paper - 5.3
|
||||
// When the entry has been safely replicated, the leader applies the
|
||||
// entry to its state machine and returns the result of that
|
||||
// execution to the client.
|
||||
//
|
||||
// "Safely replicated" is defined as being known to be present
|
||||
// on at least a majority of all peers (inclusive of the Leader).
|
||||
void
|
||||
BumpCommitIndexAndReplyToClients(Leader &leader) {
|
||||
private:
|
||||
// Raft paper - 5.3
|
||||
// When the entry has been safely replicated, the leader applies the
|
||||
// entry to its state machine and returns the result of that
|
||||
// execution to the client.
|
||||
//
|
||||
// "Safely replicated" is defined as being known to be present
|
||||
// on at least a majority of all peers (inclusive of the Leader).
|
||||
void BumpCommitIndexAndReplyToClients(Leader &leader) {
|
||||
auto confirmed_log_sizes = std::vector<LogSize>{};
|
||||
|
||||
// We include our own log size in the calculation of the log
|
||||
@@ -414,17 +402,13 @@ class Raft {
|
||||
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
|
||||
leader.pending_client_requests.erase(apply_index);
|
||||
|
||||
if (client_request.request_id == 0) {
|
||||
Log("not replying to Raft request with explicit request ID of 0");
|
||||
} else {
|
||||
const WriteResponse<WriteResponseValue> resp{
|
||||
.success = true,
|
||||
.write_return = std::move(write_return),
|
||||
.raft_index = apply_index,
|
||||
};
|
||||
const WriteResponse<WriteResponseValue> resp{
|
||||
.success = true,
|
||||
.write_return = std::move(write_return),
|
||||
.raft_index = apply_index,
|
||||
};
|
||||
|
||||
io_.Send(client_request.address, client_request.request_id, std::move(resp));
|
||||
}
|
||||
io_.Send(client_request.address, client_request.request_id, std::move(resp));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -640,7 +624,7 @@ class Raft {
|
||||
MG_ASSERT(std::max(req.term, state_.term) == req.term);
|
||||
}
|
||||
|
||||
VoteResponse res{
|
||||
const VoteResponse res{
|
||||
.term = std::max(req.term, state_.term),
|
||||
.committed_log_size = state_.committed_log_size,
|
||||
.vote_granted = new_leader,
|
||||
|
||||
@@ -65,7 +65,7 @@ class RsmClient {
|
||||
size_t addr_index = io_.Rand(addr_distrib);
|
||||
leader_ = server_addrs_[addr_index];
|
||||
|
||||
spdlog::trace("RsmClient selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
|
||||
spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
|
||||
}
|
||||
|
||||
template <typename ResponseT>
|
||||
@@ -73,7 +73,7 @@ class RsmClient {
|
||||
if (response.retry_leader) {
|
||||
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
|
||||
leader_ = response.retry_leader.value();
|
||||
spdlog::trace("RsmClient redirected to leader server {}", leader_.ToString());
|
||||
spdlog::debug("client redirected to leader server {}", leader_.ToString());
|
||||
}
|
||||
if (!response.success) {
|
||||
SelectRandomLeader();
|
||||
@@ -139,10 +139,6 @@ class RsmClient {
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
if (!async_reads_.contains(readiness_token.GetId())) {
|
||||
spdlog::debug("RsmClient async_reads_ does not contain Polled readiness token {}", readiness_token.GetId());
|
||||
return std::nullopt;
|
||||
}
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
@@ -162,7 +158,7 @@ class RsmClient {
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("RsmClient timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
return TimedOut{};
|
||||
}
|
||||
@@ -175,8 +171,7 @@ class RsmClient {
|
||||
|
||||
if (read_get_response.success) {
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
spdlog::trace("RsmClient returning read_return and erasing async state for RSM request for token {}",
|
||||
readiness_token.GetId());
|
||||
spdlog::debug("returning read_return for RSM request");
|
||||
return std::move(read_get_response.read_return);
|
||||
}
|
||||
} else {
|
||||
@@ -214,10 +209,6 @@ class RsmClient {
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
if (!async_writes_.contains(readiness_token.GetId())) {
|
||||
spdlog::debug("RsmClient async_writes_ does not contain Polled readiness token {}", readiness_token.GetId());
|
||||
return std::nullopt;
|
||||
}
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
@@ -237,7 +228,7 @@ class RsmClient {
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("RsmClient timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_writes_.erase(readiness_token.GetId());
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
@@ -22,30 +22,7 @@ namespace memgraph::io::simulator {
|
||||
|
||||
void SimulatorHandle::ShutDown() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
if (should_shut_down_) {
|
||||
spdlog::warn("Simulator's ShutDown method called multiple times.");
|
||||
return;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const size_t blocked_servers = blocked_on_receive_.size();
|
||||
|
||||
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
|
||||
|
||||
if (all_servers_blocked) {
|
||||
spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
|
||||
server_addresses_.size());
|
||||
break;
|
||||
}
|
||||
|
||||
spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
|
||||
server_addresses_.size());
|
||||
cv_.wait(lock);
|
||||
}
|
||||
|
||||
should_shut_down_ = true;
|
||||
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
std::move(dop).promise.TimeOut();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -32,7 +32,6 @@
|
||||
#include "io/simulator/simulator_stats.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
|
||||
@@ -59,16 +58,15 @@ class SimulatorHandle {
|
||||
std::uniform_int_distribution<int> drop_distrib_{0, 99};
|
||||
SimulatorConfig config_;
|
||||
MessageHistogramCollector histograms_;
|
||||
RequestId request_id_counter_{1};
|
||||
RequestId request_id_counter_{0};
|
||||
|
||||
bool TimeoutPromisesPastDeadline() {
|
||||
bool timed_anything_out = false;
|
||||
const Time now = cluster_wide_time_microseconds_;
|
||||
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
if (dop.deadline < now && config_.perform_timeouts) {
|
||||
spdlog::trace("Simulator timing out request from requester {}.", promise_key.requester_address.ToString());
|
||||
spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
|
||||
@@ -78,7 +76,6 @@ class SimulatorHandle {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
|
||||
return timed_anything_out;
|
||||
}
|
||||
|
||||
@@ -108,13 +105,13 @@ class SimulatorHandle {
|
||||
|
||||
bool ShouldShutDown() const;
|
||||
|
||||
template <utils::Message Request, utils::Message Response>
|
||||
template <Message Request, Message Response>
|
||||
ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
|
||||
std::function<bool()> &&maybe_tick_simulator,
|
||||
std::function<void()> &&fill_notifier) {
|
||||
auto type_info = TypeInfoFor(request);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
spdlog::trace("Simulator sending request {} to {}", demangled_name, to_address);
|
||||
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
|
||||
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<Response>>(
|
||||
// set notifier for when the Future::Wait is called
|
||||
@@ -158,7 +155,7 @@ class SimulatorHandle {
|
||||
return std::move(future);
|
||||
}
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
@@ -185,21 +182,20 @@ class SimulatorHandle {
|
||||
if (!should_shut_down_) {
|
||||
if (!blocked_on_receive_.contains(receiver)) {
|
||||
blocked_on_receive_.emplace(receiver);
|
||||
spdlog::trace("Simulator blocking receiver until it receives something or a timeout happens {}",
|
||||
receiver.ToPartialAddress().port);
|
||||
spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
|
||||
cv_.notify_all();
|
||||
}
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
spdlog::trace("Simulator timing out receiver {}", receiver.ToPartialAddress().port);
|
||||
spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
|
||||
spdlog::trace("Simulator sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
|
||||
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
|
||||
auto type_info = TypeInfoFor(message);
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/simulator/simulator_handle.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
|
||||
@@ -34,7 +33,7 @@ class SimulatorTransport {
|
||||
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
|
||||
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
|
||||
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
|
||||
std::function<void()> notification, Duration timeout) {
|
||||
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
|
||||
@@ -45,12 +44,12 @@ class SimulatorTransport {
|
||||
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
|
||||
}
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
|
||||
return simulator_handle_->template Receive<Ms...>(receiver_address, timeout);
|
||||
}
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
|
||||
return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,16 +22,21 @@
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
using memgraph::utils::BasicResult;
|
||||
|
||||
// TODO(tyler) ensure that Message continues to represent
|
||||
// reasonable constraints around message types over time,
|
||||
// as we adapt things to use Thrift-generated message types.
|
||||
template <typename T>
|
||||
concept Message = std::same_as<T, std::decay_t<T>>;
|
||||
|
||||
using RequestId = uint64_t;
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
struct ResponseEnvelope {
|
||||
M message;
|
||||
RequestId request_id;
|
||||
@@ -40,16 +45,16 @@ struct ResponseEnvelope {
|
||||
Duration response_latency;
|
||||
};
|
||||
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
using ResponseResult = BasicResult<TimedOut, ResponseEnvelope<M>>;
|
||||
|
||||
template <utils::Message M>
|
||||
using ResponseFuture = Future<ResponseResult<M>>;
|
||||
template <Message M>
|
||||
using ResponseFuture = memgraph::io::Future<ResponseResult<M>>;
|
||||
|
||||
template <utils::Message M>
|
||||
using ResponsePromise = Promise<ResponseResult<M>>;
|
||||
template <Message M>
|
||||
using ResponsePromise = memgraph::io::Promise<ResponseResult<M>>;
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
struct RequestEnvelope {
|
||||
std::variant<Ms...> message;
|
||||
RequestId request_id;
|
||||
@@ -57,7 +62,7 @@ struct RequestEnvelope {
|
||||
Address from_address;
|
||||
};
|
||||
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
using RequestResult = BasicResult<TimedOut, RequestEnvelope<Ms...>>;
|
||||
|
||||
template <typename I>
|
||||
@@ -77,7 +82,7 @@ class Io {
|
||||
Duration GetDefaultTimeout() { return default_timeout_; }
|
||||
|
||||
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = nullptr;
|
||||
@@ -87,7 +92,7 @@ class Io {
|
||||
|
||||
/// Issue a request that times out after the default timeout. This tends
|
||||
/// to be used by clients.
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
|
||||
const Duration timeout = default_timeout_;
|
||||
const Address from_address = address_;
|
||||
@@ -97,7 +102,7 @@ class Io {
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
|
||||
ReadinessToken readiness_token) {
|
||||
const Duration timeout = default_timeout_;
|
||||
@@ -108,7 +113,7 @@ class Io {
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <utils::Message RequestT, utils::Message ResponseT>
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
|
||||
ReadinessToken readiness_token, Duration timeout) {
|
||||
const Address from_address = address_;
|
||||
@@ -119,14 +124,14 @@ class Io {
|
||||
|
||||
/// Wait for an explicit number of microseconds for a request of one of the
|
||||
/// provided types to arrive. This tends to be used by servers.
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
|
||||
return implementation_.template Receive<Ms...>(address_, timeout);
|
||||
}
|
||||
|
||||
/// Wait the default number of microseconds for a request of one of the
|
||||
/// provided types to arrive. This tends to be used by servers.
|
||||
template <utils::Message... Ms>
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
|
||||
const Duration timeout = default_timeout_;
|
||||
return implementation_.template Receive<Ms...>(address_, timeout);
|
||||
@@ -135,7 +140,7 @@ class Io {
|
||||
/// Send a message in a best-effort fashion. This is used for messaging where
|
||||
/// responses are not necessarily expected, and for servers to respond to requests.
|
||||
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
|
||||
template <utils::Message M>
|
||||
template <Message M>
|
||||
void Send(Address to_address, RequestId request_id, M message) {
|
||||
Address from_address = address_;
|
||||
return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
|
||||
@@ -156,7 +161,6 @@ class Io {
|
||||
}
|
||||
|
||||
Address GetAddress() { return address_; }
|
||||
|
||||
void SetAddress(Address address) { address_ = address; }
|
||||
|
||||
Io<I> ForkLocal(boost::uuids::uuid uuid) {
|
||||
@@ -169,16 +173,5 @@ class Io {
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
|
||||
|
||||
template <utils::Message M>
|
||||
utils::Sender<M> GetSender(Address address) {
|
||||
Io<I> io_copy = Io(implementation_, address_);
|
||||
|
||||
std::function<void(M)> sender = [address, io_copy](M &&message) mutable {
|
||||
io_copy.template Send<M>(address, 0, std::forward<M>(message));
|
||||
};
|
||||
|
||||
return utils::Sender{sender};
|
||||
}
|
||||
};
|
||||
}; // namespace memgraph::io
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -80,7 +80,7 @@ class MachineManager {
|
||||
config_(config),
|
||||
coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
|
||||
shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
|
||||
coordinator_address_, config_.sync_message_handling} {
|
||||
coordinator_address_} {
|
||||
auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
|
||||
CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
|
||||
coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
|
||||
@@ -93,7 +93,6 @@ class MachineManager {
|
||||
|
||||
~MachineManager() {
|
||||
if (coordinator_handle_.joinable()) {
|
||||
MaybeBlockOnSyncHandling();
|
||||
coordinator_queue_.Push(coordinator::coordinator_worker::ShutDown{});
|
||||
coordinator_handle_.join();
|
||||
}
|
||||
@@ -115,7 +114,10 @@ class MachineManager {
|
||||
uint64_t next_us = next_cron_.time_since_epoch().count();
|
||||
|
||||
if (now >= next_cron_) {
|
||||
spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
|
||||
next_cron_ = Cron();
|
||||
} else {
|
||||
spdlog::info("now {} < next_cron_ {}", now_us, next_us);
|
||||
}
|
||||
|
||||
Duration receive_timeout = std::max(next_cron_, now) - now;
|
||||
@@ -125,17 +127,16 @@ class MachineManager {
|
||||
std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
|
||||
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse,
|
||||
WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
|
||||
AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>, msgs::SuggestedSplitInfo,
|
||||
msgs::InitializeSplitShard>;
|
||||
AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
|
||||
|
||||
spdlog::trace("MachineManager waiting on Receive on address {}", io_.GetAddress().ToString());
|
||||
spdlog::info("MM waiting on Receive on address {}", io_.GetAddress().ToString());
|
||||
|
||||
// Note: this parameter pack must be kept in-sync with the AllMessages parameter pack above
|
||||
auto request_result = io_.template ReceiveWithTimeout<
|
||||
ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
|
||||
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse, WriteResponse<CoordinatorWriteResponses>,
|
||||
ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>,
|
||||
msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>(receive_timeout);
|
||||
ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>(
|
||||
receive_timeout);
|
||||
|
||||
if (request_result.HasError()) {
|
||||
// time to do Cron
|
||||
@@ -144,13 +145,11 @@ class MachineManager {
|
||||
|
||||
auto &&request_envelope = std::move(request_result.GetValue());
|
||||
|
||||
spdlog::trace("MachineManager received message addressed to {}", request_envelope.to_address.ToString());
|
||||
spdlog::info("MM got message to {}", request_envelope.to_address.ToString());
|
||||
|
||||
// If message is for the coordinator, cast it to subset and pass it to the coordinator
|
||||
bool to_coordinator = coordinator_address_ == request_envelope.to_address;
|
||||
if (to_coordinator) {
|
||||
spdlog::trace("MachineManager got message addressed to the coordinator");
|
||||
|
||||
std::optional<CoordinatorMessages> conversion_attempt =
|
||||
ConvertVariant<AllMessages, ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>,
|
||||
AppendResponse, WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>(
|
||||
@@ -158,6 +157,8 @@ class MachineManager {
|
||||
|
||||
MG_ASSERT(conversion_attempt.has_value(), "coordinator message conversion failed");
|
||||
|
||||
spdlog::info("got coordinator message");
|
||||
|
||||
CoordinatorMessages &&cm = std::move(conversion_attempt.value());
|
||||
|
||||
CoordinatorRouteMessage route_message{
|
||||
@@ -171,20 +172,14 @@ class MachineManager {
|
||||
}
|
||||
|
||||
bool to_sm = shard_manager_.GetAddress() == request_envelope.to_address;
|
||||
spdlog::info("smm: {}", shard_manager_.GetAddress().ToString());
|
||||
if (to_sm) {
|
||||
spdlog::trace("MachineManager got shard manager message, addressed to {}", request_envelope.to_address);
|
||||
|
||||
std::optional<ShardManagerMessages> conversion_attempt =
|
||||
ConvertVariant<AllMessages, msgs::SuggestedSplitInfo, msgs::InitializeSplitShard>(
|
||||
std::move(request_envelope.message));
|
||||
ConvertVariant<AllMessages, WriteResponse<CoordinatorWriteResponses>>(std::move(request_envelope.message));
|
||||
|
||||
if (!conversion_attempt.has_value()) {
|
||||
spdlog::debug(
|
||||
"MachineManager dropping message with an unexpected type addressed to the ShardManager, assuming it was "
|
||||
"a HeartbeatResponse that "
|
||||
"timed out");
|
||||
continue;
|
||||
}
|
||||
MG_ASSERT(conversion_attempt.has_value(), "shard manager message conversion failed");
|
||||
|
||||
spdlog::info("got shard manager message");
|
||||
|
||||
ShardManagerMessages &&smm = std::move(conversion_attempt.value());
|
||||
shard_manager_.Receive(std::forward<ShardManagerMessages>(smm), request_envelope.request_id,
|
||||
@@ -202,7 +197,7 @@ class MachineManager {
|
||||
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {} - incorrect message type",
|
||||
request_envelope.to_address.ToString());
|
||||
|
||||
spdlog::trace("MachineManager forwarding message to shard rsm");
|
||||
spdlog::info("got shard rsm message");
|
||||
|
||||
ShardMessages &&sm = std::move(conversion_attempt.value());
|
||||
shard_manager_.Route(std::forward<ShardMessages>(sm), request_envelope.request_id, request_envelope.to_address,
|
||||
@@ -226,7 +221,7 @@ class MachineManager {
|
||||
}
|
||||
|
||||
Time Cron() {
|
||||
spdlog::trace("MachineManager running Cron, address {}", io_.GetAddress().ToString());
|
||||
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
|
||||
coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
|
||||
MaybeBlockOnSyncHandling();
|
||||
Time ret = shard_manager_.Cron();
|
||||
|
||||
@@ -622,16 +622,13 @@ int main(int argc, char **argv) {
|
||||
if (FLAGS_split_file.empty()) {
|
||||
const std::string property{"property"};
|
||||
const std::string label{"label"};
|
||||
// TODO(tyler) make this more easily configurable in the short-term
|
||||
const auto split_threshold = 4;
|
||||
|
||||
auto prop_map = sm.AllocatePropertyIds(std::vector<std::string>{property});
|
||||
auto edge_type_map = sm.AllocateEdgeTypeIds(std::vector<std::string>{"TO"});
|
||||
std::vector<memgraph::storage::v3::SchemaProperty> schema{
|
||||
{prop_map.at(property), memgraph::common::SchemaType::INT}};
|
||||
|
||||
// TODO(tyler) remove pre-initialization completely along with split files
|
||||
sm.InitializeNewLabel(label, schema, 1, split_threshold, sm.shard_map_version);
|
||||
sm.InitializeNewLabel(label, schema, 1, sm.shard_map_version);
|
||||
sm.SplitShard(sm.GetHlc(), *sm.GetLabelId(label),
|
||||
std::vector<memgraph::storage::v3::PropertyValue>{memgraph::storage::v3::PropertyValue{2}});
|
||||
} else {
|
||||
std::ifstream input{FLAGS_split_file, std::ios::in};
|
||||
MG_ASSERT(input.is_open(), "Cannot open split file to read: {}", FLAGS_split_file);
|
||||
|
||||
@@ -295,7 +295,6 @@ class Interpreter final {
|
||||
void Abort();
|
||||
|
||||
const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
|
||||
request_router_->InstallSimulatorTicker(tick_simulator);
|
||||
}
|
||||
|
||||
@@ -99,7 +99,6 @@ class RequestRouterInterface {
|
||||
|
||||
virtual ~RequestRouterInterface() = default;
|
||||
|
||||
virtual coordinator::Hlc RefreshShardMap() = 0;
|
||||
virtual void StartTransaction() = 0;
|
||||
virtual void Commit() = 0;
|
||||
virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) = 0;
|
||||
@@ -151,92 +150,77 @@ class RequestRouter : public RequestRouterInterface {
|
||||
notifier_.InstallSimulatorTicker(tick_simulator);
|
||||
}
|
||||
|
||||
coordinator::Hlc RefreshShardMap() override {
|
||||
coordinator::HlcRequest req{.last_shard_map_version = shard_map_.GetHlc()};
|
||||
void StartTransaction() override {
|
||||
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
spdlog::trace("RequestRouter sending hlc request to get an HLC and refresh the ShardMap");
|
||||
spdlog::trace("sending hlc request to start transaction");
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
spdlog::trace("RequestRouter received hlc response to start transaction");
|
||||
|
||||
// TODO(tyler) enforce max retries here
|
||||
while (write_res.HasError()) {
|
||||
spdlog::debug("RequestRouter retrying HlcRequest to coordinator after timeout");
|
||||
write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
spdlog::trace("received hlc response to start transaction");
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
// Transaction ID to be used later...
|
||||
transaction_id_ = hlc_response.new_hlc;
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
}
|
||||
|
||||
void Commit() override {
|
||||
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
spdlog::trace("sending hlc request before committing transaction");
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
spdlog::trace("received hlc response before committing transaction");
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request for commit failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shard_map_ = hlc_response.fresher_shard_map.value();
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
auto commit_timestamp = hlc_response.new_hlc;
|
||||
|
||||
return hlc_response.new_hlc;
|
||||
}
|
||||
|
||||
void StartTransaction() override {
|
||||
// Transaction ID to be used later...
|
||||
transaction_id_ = RefreshShardMap();
|
||||
}
|
||||
|
||||
bool CommitInner(coordinator::Hlc commit_timestamp) {
|
||||
msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
|
||||
|
||||
for (const auto &[label, space] : shard_map_.label_spaces) {
|
||||
for (const auto &[label, space] : shards_map_.label_spaces) {
|
||||
for (const auto &[key, shard] : space.shards) {
|
||||
auto &storage_client = GetStorageClientForShard(shard);
|
||||
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
|
||||
// instead.
|
||||
auto commit_response = storage_client.SendWriteRequest(commit_req);
|
||||
while (commit_response.HasError()) {
|
||||
spdlog::debug("RequestRouter retrying Commit request due to timeout");
|
||||
commit_response = storage_client.SendWriteRequest(commit_req);
|
||||
// RETRY on timeouts?
|
||||
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
|
||||
if (commit_response.HasError()) {
|
||||
throw std::runtime_error("Commit request timed out");
|
||||
}
|
||||
msgs::WriteResponses write_response_variant = commit_response.GetValue();
|
||||
auto &response = std::get<msgs::CommitResponse>(write_response_variant);
|
||||
if (response.error) {
|
||||
if (response.error->code == common::ErrorCode::STALE_SHARD_MAP) {
|
||||
RefreshShardMap();
|
||||
|
||||
// signal to caller that we should retry
|
||||
return false;
|
||||
} else {
|
||||
spdlog::warn("RequestRouter throwing because of unhandled commit failure 3: {}",
|
||||
common::ErrorCodeToString(response.error->code));
|
||||
throw std::runtime_error("Commit request did not succeed");
|
||||
}
|
||||
throw std::runtime_error("Commit request did not succeed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Commit() override {
|
||||
spdlog::trace("sending hlc request before committing transaction");
|
||||
auto commit_timestamp = RefreshShardMap();
|
||||
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// Commit is idempotent, so it's fine to retry it to the same shards if it fails
|
||||
bool success = CommitInner(commit_timestamp);
|
||||
if (success) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
|
||||
return shard_map_.GetEdgeTypeId(name).value();
|
||||
return shards_map_.GetEdgeTypeId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::PropertyId NameToProperty(const std::string &name) const override {
|
||||
return shard_map_.GetPropertyId(name).value();
|
||||
return shards_map_.GetPropertyId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::LabelId NameToLabel(const std::string &name) const override {
|
||||
return shard_map_.GetLabelId(name).value();
|
||||
return shards_map_.GetLabelId(name).value();
|
||||
}
|
||||
|
||||
const std::string &PropertyToName(storage::v3::PropertyId id) const override {
|
||||
@@ -248,8 +232,8 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
|
||||
const auto schema_it = shard_map_.schemas.find(primary_label);
|
||||
MG_ASSERT(schema_it != shard_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
|
||||
const auto schema_it = shards_map_.schemas.find(primary_label);
|
||||
MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
|
||||
|
||||
return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
|
||||
return schema_prop.property_id == property;
|
||||
@@ -257,136 +241,94 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const override {
|
||||
return shard_map_.schemas.at(label);
|
||||
return shards_map_.schemas.at(label);
|
||||
}
|
||||
|
||||
bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shard_map_.label_spaces.contains(label); }
|
||||
bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); }
|
||||
|
||||
// TODO(kostasrim) Simplify return result
|
||||
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// create requests
|
||||
auto requests_to_be_sent = RequestsForScanVertices(label);
|
||||
// create requests
|
||||
auto requests_to_be_sent = RequestsForScanVertices(label);
|
||||
|
||||
spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
|
||||
spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
auto responses_result =
|
||||
DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
|
||||
|
||||
if (responses_result.HasError()) {
|
||||
spdlog::debug(
|
||||
"RequestRouter refreshing ShardMap and re-sending requests for ScanVertices after Shard version mismatch");
|
||||
RefreshShardMap();
|
||||
continue;
|
||||
}
|
||||
|
||||
auto responses = responses_result.GetValue();
|
||||
|
||||
spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
|
||||
|
||||
// convert responses into VertexAccessor objects to return
|
||||
std::vector<VertexAccessor> accessors;
|
||||
accessors.reserve(responses.size());
|
||||
for (auto &response : responses) {
|
||||
for (auto &result_row : response.results) {
|
||||
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
|
||||
}
|
||||
}
|
||||
|
||||
return accessors;
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
|
||||
spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
|
||||
|
||||
// convert responses into VertexAccessor objects to return
|
||||
std::vector<VertexAccessor> accessors;
|
||||
accessors.reserve(responses.size());
|
||||
for (auto &response : responses) {
|
||||
for (auto &result_row : response.results) {
|
||||
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
|
||||
}
|
||||
}
|
||||
|
||||
return accessors;
|
||||
}
|
||||
|
||||
std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) override {
|
||||
MG_ASSERT(!new_vertices.empty());
|
||||
|
||||
for (auto &new_vertex : new_vertices) {
|
||||
new_vertex.idempotency_token = idempotency_token_generator_++;
|
||||
}
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
|
||||
RequestsForCreateVertices(new_vertices);
|
||||
spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
|
||||
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
|
||||
RequestsForCreateVertices(new_vertices);
|
||||
spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
for (auto &new_vertex : request.request.new_vertices) {
|
||||
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
|
||||
}
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
auto result = DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
|
||||
|
||||
if (result.HasValue()) {
|
||||
return result.GetValue();
|
||||
} else {
|
||||
// retry request that failed due to outdated ShardMap
|
||||
RefreshShardMap();
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
for (auto &new_vertex : request.request.new_vertices) {
|
||||
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
|
||||
}
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
return DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
|
||||
}
|
||||
|
||||
std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) override {
|
||||
MG_ASSERT(!new_edges.empty());
|
||||
|
||||
for (auto &new_edge : new_edges) {
|
||||
new_edge.idempotency_token = idempotency_token_generator_++;
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent =
|
||||
RequestsForCreateExpand(std::move(new_edges));
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateExpandRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::WriteRequests req = request.request;
|
||||
storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent =
|
||||
RequestsForCreateExpand(new_edges);
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateExpandRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::WriteRequests req = request.request;
|
||||
storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto result = DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
|
||||
|
||||
if (result.HasValue()) {
|
||||
return result.GetValue();
|
||||
} else {
|
||||
// retry request that failed due to outdated ShardMap
|
||||
RefreshShardMap();
|
||||
}
|
||||
}
|
||||
// drive requests to completion
|
||||
return DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
|
||||
}
|
||||
|
||||
std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) override {
|
||||
@@ -396,107 +338,81 @@ class RequestRouter : public RequestRouterInterface {
|
||||
// For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
|
||||
// must be fetched again with an ExpandOne(Edges.dst)
|
||||
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ExpandOneRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses_result = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
|
||||
|
||||
if (responses_result.HasError()) {
|
||||
spdlog::debug(
|
||||
"RequestRouter refreshing ShardMap and re-sending requests for ExpandOne after Shard version mismatch");
|
||||
RefreshShardMap();
|
||||
continue;
|
||||
}
|
||||
|
||||
auto responses = responses_result.GetValue();
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::ExpandOneResultRow> result_rows;
|
||||
const auto total_row_count = std::accumulate(
|
||||
responses.begin(), responses.end(), 0, [](const int64_t partial_count, const msgs::ExpandOneResponse &resp) {
|
||||
return partial_count + resp.result.size();
|
||||
});
|
||||
result_rows.reserve(total_row_count);
|
||||
|
||||
for (auto &response : responses) {
|
||||
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
|
||||
std::make_move_iterator(response.result.end()));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ExpandOneRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::ExpandOneResultRow> result_rows;
|
||||
const auto total_row_count = std::accumulate(responses.begin(), responses.end(), 0,
|
||||
[](const int64_t partial_count, const msgs::ExpandOneResponse &resp) {
|
||||
return partial_count + resp.result.size();
|
||||
});
|
||||
result_rows.reserve(total_row_count);
|
||||
|
||||
for (auto &response : responses) {
|
||||
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
|
||||
std::make_move_iterator(response.result.end()));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
}
|
||||
|
||||
std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest requests) override {
|
||||
requests.transaction_id = transaction_id_;
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
|
||||
RequestsForGetProperties(std::move(requests));
|
||||
|
||||
// TODO(tyler) enforce max commit retries here when we go distributed
|
||||
while (true) {
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
|
||||
RequestsForGetProperties(requests);
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses_result =
|
||||
DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
|
||||
|
||||
if (responses_result.HasError()) {
|
||||
spdlog::debug(
|
||||
"RequestRouter refreshing ShardMap and re-sending requests for GetProperties after Shard version mismatch");
|
||||
RefreshShardMap();
|
||||
continue;
|
||||
}
|
||||
|
||||
auto responses = responses_result.GetValue();
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::GetPropertiesResultRow> result_rows;
|
||||
|
||||
for (auto &&response : responses) {
|
||||
std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::GetPropertiesResultRow> result_rows;
|
||||
|
||||
for (auto &&response : responses) {
|
||||
std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
}
|
||||
|
||||
std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
|
||||
return shard_map_.GetPropertyId(name);
|
||||
return shards_map_.GetPropertyId(name);
|
||||
}
|
||||
|
||||
std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const override {
|
||||
return shard_map_.GetEdgeTypeId(name);
|
||||
return shards_map_.GetEdgeTypeId(name);
|
||||
}
|
||||
|
||||
std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const override {
|
||||
return shard_map_.GetLabelId(name);
|
||||
return shards_map_.GetLabelId(name);
|
||||
}
|
||||
|
||||
private:
|
||||
@@ -504,13 +420,12 @@ class RequestRouter : public RequestRouterInterface {
|
||||
const std::vector<msgs::NewVertex> &new_vertices) {
|
||||
std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
|
||||
|
||||
for (const auto &new_vertex : new_vertices) {
|
||||
for (auto &new_vertex : new_vertices) {
|
||||
MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
|
||||
auto shard = shard_map_.GetShardForKey(new_vertex.label_ids[0].id,
|
||||
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
|
||||
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
|
||||
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
msgs::CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_,
|
||||
.shard_map_version = shard_map_.shard_map_version};
|
||||
msgs::CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
|
||||
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
|
||||
}
|
||||
per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
|
||||
@@ -530,7 +445,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> RequestsForCreateExpand(
|
||||
const std::vector<msgs::NewExpand> &new_expands) {
|
||||
std::vector<msgs::NewExpand> new_expands) {
|
||||
std::map<ShardMetadata, msgs::CreateExpandRequest> per_shard_request_table;
|
||||
auto ensure_shard_exists_in_table = [&per_shard_request_table,
|
||||
transaction_id = transaction_id_](const ShardMetadata &shard) {
|
||||
@@ -541,14 +456,14 @@ class RequestRouter : public RequestRouterInterface {
|
||||
};
|
||||
|
||||
for (auto &new_expand : new_expands) {
|
||||
const auto shard_src_vertex = shard_map_.GetShardForKey(
|
||||
const auto shard_src_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
|
||||
const auto shard_dest_vertex = shard_map_.GetShardForKey(
|
||||
const auto shard_dest_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
|
||||
|
||||
ensure_shard_exists_in_table(shard_src_vertex);
|
||||
|
||||
if (shard_src_vertex.peers != shard_dest_vertex.peers) {
|
||||
if (shard_src_vertex != shard_dest_vertex) {
|
||||
ensure_shard_exists_in_table(shard_dest_vertex);
|
||||
per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
|
||||
}
|
||||
@@ -572,12 +487,12 @@ class RequestRouter : public RequestRouterInterface {
|
||||
const std::optional<std::string> &label) {
|
||||
std::vector<coordinator::Shards> multi_shards;
|
||||
if (label) {
|
||||
const auto label_id = shard_map_.GetLabelId(*label);
|
||||
const auto label_id = shards_map_.GetLabelId(*label);
|
||||
MG_ASSERT(label_id);
|
||||
MG_ASSERT(IsPrimaryLabel(*label_id));
|
||||
multi_shards = {shard_map_.GetShardsForLabel(*label)};
|
||||
multi_shards = {shards_map_.GetShardsForLabel(*label)};
|
||||
} else {
|
||||
multi_shards = shard_map_.GetAllShards();
|
||||
multi_shards = shards_map_.GetAllShards();
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
|
||||
@@ -588,7 +503,6 @@ class RequestRouter : public RequestRouterInterface {
|
||||
|
||||
msgs::ScanVerticesRequest request;
|
||||
request.transaction_id = transaction_id_;
|
||||
request.shard_map_version = shard_map_.shard_map_version;
|
||||
request.start_id.second = storage::conversions::ConvertValueVector(key);
|
||||
|
||||
ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
|
||||
@@ -611,7 +525,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
|
||||
for (auto &vertex : request.src_vertices) {
|
||||
auto shard =
|
||||
shard_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
@@ -633,7 +547,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> RequestsForGetProperties(
|
||||
const msgs::GetPropertiesRequest &request) {
|
||||
msgs::GetPropertiesRequest &&request) {
|
||||
std::map<ShardMetadata, msgs::GetPropertiesRequest> per_shard_request_table;
|
||||
auto top_level_rqst_template = request;
|
||||
top_level_rqst_template.transaction_id = transaction_id_;
|
||||
@@ -642,7 +556,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
|
||||
for (auto &&vertex : request.vertex_ids) {
|
||||
auto shard =
|
||||
shard_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
@@ -651,7 +565,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
|
||||
for (auto &[vertex, maybe_edge] : request.vertices_and_edges) {
|
||||
auto shard =
|
||||
shard_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
@@ -680,7 +594,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
|
||||
auto shard = shard_map_.GetShardForKey(label, key);
|
||||
auto shard = shards_map_.GetShardForKey(label, key);
|
||||
return GetStorageClientForShard(std::move(shard));
|
||||
}
|
||||
|
||||
@@ -697,21 +611,16 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
utils::BasicResult<io::ShardVersionMismatch, std::vector<ResponseT>> DriveReadResponses(
|
||||
RunningRequests<RequestT> &running_requests) {
|
||||
std::vector<ResponseT> DriveReadResponses(RunningRequests<RequestT> &running_requests) {
|
||||
// Store responses in a map based on the corresponding request
|
||||
// offset, so that they can be reassembled in the correct order
|
||||
// even if they came back in randomized orders.
|
||||
std::map<size_t, ResponseT> response_map;
|
||||
|
||||
spdlog::trace("waiting on readiness for token in DriveReadResponses");
|
||||
size_t polls = 0;
|
||||
spdlog::trace("waiting on readiness for token");
|
||||
while (response_map.size() < running_requests.size()) {
|
||||
auto ready = notifier_.Await();
|
||||
spdlog::trace("got readiness for token {}", ready.GetId());
|
||||
|
||||
MG_ASSERT(polls++ / running_requests.size() < 1000,
|
||||
"polled over 1000 times per operation (almost certainly due to a bug) when performing request");
|
||||
auto &request = running_requests.at(ready.GetId());
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
|
||||
@@ -723,23 +632,13 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
storage_client.SendAsyncReadRequest(request.request, notifier_, ready);
|
||||
continue;
|
||||
throw std::runtime_error("RequestRouter Read request timed out");
|
||||
}
|
||||
|
||||
msgs::ReadResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<ResponseT>(response_variant);
|
||||
if (response.error) {
|
||||
if (response.error->code == common::ErrorCode::STALE_SHARD_MAP) {
|
||||
RefreshShardMap();
|
||||
|
||||
// signal to caller that we should retry
|
||||
return io::ShardVersionMismatch{};
|
||||
} else {
|
||||
spdlog::warn("throwing in DriveReadResponses because of unhandled error: {}",
|
||||
common::ErrorCodeToString(response.error->code));
|
||||
throw std::runtime_error("RequestRouter Read request did not succeed");
|
||||
}
|
||||
throw std::runtime_error("RequestRouter Read request did not succeed");
|
||||
}
|
||||
|
||||
// the readiness token has an ID based on the request vector offset
|
||||
@@ -760,8 +659,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
utils::BasicResult<io::ShardVersionMismatch, std::vector<ResponseT>> DriveWriteResponses(
|
||||
RunningRequests<RequestT> &running_requests) {
|
||||
std::vector<ResponseT> DriveWriteResponses(RunningRequests<RequestT> &running_requests) {
|
||||
// Store responses in a map based on the corresponding request
|
||||
// offset, so that they can be reassembled in the correct order
|
||||
// even if they came back in randomized orders.
|
||||
@@ -780,23 +678,13 @@ class RequestRouter : public RequestRouterInterface {
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
storage_client.SendAsyncWriteRequest(request.request, notifier_, ready);
|
||||
continue;
|
||||
throw std::runtime_error("RequestRouter Write request timed out");
|
||||
}
|
||||
|
||||
msgs::WriteResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<ResponseT>(response_variant);
|
||||
if (response.error) {
|
||||
if (response.error->code == common::ErrorCode::STALE_SHARD_MAP) {
|
||||
RefreshShardMap();
|
||||
|
||||
// signal to caller that we should retry
|
||||
return io::ShardVersionMismatch{};
|
||||
} else {
|
||||
spdlog::warn("throwing in DriveWriteResponses because of unhandled error: {}",
|
||||
common::ErrorCodeToString(response.error->code));
|
||||
throw std::runtime_error("RequestRouter Write request did not succeed");
|
||||
}
|
||||
throw std::runtime_error("RequestRouter Write request did not succeed");
|
||||
}
|
||||
|
||||
// the readiness token has an ID based on the request vector offset
|
||||
@@ -818,17 +706,17 @@ class RequestRouter : public RequestRouterInterface {
|
||||
|
||||
void SetUpNameIdMappers() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_name;
|
||||
for (const auto &[name, id] : shard_map_.labels) {
|
||||
for (const auto &[name, id] : shards_map_.labels) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
labels_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shard_map_.properties) {
|
||||
for (const auto &[name, id] : shards_map_.properties) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
properties_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shard_map_.edge_types) {
|
||||
for (const auto &[name, id] : shards_map_.edge_types) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
edge_types_.StoreMapping(std::move(id_to_name));
|
||||
@@ -851,7 +739,7 @@ class RequestRouter : public RequestRouterInterface {
|
||||
return {};
|
||||
}
|
||||
|
||||
ShardMap shard_map_;
|
||||
ShardMap shards_map_;
|
||||
storage::v3::NameIdMapper properties_;
|
||||
storage::v3::NameIdMapper edge_types_;
|
||||
storage::v3::NameIdMapper labels_;
|
||||
@@ -860,7 +748,6 @@ class RequestRouter : public RequestRouterInterface {
|
||||
io::Io<TTransport> io_;
|
||||
coordinator::Hlc transaction_id_;
|
||||
io::Notifier notifier_ = {};
|
||||
std::atomic<uint64_t> idempotency_token_generator_;
|
||||
// TODO(kostasrim) Add batch prefetching
|
||||
};
|
||||
|
||||
@@ -913,19 +800,15 @@ class SimulatedRequestRouterFactory : public RequestRouterFactory {
|
||||
io::Address unique_local_addr_query;
|
||||
|
||||
// The simulated RR should not introduce stochastic behavior.
|
||||
random_uuid = boost::uuids::uuid{3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
|
||||
random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
|
||||
|
||||
auto io = simulator_->Register(unique_local_addr_query);
|
||||
auto query_io = io.ForkLocal(random_uuid);
|
||||
|
||||
auto ret = std::make_unique<RequestRouter<TransportType>>(
|
||||
return std::make_unique<RequestRouter<TransportType>>(
|
||||
coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
|
||||
std::move(io));
|
||||
|
||||
ret->InstallSimulatorTicker(simulator_->GetSimulatorTickClosure());
|
||||
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
@@ -22,13 +21,10 @@
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
|
||||
namespace memgraph::msgs {
|
||||
@@ -344,7 +340,6 @@ enum class StorageView { OLD = 0, NEW = 1 };
|
||||
|
||||
struct ScanVerticesRequest {
|
||||
Hlc transaction_id;
|
||||
Hlc shard_map_version;
|
||||
// This should be optional
|
||||
VertexId start_id;
|
||||
// The empty optional means return all of the properties, while an empty list means do not return any properties
|
||||
@@ -492,7 +487,6 @@ struct UpdateEdgeProp {
|
||||
* Vertices
|
||||
*/
|
||||
struct NewVertex {
|
||||
uint64_t idempotency_token;
|
||||
std::vector<Label> label_ids;
|
||||
PrimaryKey primary_key;
|
||||
// This should be a map
|
||||
@@ -501,7 +495,6 @@ struct NewVertex {
|
||||
|
||||
struct CreateVerticesRequest {
|
||||
Hlc transaction_id;
|
||||
Hlc shard_map_version;
|
||||
std::vector<NewVertex> new_vertices;
|
||||
};
|
||||
|
||||
@@ -535,7 +528,6 @@ struct UpdateVerticesResponse {
|
||||
// No need for specifying direction since it has to be in one, and src and dest
|
||||
// vertices clearly communicate the direction
|
||||
struct NewExpand {
|
||||
uint64_t idempotency_token;
|
||||
EdgeId id;
|
||||
EdgeType type;
|
||||
VertexId src_vertex;
|
||||
@@ -579,65 +571,13 @@ struct CommitResponse {
|
||||
std::optional<ShardError> error;
|
||||
};
|
||||
|
||||
struct SuggestedSplitInfo {
|
||||
boost::uuids::uuid shard_to_split_uuid;
|
||||
LabelId label_id;
|
||||
PrimaryKey splitting_shard_low_key;
|
||||
PrimaryKey split_key;
|
||||
Hlc shard_version;
|
||||
|
||||
friend bool operator<(const SuggestedSplitInfo &lhs, const SuggestedSplitInfo &rhs) {
|
||||
if (lhs.shard_to_split_uuid != rhs.shard_to_split_uuid) {
|
||||
return lhs.shard_to_split_uuid < rhs.shard_to_split_uuid;
|
||||
}
|
||||
|
||||
/*
|
||||
TODO(tyler) fix this
|
||||
if (lhs.split_key != rhs.split_key) {
|
||||
return lhs.split_key < rhs.split_key;
|
||||
}
|
||||
*/
|
||||
|
||||
return lhs.shard_version < rhs.shard_version;
|
||||
}
|
||||
};
|
||||
|
||||
struct SplitInfo {
|
||||
PrimaryKey split_key;
|
||||
boost::uuids::uuid right_side_uuid;
|
||||
Hlc shard_version;
|
||||
};
|
||||
|
||||
struct SplitRequest {
|
||||
PrimaryKey split_key;
|
||||
Hlc old_shard_version;
|
||||
Hlc new_lhs_shard_version;
|
||||
Hlc new_rhs_shard_version;
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
|
||||
};
|
||||
|
||||
struct InitializeSplitShard {
|
||||
// TODO(jbajic) Make it unique by solving that is not copyable int std::any
|
||||
std::shared_ptr<storage::v3::Shard> shard;
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> uuid_mapping;
|
||||
};
|
||||
|
||||
struct InitializeSplitShardByUUID {
|
||||
std::shared_ptr<storage::v3::Shard> shard;
|
||||
boost::uuids::uuid shard_uuid;
|
||||
};
|
||||
|
||||
struct SplitResponse {};
|
||||
|
||||
using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
|
||||
using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse>;
|
||||
|
||||
using WriteRequests =
|
||||
std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest, CreateExpandRequest,
|
||||
DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest, SplitRequest>;
|
||||
using WriteResponses =
|
||||
std::variant<CreateVerticesResponse, DeleteVerticesResponse, UpdateVerticesResponse, CreateExpandResponse,
|
||||
DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse, SplitResponse>;
|
||||
using WriteRequests = std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest,
|
||||
CreateExpandRequest, DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest>;
|
||||
using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesResponse, UpdateVerticesResponse,
|
||||
CreateExpandResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>;
|
||||
|
||||
} // namespace memgraph::msgs
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ set(storage_v3_src_files
|
||||
bindings/typed_value.cpp
|
||||
expr.cpp
|
||||
vertex.cpp
|
||||
splitter.cpp
|
||||
request_helper.cpp)
|
||||
|
||||
# ######################
|
||||
|
||||
@@ -81,10 +81,8 @@ class DbAccessor final {
|
||||
storage::v3::ShardResult<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
const storage::v3::EdgeTypeId &edge_type) {
|
||||
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
|
||||
const uint64_t dummy_idempotency_token = 0;
|
||||
auto maybe_edge =
|
||||
accessor_->CreateEdge(from->Id(storage::v3::View::NEW).GetValue(), to->Id(storage::v3::View::NEW).GetValue(),
|
||||
edge_type, kDummyGid, dummy_idempotency_token);
|
||||
auto maybe_edge = accessor_->CreateEdge(from->Id(storage::v3::View::NEW).GetValue(),
|
||||
to->Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
|
||||
if (maybe_edge.HasError()) return {maybe_edge.GetError()};
|
||||
return EdgeAccessor(*maybe_edge);
|
||||
}
|
||||
|
||||
@@ -30,10 +30,6 @@ struct Config {
|
||||
io::Duration reclamation_interval{};
|
||||
} gc;
|
||||
|
||||
struct Split {
|
||||
uint64_t max_shard_vertex_size{500'000};
|
||||
} split;
|
||||
|
||||
struct Items {
|
||||
bool properties_on_edges{true};
|
||||
} items;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -13,14 +13,12 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "storage/v3/edge_ref.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
@@ -29,11 +27,6 @@ struct Edge;
|
||||
struct Delta;
|
||||
struct CommitInfo;
|
||||
|
||||
inline uint64_t GetNextDeltaId() noexcept {
|
||||
static utils::Synchronized<uint64_t, utils::SpinLock> delta_id{0};
|
||||
return delta_id.WithLock([](auto &id) { return id++; });
|
||||
}
|
||||
|
||||
// This class stores one of three pointers (`Delta`, `Vertex` and `Edge`)
|
||||
// without using additional memory for storing the type. The type is stored in
|
||||
// the pointer itself in the lower bits. All of those structures contain large
|
||||
@@ -165,59 +158,46 @@ struct Delta {
|
||||
struct RemoveInEdgeTag {};
|
||||
struct RemoveOutEdgeTag {};
|
||||
|
||||
Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id,
|
||||
uint64_t idempotency_token)
|
||||
: action(Action::DELETE_OBJECT),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
idempotency_token(idempotency_token) {}
|
||||
Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
|
||||
: action(Action::DELETE_OBJECT), commit_info(commit_info), command_id(command_id) {}
|
||||
|
||||
Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
|
||||
: action(Action::RECREATE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
|
||||
Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
|
||||
: action(Action::RECREATE_OBJECT), commit_info(commit_info), command_id(command_id) {}
|
||||
|
||||
Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
|
||||
: action(Action::ADD_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
|
||||
Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
|
||||
: action(Action::ADD_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
|
||||
|
||||
Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
|
||||
: action(Action::REMOVE_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
|
||||
Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
|
||||
: action(Action::REMOVE_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
|
||||
|
||||
Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, CommitInfo *commit_info,
|
||||
uint64_t delta_id, uint64_t command_id)
|
||||
: action(Action::SET_PROPERTY),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
property({key, value}) {}
|
||||
uint64_t command_id)
|
||||
: action(Action::SET_PROPERTY), commit_info(commit_info), command_id(command_id), property({key, value}) {}
|
||||
|
||||
Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
|
||||
uint64_t delta_id, uint64_t command_id)
|
||||
uint64_t command_id)
|
||||
: action(Action::ADD_IN_EDGE),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
|
||||
|
||||
Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
|
||||
uint64_t delta_id, uint64_t command_id)
|
||||
uint64_t command_id)
|
||||
: action(Action::ADD_OUT_EDGE),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
|
||||
|
||||
Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
|
||||
uint64_t delta_id, uint64_t command_id)
|
||||
uint64_t command_id)
|
||||
: action(Action::REMOVE_IN_EDGE),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
|
||||
|
||||
Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
|
||||
uint64_t delta_id, uint64_t command_id)
|
||||
uint64_t command_id)
|
||||
: action(Action::REMOVE_OUT_EDGE),
|
||||
id(delta_id),
|
||||
commit_info(commit_info),
|
||||
command_id(command_id),
|
||||
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
|
||||
@@ -246,16 +226,13 @@ struct Delta {
|
||||
}
|
||||
}
|
||||
|
||||
friend bool operator==(const Delta &lhs, const Delta &rhs) noexcept { return lhs.id == rhs.id; }
|
||||
|
||||
Action action;
|
||||
uint64_t id;
|
||||
|
||||
// TODO: optimize with in-place copy
|
||||
CommitInfo *commit_info;
|
||||
uint64_t command_id;
|
||||
PreviousPtr prev;
|
||||
Delta *next{nullptr};
|
||||
uint64_t idempotency_token;
|
||||
|
||||
union {
|
||||
LabelId label;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -325,7 +325,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
|
||||
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
|
||||
@@ -353,7 +353,7 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
}
|
||||
}
|
||||
|
||||
LabelIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction,
|
||||
LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: index_container_(&index_container),
|
||||
label_(label),
|
||||
@@ -465,7 +465,7 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
|
||||
}
|
||||
}
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
|
||||
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
|
||||
@@ -526,7 +526,7 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
|
||||
const PropertyValue kSmallestTemporalData =
|
||||
PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
|
||||
LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
|
||||
Transaction *transaction, Indices *indices, Config::Items config,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,8 +18,6 @@
|
||||
#include <utility>
|
||||
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
@@ -42,18 +40,12 @@ class LabelIndex {
|
||||
bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
|
||||
};
|
||||
|
||||
using IndexType = LabelId;
|
||||
|
||||
public:
|
||||
using IndexContainer = std::set<Entry>;
|
||||
using LabelIndexContainer = std::set<Entry>;
|
||||
|
||||
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
|
||||
|
||||
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator,
|
||||
std::map<LabelId, IndexContainer> &data)
|
||||
: index_{std::move(data)}, indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
|
||||
|
||||
@@ -71,12 +63,12 @@ class LabelIndex {
|
||||
|
||||
class Iterable {
|
||||
public:
|
||||
Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
|
||||
Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
|
||||
Config::Items config, const VertexValidator &vertex_validator);
|
||||
|
||||
class Iterator {
|
||||
public:
|
||||
Iterator(Iterable *self, IndexContainer::iterator index_iterator);
|
||||
Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
|
||||
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
@@ -89,7 +81,7 @@ class LabelIndex {
|
||||
void AdvanceUntilValid();
|
||||
|
||||
Iterable *self_;
|
||||
IndexContainer::iterator index_iterator_;
|
||||
LabelIndexContainer::iterator index_iterator_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
@@ -98,7 +90,7 @@ class LabelIndex {
|
||||
Iterator end() { return {this, index_container_->end()}; }
|
||||
|
||||
private:
|
||||
IndexContainer *index_container_;
|
||||
LabelIndexContainer *index_container_;
|
||||
LabelId label_;
|
||||
View view_;
|
||||
Transaction *transaction_;
|
||||
@@ -122,35 +114,8 @@ class LabelIndex {
|
||||
|
||||
void Clear() { index_.clear(); }
|
||||
|
||||
std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
|
||||
if (index_.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Cloned index entries will contain new index entry iterators, but old
|
||||
// vertices address which need to be adjusted after extracting vertices
|
||||
std::map<IndexType, IndexContainer> cloned_indices;
|
||||
for (auto &[index_type_val, index] : index_) {
|
||||
auto entry_it = index.begin();
|
||||
auto &cloned_indices_container = cloned_indices[index_type_val];
|
||||
while (entry_it != index.end()) {
|
||||
// We need to save the next pointer since the current one will be
|
||||
// invalidated after extract
|
||||
auto next_entry_it = std::next(entry_it);
|
||||
if (entry_it->vertex->first > split_key) {
|
||||
[[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
|
||||
cloned_indices_container.insert(index.extract(entry_it));
|
||||
MG_ASSERT(inserted, "Failed to extract index entry!");
|
||||
}
|
||||
entry_it = next_entry_it;
|
||||
}
|
||||
}
|
||||
|
||||
return cloned_indices;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<LabelId, IndexContainer> index_;
|
||||
std::map<LabelId, LabelIndexContainer> index_;
|
||||
Indices *indices_;
|
||||
Config::Items config_;
|
||||
const VertexValidator *vertex_validator_;
|
||||
@@ -168,10 +133,9 @@ class LabelPropertyIndex {
|
||||
bool operator<(const PropertyValue &rhs) const;
|
||||
bool operator==(const PropertyValue &rhs) const;
|
||||
};
|
||||
using IndexType = std::pair<LabelId, PropertyId>;
|
||||
|
||||
public:
|
||||
using IndexContainer = std::set<Entry>;
|
||||
using LabelPropertyIndexContainer = std::set<Entry>;
|
||||
|
||||
LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
|
||||
@@ -195,14 +159,14 @@ class LabelPropertyIndex {
|
||||
|
||||
class Iterable {
|
||||
public:
|
||||
Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
|
||||
Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
|
||||
|
||||
class Iterator {
|
||||
public:
|
||||
Iterator(Iterable *self, IndexContainer::iterator index_iterator);
|
||||
Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
|
||||
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
@@ -215,7 +179,7 @@ class LabelPropertyIndex {
|
||||
void AdvanceUntilValid();
|
||||
|
||||
Iterable *self_;
|
||||
IndexContainer::iterator index_iterator_;
|
||||
LabelPropertyIndexContainer::iterator index_iterator_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
@@ -224,7 +188,7 @@ class LabelPropertyIndex {
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
IndexContainer *index_container_;
|
||||
LabelPropertyIndexContainer *index_container_;
|
||||
LabelId label_;
|
||||
PropertyId property_;
|
||||
std::optional<utils::Bound<PropertyValue>> lower_bound_;
|
||||
@@ -265,35 +229,8 @@ class LabelPropertyIndex {
|
||||
|
||||
void Clear() { index_.clear(); }
|
||||
|
||||
std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
|
||||
if (index_.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Cloned index entries will contain new index entry iterators, but old
|
||||
// vertices address which need to be adjusted after extracting vertices
|
||||
std::map<IndexType, IndexContainer> cloned_indices;
|
||||
for (auto &[index_type_val, index] : index_) {
|
||||
auto entry_it = index.begin();
|
||||
auto &cloned_index_container = cloned_indices[index_type_val];
|
||||
while (entry_it != index.end()) {
|
||||
// We need to save the next pointer since the current one will be
|
||||
// invalidated after extract
|
||||
auto next_entry_it = std::next(entry_it);
|
||||
if (entry_it->vertex->first > split_key) {
|
||||
[[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
|
||||
cloned_index_container.insert(index.extract(entry_it));
|
||||
MG_ASSERT(inserted, "Failed to extract index entry!");
|
||||
}
|
||||
entry_it = next_entry_it;
|
||||
}
|
||||
}
|
||||
|
||||
return cloned_indices;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::pair<LabelId, PropertyId>, IndexContainer> index_;
|
||||
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
|
||||
Indices *indices_;
|
||||
Config::Items config_;
|
||||
const VertexValidator *vertex_validator_;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -107,9 +107,9 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
|
||||
/// and is primarily used to create the first delta for an object (that must be
|
||||
/// a `DELETE_OBJECT` delta).
|
||||
/// @throw std::bad_alloc
|
||||
inline Delta *CreateDeleteObjectDelta(Transaction *transaction, uint64_t idempotency_token) {
|
||||
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(), GetNextDeltaId(),
|
||||
transaction->command_id, idempotency_token);
|
||||
inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(),
|
||||
transaction->command_id);
|
||||
}
|
||||
|
||||
/// This function creates a delta in the transaction for the object and links
|
||||
@@ -119,7 +119,7 @@ template <typename TObj, class... Args>
|
||||
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
|
||||
inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
|
||||
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
|
||||
GetNextDeltaId(), transaction->command_id);
|
||||
transaction->command_id);
|
||||
auto *delta_holder = GetDeltaHolder(object);
|
||||
|
||||
// The operations are written in such order so that both `next` and `prev`
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,14 +18,14 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <variant>
|
||||
|
||||
#include <bits/ranges_algo.h>
|
||||
#include <gflags/gflags.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "storage/v3/delta.hpp"
|
||||
#include "storage/v3/edge.hpp"
|
||||
#include "storage/v3/edge_accessor.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/indices.hpp"
|
||||
@@ -36,7 +36,6 @@
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
@@ -324,90 +323,30 @@ bool VerticesIterable::Iterator::operator==(const Iterator &other) const {
|
||||
}
|
||||
|
||||
Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key,
|
||||
const std::optional<PrimaryKey> max_primary_key, std::vector<SchemaProperty> schema, Hlc shard_version,
|
||||
Config config, std::unordered_map<uint64_t, std::string> id_to_name)
|
||||
const std::optional<PrimaryKey> max_primary_key, std::vector<SchemaProperty> schema, Config config,
|
||||
std::unordered_map<uint64_t, std::string> id_to_name)
|
||||
: primary_label_{primary_label},
|
||||
min_primary_key_{min_primary_key},
|
||||
max_primary_key_{max_primary_key},
|
||||
shard_version_{shard_version},
|
||||
schema_validator_{schemas_, name_id_mapper_},
|
||||
vertex_validator_{schema_validator_, primary_label},
|
||||
indices_{config.items, vertex_validator_},
|
||||
isolation_level_{config.transaction.isolation_level},
|
||||
config_{config},
|
||||
shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
|
||||
name_id_mapper_) {
|
||||
spdlog::trace("Shard constructed with low key {} (path 1)", min_primary_key_.back());
|
||||
uuid_{utils::GenerateUUID()},
|
||||
epoch_id_{utils::GenerateUUID()},
|
||||
global_locker_{file_retainer_.AddLocker()} {
|
||||
CreateSchema(primary_label_, schema);
|
||||
StoreMapping(std::move(id_to_name));
|
||||
}
|
||||
|
||||
Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
|
||||
const std::unordered_map<uint64_t, std::string> &id_to_name, const Hlc shard_version)
|
||||
: primary_label_{primary_label},
|
||||
min_primary_key_{min_primary_key},
|
||||
max_primary_key_{max_primary_key},
|
||||
vertices_(std::move(vertices)),
|
||||
edges_(std::move(edges)),
|
||||
shard_version_(shard_version),
|
||||
schema_validator_{schemas_, name_id_mapper_},
|
||||
vertex_validator_{schema_validator_, primary_label},
|
||||
indices_{config.items, vertex_validator_},
|
||||
isolation_level_{config.transaction.isolation_level},
|
||||
config_{config},
|
||||
start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)),
|
||||
shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
|
||||
name_id_mapper_) {
|
||||
spdlog::trace("Shard constructed with low key {} (path 2)", min_primary_key_.back());
|
||||
CreateSchema(primary_label_, schema);
|
||||
StoreMapping(id_to_name);
|
||||
}
|
||||
|
||||
Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, VertexContainer &&vertices,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
|
||||
const std::unordered_map<uint64_t, std::string> &id_to_name, const Hlc shard_version)
|
||||
: primary_label_{primary_label},
|
||||
min_primary_key_{min_primary_key},
|
||||
max_primary_key_{max_primary_key},
|
||||
vertices_(std::move(vertices)),
|
||||
shard_version_(shard_version),
|
||||
schema_validator_{schemas_, name_id_mapper_},
|
||||
vertex_validator_{schema_validator_, primary_label},
|
||||
indices_{config.items, vertex_validator_},
|
||||
isolation_level_{config.transaction.isolation_level},
|
||||
config_{config},
|
||||
start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)),
|
||||
shard_splitter_(primary_label, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_, schema,
|
||||
name_id_mapper_) {
|
||||
spdlog::trace("Shard constructed with low key {} (path 3)", min_primary_key_.back());
|
||||
CreateSchema(primary_label_, schema);
|
||||
StoreMapping(id_to_name);
|
||||
}
|
||||
|
||||
Shard::~Shard() {}
|
||||
|
||||
std::unique_ptr<Shard> Shard::FromSplitData(SplitData &&split_data) {
|
||||
if (split_data.config.items.properties_on_edges) [[likely]] {
|
||||
return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
|
||||
split_data.schema, std::move(split_data.vertices), std::move(*split_data.edges),
|
||||
std::move(split_data.transactions), split_data.config, split_data.id_to_name,
|
||||
split_data.shard_version);
|
||||
}
|
||||
const auto low_key_int = split_data.min_primary_key.back();
|
||||
spdlog::trace("Shard constructed FromSplitData with low key {}", low_key_int);
|
||||
return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
|
||||
split_data.schema, std::move(split_data.vertices), std::move(split_data.transactions),
|
||||
split_data.config, split_data.id_to_name, split_data.shard_version);
|
||||
}
|
||||
|
||||
Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
|
||||
: shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
|
||||
|
||||
std::optional<ShardError> Shard::Accessor::CreateVertexAndValidate(
|
||||
const uint64_t idempotency_token, const std::vector<LabelId> &labels, const PrimaryKey &primary_properties,
|
||||
ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
|
||||
const std::vector<LabelId> &labels, const PrimaryKey &primary_properties,
|
||||
const std::vector<std::pair<PropertyId, PropertyValue>> &properties) {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
const auto schema = shard_->GetSchema(shard_->primary_label_)->second;
|
||||
@@ -418,19 +357,14 @@ std::optional<ShardError> Shard::Accessor::CreateVertexAndValidate(
|
||||
return {std::move(maybe_schema_violation.GetError())};
|
||||
}
|
||||
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_, idempotency_token);
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_);
|
||||
auto [it, inserted] = shard_->vertices_.emplace(primary_properties, VertexData{delta});
|
||||
delta->prev.Set(&*it);
|
||||
|
||||
VertexAccessor vertex_acc{&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
|
||||
if (!inserted) {
|
||||
if (it->second.delta->idempotency_token == idempotency_token) {
|
||||
// this is a benign race condition due to request retries - signal success
|
||||
return std::nullopt;
|
||||
}
|
||||
return SHARD_ERROR(ErrorCode::VERTEX_ALREADY_INSERTED);
|
||||
}
|
||||
|
||||
delta->prev.Set(&*it);
|
||||
VertexAccessor vertex_acc{&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
|
||||
MG_ASSERT(it != shard_->vertices_.end(), "Invalid Vertex accessor!");
|
||||
|
||||
// TODO(jbajic) Improve, maybe delay index update
|
||||
@@ -447,8 +381,7 @@ std::optional<ShardError> Shard::Accessor::CreateVertexAndValidate(
|
||||
return {err.GetError()};
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
return vertex_acc;
|
||||
}
|
||||
|
||||
std::optional<VertexAccessor> Shard::Accessor::FindVertex(std::vector<PropertyValue> primary_key, View view) {
|
||||
@@ -503,7 +436,7 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
|
||||
}
|
||||
|
||||
std::vector<EdgeAccessor> deleted_edges;
|
||||
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)};
|
||||
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)}; // TODO Replace
|
||||
for (const auto &item : in_edges) {
|
||||
auto [edge_type, from_vertex, edge] = item;
|
||||
EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
|
||||
@@ -548,8 +481,7 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
|
||||
}
|
||||
|
||||
ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id,
|
||||
const EdgeTypeId edge_type, const Gid gid,
|
||||
const uint64_t idempotency_token) {
|
||||
const EdgeTypeId edge_type, const Gid gid) {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
Vertex *from_vertex{nullptr};
|
||||
Vertex *to_vertex{nullptr};
|
||||
@@ -583,7 +515,7 @@ ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, V
|
||||
|
||||
EdgeRef edge(gid);
|
||||
if (config_.properties_on_edges) {
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_, idempotency_token);
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_);
|
||||
auto [it, inserted] = shard_->edges_.emplace(gid, Edge{gid, delta});
|
||||
MG_ASSERT(inserted, "The edge must be inserted here!");
|
||||
MG_ASSERT(it != shard_->edges_.end(), "Invalid Edge accessor!");
|
||||
@@ -732,16 +664,9 @@ void Shard::Accessor::Commit(coordinator::Hlc commit_timestamp) {
|
||||
MG_ASSERT(!transaction_->must_abort, "The transaction can't be committed!");
|
||||
MG_ASSERT(transaction_->start_timestamp.logical_id < commit_timestamp.logical_id,
|
||||
"Commit timestamp must be older than start timestamp!");
|
||||
if (transaction_->commit_info->is_locally_committed) {
|
||||
MG_ASSERT(transaction_->commit_info->start_or_commit_timestamp == commit_timestamp);
|
||||
spdlog::debug(
|
||||
"Shard::Accessor::Commit called for already-committed transaction, probably due to a retry request that timed "
|
||||
"out for the RequestRouter");
|
||||
} else {
|
||||
MG_ASSERT(transaction_->commit_info->start_or_commit_timestamp != commit_timestamp);
|
||||
transaction_->commit_info->start_or_commit_timestamp = commit_timestamp;
|
||||
transaction_->commit_info->is_locally_committed = true;
|
||||
}
|
||||
MG_ASSERT(!transaction_->commit_info->is_locally_committed, "The transaction is already committed!");
|
||||
transaction_->commit_info->start_or_commit_timestamp = commit_timestamp;
|
||||
transaction_->commit_info->is_locally_committed = true;
|
||||
}
|
||||
|
||||
void Shard::Accessor::Abort() {
|
||||
@@ -972,8 +897,6 @@ StorageInfo Shard::GetInfo() const {
|
||||
return {vertex_count, edge_count_, average_degree, utils::GetMemoryUsage()};
|
||||
}
|
||||
|
||||
Hlc Shard::Version() const { return shard_version_; }
|
||||
|
||||
VerticesIterable Shard::Accessor::Vertices(LabelId label, View view) {
|
||||
return VerticesIterable(shard_->indices_.label_index.Vertices(label, view, transaction_));
|
||||
}
|
||||
@@ -1125,47 +1048,6 @@ void Shard::StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name) {
|
||||
name_id_mapper_.StoreMapping(std::move(id_to_name));
|
||||
}
|
||||
|
||||
std::optional<ShardSuggestedSplitInfo> Shard::ShouldSplit() const noexcept {
|
||||
if (vertices_.size() >= config_.split.max_shard_vertex_size) {
|
||||
spdlog::debug("Shard::ShouldSplit is signalling that the split process may begin");
|
||||
auto mid_elem = vertices_.begin();
|
||||
|
||||
// TODO(tyler) the first time we calculate the split point, we should store it so that we don't have to
|
||||
// iterate over half of the entire index each time Cron is run until the split succeeds.
|
||||
std::ranges::advance(mid_elem, static_cast<VertexContainer::difference_type>(vertices_.size() / 2));
|
||||
|
||||
spdlog::debug("Shard::ShouldSplit returning split key with end Value {} for shard version {}",
|
||||
mid_elem->first.back(), shard_version_.logical_id);
|
||||
|
||||
return ShardSuggestedSplitInfo{
|
||||
.label_id = PrimaryLabel(),
|
||||
.splitting_shard_low_key = min_primary_key_,
|
||||
.split_key = mid_elem->first,
|
||||
.shard_version = shard_version_,
|
||||
};
|
||||
}
|
||||
|
||||
spdlog::trace("Shard not splitting because we have {} vertices, lower than split threshold of {}", vertices_.size(),
|
||||
config_.split.max_shard_vertex_size);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<SplitData> Shard::PerformSplit(const PrimaryKey &split_key, const Hlc old_shard_version,
|
||||
const Hlc new_lhs_shard_version, const Hlc new_rhs_shard_version) {
|
||||
if (old_shard_version != shard_version_) {
|
||||
spdlog::debug("Shard::PerformSplit - Curent shard version {} does not match given {}", shard_version_,
|
||||
old_shard_version);
|
||||
return std::nullopt;
|
||||
}
|
||||
spdlog::debug("Shard::PerformSplit - splitting from shard version {} to versions {} and {}", old_shard_version,
|
||||
new_lhs_shard_version, new_rhs_shard_version);
|
||||
|
||||
shard_version_ = new_lhs_shard_version;
|
||||
const auto old_max_key = max_primary_key_;
|
||||
max_primary_key_ = split_key;
|
||||
return shard_splitter_.SplitShard(split_key, old_max_key, new_rhs_shard_version);
|
||||
}
|
||||
|
||||
bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const {
|
||||
return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ &&
|
||||
(!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,15 +14,12 @@
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <shared_mutex>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "io/time.hpp"
|
||||
@@ -40,13 +37,11 @@
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/splitter.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/concrete_msg_sender.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
@@ -58,8 +53,6 @@
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
using coordinator::Hlc;
|
||||
|
||||
// The storage is based on this paper:
|
||||
// https://db.in.tum.de/~muehlbau/papers/mvcc.pdf
|
||||
// The paper implements a fully serializable storage, in our implementation we
|
||||
@@ -189,31 +182,13 @@ struct StorageInfo {
|
||||
uint64_t memory_usage;
|
||||
};
|
||||
|
||||
struct ShardSuggestedSplitInfo {
|
||||
boost::uuids::uuid shard_to_split_uuid;
|
||||
LabelId label_id;
|
||||
PrimaryKey splitting_shard_low_key;
|
||||
PrimaryKey split_key;
|
||||
Hlc shard_version;
|
||||
};
|
||||
|
||||
class Shard final {
|
||||
public:
|
||||
/// @throw std::system_error
|
||||
/// @throw std::bad_alloc
|
||||
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, Hlc shard_version, Config config = Config(),
|
||||
std::unordered_map<uint64_t, std::string> id_to_name = {});
|
||||
|
||||
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
|
||||
const std::unordered_map<uint64_t, std::string> &id_to_name, Hlc shard_version);
|
||||
|
||||
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, VertexContainer &&vertices,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
|
||||
const std::unordered_map<uint64_t, std::string> &id_to_name, Hlc shard_version);
|
||||
explicit Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
|
||||
std::vector<SchemaProperty> schema, Config config = Config(),
|
||||
std::unordered_map<uint64_t, std::string> id_to_name = {});
|
||||
|
||||
Shard(const Shard &) = delete;
|
||||
Shard(Shard &&) noexcept = delete;
|
||||
@@ -221,8 +196,6 @@ class Shard final {
|
||||
Shard operator=(Shard &&) noexcept = delete;
|
||||
~Shard();
|
||||
|
||||
static std::unique_ptr<Shard> FromSplitData(SplitData &&split_data);
|
||||
|
||||
class Accessor final {
|
||||
private:
|
||||
friend class Shard;
|
||||
@@ -231,9 +204,8 @@ class Shard final {
|
||||
|
||||
public:
|
||||
/// @throw std::bad_alloc
|
||||
std::optional<ShardError> CreateVertexAndValidate(
|
||||
const uint64_t idempotency_token, const std::vector<LabelId> &labels,
|
||||
const std::vector<PropertyValue> &primary_properties,
|
||||
ShardResult<VertexAccessor> CreateVertexAndValidate(
|
||||
const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
|
||||
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
|
||||
|
||||
std::optional<VertexAccessor> FindVertex(std::vector<PropertyValue> primary_key, View view);
|
||||
@@ -294,8 +266,7 @@ class Shard final {
|
||||
VertexAccessor *vertex);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid,
|
||||
const uint64_t idempotency_token);
|
||||
ShardResult<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
|
||||
|
||||
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
@@ -358,10 +329,6 @@ class Shard final {
|
||||
|
||||
LabelId PrimaryLabel() const;
|
||||
|
||||
Hlc Version() const;
|
||||
|
||||
PrimaryKey LowKey() const { return min_primary_key_; }
|
||||
|
||||
[[nodiscard]] bool IsVertexBelongToShard(const VertexId &vertex_id) const;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
@@ -393,11 +360,6 @@ class Shard final {
|
||||
|
||||
void StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name);
|
||||
|
||||
std::optional<ShardSuggestedSplitInfo> ShouldSplit() const noexcept;
|
||||
|
||||
std::optional<SplitData> PerformSplit(const PrimaryKey &split_key, Hlc old_shard_version,
|
||||
const Hlc new_lhs_shard_version, const Hlc new_rhs_shard_version);
|
||||
|
||||
private:
|
||||
Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level);
|
||||
|
||||
@@ -415,7 +377,6 @@ class Shard final {
|
||||
// list is used only when properties are enabled for edges. Because of that we
|
||||
// keep a separate count of edges that is always updated.
|
||||
uint64_t edge_count_{0};
|
||||
Hlc shard_version_{};
|
||||
|
||||
SchemaValidator schema_validator_;
|
||||
VertexValidator vertex_validator_;
|
||||
@@ -435,10 +396,41 @@ class Shard final {
|
||||
// storage.
|
||||
std::list<Gid> deleted_edges_;
|
||||
|
||||
// UUID used to distinguish snapshots and to link snapshots to WALs
|
||||
std::string uuid_;
|
||||
// Sequence number used to keep track of the chain of WALs.
|
||||
uint64_t wal_seq_num_{0};
|
||||
|
||||
// UUID to distinguish different main instance runs for replication process
|
||||
// on SAME storage.
|
||||
// Multiple instances can have same storage UUID and be MAIN at the same time.
|
||||
// We cannot compare commit timestamps of those instances if one of them
|
||||
// becomes the replica of the other so we use epoch_id_ as additional
|
||||
// discriminating property.
|
||||
// Example of this:
|
||||
// We have 2 instances of the same storage, S1 and S2.
|
||||
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
|
||||
// At the moment when S1 commited a transaction with timestamp 20, and S2
|
||||
// a different transaction with timestamp 15, we change S2's role to REPLICA
|
||||
// and register it on S1.
|
||||
// Without using the epoch_id, we don't know that S1 and S2 have completely
|
||||
// different transactions, we think that the S2 is behind only by 5 commits.
|
||||
std::string epoch_id_;
|
||||
// History of the previous epoch ids.
|
||||
// Each value consists of the epoch id along the last commit belonging to that
|
||||
// epoch.
|
||||
std::deque<std::pair<std::string, uint64_t>> epoch_history_;
|
||||
|
||||
uint64_t wal_unsynced_transactions_{0};
|
||||
|
||||
utils::FileRetainer file_retainer_;
|
||||
|
||||
// Global locker that is used for clients file locking
|
||||
utils::FileRetainer::FileLocker global_locker_;
|
||||
|
||||
// Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but
|
||||
// haven't been cleaned up yet
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> start_logical_id_to_transaction_{};
|
||||
Splitter shard_splitter_;
|
||||
bool has_any_transaction_aborted_since_last_gc{false};
|
||||
};
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,9 +12,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <queue>
|
||||
#include <set>
|
||||
#include <unordered_map>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/functional/hash.hpp>
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
@@ -32,8 +31,6 @@
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/shard_rsm.hpp"
|
||||
#include "storage/v3/shard_worker.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
@@ -45,6 +42,7 @@ using coordinator::HeartbeatRequest;
|
||||
using coordinator::HeartbeatResponse;
|
||||
using io::Address;
|
||||
using io::Duration;
|
||||
using io::Message;
|
||||
using io::RequestId;
|
||||
using io::ResponseFuture;
|
||||
using io::Time;
|
||||
@@ -76,17 +74,14 @@ static_assert(kMinimumCronInterval < kMaximumCronInterval,
|
||||
/// * reconciling the storage engine's local configuration with the Coordinator's
|
||||
/// intentions for how it should participate in multiple raft clusters
|
||||
/// * replying to heartbeat requests to the Coordinator
|
||||
/// * routing incoming messages to the appropriate RSM
|
||||
/// * routing incoming messages to the appropriate sRSM
|
||||
///
|
||||
/// Every storage engine has exactly one RsmEngine.
|
||||
template <typename IoImpl>
|
||||
class ShardManager {
|
||||
public:
|
||||
ShardManager(io::Io<IoImpl> io, size_t shard_worker_threads, Address coordinator_leader,
|
||||
bool serialize_shard_splits_for_determinisim = false)
|
||||
: io_(io),
|
||||
coordinator_leader_(coordinator_leader),
|
||||
serialize_shard_splits_for_determinisim_(serialize_shard_splits_for_determinisim) {
|
||||
ShardManager(io::Io<IoImpl> io, size_t shard_worker_threads, Address coordinator_leader)
|
||||
: io_(io), coordinator_leader_(coordinator_leader) {
|
||||
MG_ASSERT(shard_worker_threads >= 1);
|
||||
|
||||
for (int i = 0; i < shard_worker_threads; i++) {
|
||||
@@ -154,7 +149,7 @@ class ShardManager {
|
||||
/// Periodic protocol maintenance. Returns the time that Cron should be called again
|
||||
/// in the future.
|
||||
Time Cron() {
|
||||
spdlog::trace("ShardManager running Cron, address {}", io_.GetAddress().ToString());
|
||||
spdlog::info("running ShardManager::Cron, address {}", io_.GetAddress().ToString());
|
||||
Time now = io_.Now();
|
||||
|
||||
if (now >= next_reconciliation_) {
|
||||
@@ -179,39 +174,7 @@ class ShardManager {
|
||||
/// Returns the Address for our underlying Io implementation
|
||||
Address GetAddress() { return io_.GetAddress(); }
|
||||
|
||||
void InitializeSplitShard(msgs::InitializeSplitShard &&init_split_shard) {
|
||||
spdlog::trace("ShardManager received InitializeSplitShard message");
|
||||
for (const auto &[from_uuid, new_uuid] : init_split_shard.uuid_mapping) {
|
||||
bool has_source = rsm_worker_mapping_.contains(from_uuid);
|
||||
if (has_source) {
|
||||
const auto low_key = init_split_shard.shard->LowKey();
|
||||
coordinator::ShardId new_shard_id = std::make_pair(init_split_shard.shard->PrimaryLabel(), low_key);
|
||||
|
||||
spdlog::debug("ShardManager initialized split shard {} with uuid {} and low key {}",
|
||||
init_split_shard.shard->Version().logical_id, new_uuid, low_key.back());
|
||||
msgs::InitializeSplitShardByUUID msg{.shard = std::move(init_split_shard.shard), .shard_uuid = new_uuid};
|
||||
SendToWorkerByUuid(new_uuid, std::move(msg));
|
||||
|
||||
initialized_but_not_confirmed_rsm_.emplace(new_uuid, new_shard_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Receive(ShardManagerMessages &&smm, RequestId request_id, Address from) {
|
||||
std::visit(utils::Overloaded{[this](msgs::SuggestedSplitInfo &&split_info) {
|
||||
spdlog::debug(
|
||||
"ShardManager adding new suggested split info to the pending_splits_ structure");
|
||||
pending_splits_.emplace(std::move(split_info));
|
||||
},
|
||||
[this](msgs::InitializeSplitShard &&init_split_shard) {
|
||||
// TODO(jbajic) remove pending split for this completed split
|
||||
// TODO(jbajic) Add new shard to initialized but not confirmed rsm
|
||||
spdlog::debug("ShardManager received a new split shard that it will now initialize");
|
||||
InitializeSplitShard(std::move(init_split_shard));
|
||||
}},
|
||||
std::move(smm));
|
||||
}
|
||||
void Receive(ShardManagerMessages &&smm, RequestId request_id, Address from) {}
|
||||
|
||||
void Route(ShardMessages &&sm, RequestId request_id, Address to, Address from) {
|
||||
Address address = io_.GetAddress();
|
||||
@@ -238,13 +201,14 @@ class ShardManager {
|
||||
std::vector<shard_worker::Queue> workers_;
|
||||
std::vector<std::jthread> worker_handles_;
|
||||
std::vector<size_t> worker_rsm_counts_;
|
||||
std::set<msgs::SuggestedSplitInfo> pending_splits_;
|
||||
std::unordered_map<uuid, size_t, boost::hash<boost::uuids::uuid>> rsm_worker_mapping_;
|
||||
Time next_reconciliation_ = Time::min();
|
||||
Address coordinator_leader_;
|
||||
std::optional<ResponseFuture<WriteResponse<CoordinatorWriteResponses>>> heartbeat_res_;
|
||||
std::map<boost::uuids::uuid, coordinator::ShardId> initialized_but_not_confirmed_rsm_;
|
||||
bool serialize_shard_splits_for_determinisim_;
|
||||
|
||||
// TODO(tyler) over time remove items from initialized_but_not_confirmed_rsm_
|
||||
// after the Coordinator is clearly aware of them
|
||||
std::set<boost::uuids::uuid> initialized_but_not_confirmed_rsm_;
|
||||
|
||||
void Reconciliation() {
|
||||
if (heartbeat_res_.has_value()) {
|
||||
@@ -254,18 +218,18 @@ class ShardManager {
|
||||
heartbeat_res_.reset();
|
||||
|
||||
if (response_result.HasError()) {
|
||||
spdlog::info("ShardManager timed out while trying to reach Coordinator");
|
||||
spdlog::error("SM timed out while trying to reach C");
|
||||
} else {
|
||||
auto response_envelope = response_result.GetValue();
|
||||
WriteResponse<CoordinatorWriteResponses> wr = response_envelope.message;
|
||||
|
||||
if (wr.retry_leader.has_value()) {
|
||||
spdlog::info("ShardManager redirected to new Coordinator leader");
|
||||
spdlog::info("SM redirected to new C leader");
|
||||
coordinator_leader_ = wr.retry_leader.value();
|
||||
} else if (wr.success) {
|
||||
CoordinatorWriteResponses cwr = wr.write_return;
|
||||
HeartbeatResponse hr = std::get<HeartbeatResponse>(cwr);
|
||||
spdlog::info("ShardManager received heartbeat response from Coordinator");
|
||||
spdlog::info("SM received heartbeat response from C");
|
||||
|
||||
EnsureShardsInitialized(hr);
|
||||
}
|
||||
@@ -278,35 +242,28 @@ class ShardManager {
|
||||
HeartbeatRequest req{
|
||||
.from_storage_manager = GetAddress(),
|
||||
.initialized_rsms = initialized_but_not_confirmed_rsm_,
|
||||
.suggested_splits = std::move(pending_splits_),
|
||||
};
|
||||
|
||||
CoordinatorWriteRequests cwr = req;
|
||||
WriteRequest<CoordinatorWriteRequests> ww;
|
||||
ww.operation = cwr;
|
||||
|
||||
spdlog::info("ShardManager sending heartbeat to coordinator {} with {} initialized rsms",
|
||||
coordinator_leader_.ToString(), initialized_but_not_confirmed_rsm_.size());
|
||||
spdlog::info("SM sending heartbeat to coordinator {}", coordinator_leader_.ToString());
|
||||
heartbeat_res_.emplace(std::move(
|
||||
io_.template Request<WriteRequest<CoordinatorWriteRequests>, WriteResponse<CoordinatorWriteResponses>>(
|
||||
coordinator_leader_, ww)));
|
||||
spdlog::info("ShardManager sent heartbeat");
|
||||
spdlog::info("SM sent heartbeat");
|
||||
}
|
||||
|
||||
void EnsureShardsInitialized(HeartbeatResponse hr) {
|
||||
for (const auto &acknowledged_rsm : hr.acknowledged_initialized_rsms) {
|
||||
initialized_but_not_confirmed_rsm_.erase(acknowledged_rsm);
|
||||
}
|
||||
|
||||
for (const auto &to_init : hr.shards_to_initialize) {
|
||||
coordinator::ShardId new_shard_id = std::make_pair(to_init.label_id, to_init.min_key);
|
||||
spdlog::trace("ShardManager has been told to initialize shard rsm with uuid {}", to_init.uuid);
|
||||
initialized_but_not_confirmed_rsm_.emplace(to_init.uuid, new_shard_id);
|
||||
initialized_but_not_confirmed_rsm_.emplace(to_init.uuid);
|
||||
|
||||
if (rsm_worker_mapping_.contains(to_init.uuid)) {
|
||||
spdlog::info(
|
||||
"ShardManager forwarding shard intialization request to worker despite a mapping already existing. This "
|
||||
"can happen due to benign race conditions.");
|
||||
// it's not a bug for the coordinator to send us UUIDs that we have
|
||||
// already created, because there may have been lag that caused
|
||||
// the coordinator not to hear back from us.
|
||||
return;
|
||||
}
|
||||
|
||||
size_t worker_index = UuidToWorkerIndex(to_init.uuid);
|
||||
@@ -315,54 +272,6 @@ class ShardManager {
|
||||
|
||||
rsm_worker_mapping_.emplace(to_init.uuid, worker_index);
|
||||
}
|
||||
|
||||
for (const auto &to_split : hr.shards_to_split) {
|
||||
for (const auto &[source, destination] : to_split.uuid_mapping) {
|
||||
if (rsm_worker_mapping_.contains(source)) {
|
||||
// Create the proper layered request providing a Raft write
|
||||
// request to the local shard rsm, under the guess that it is
|
||||
// the current leader. Most of the time this will be an incorrect
|
||||
// guess, but it will eventually succeed when the right ShardManager
|
||||
// happens to send the message to the leader shard that is local.
|
||||
// This is done to avoid blocking on an RsmClient or maintaining
|
||||
// complex async request logic. It's fine to fire-and-forget because
|
||||
// it is rare and will eventually succeed.
|
||||
msgs::WriteRequests split_request_1 =
|
||||
msgs::SplitRequest{.split_key = conversions::ConvertValueVector(to_split.split_key),
|
||||
.old_shard_version = to_split.old_shard_version,
|
||||
.new_lhs_shard_version = to_split.new_lhs_shard_version,
|
||||
.new_rhs_shard_version = to_split.new_rhs_shard_version,
|
||||
.uuid_mapping = to_split.uuid_mapping};
|
||||
|
||||
WriteRequest<msgs::WriteRequests> split_request_2;
|
||||
split_request_2.operation = split_request_1;
|
||||
ShardMessages split_request_3 = split_request_2;
|
||||
|
||||
const Address our_address = io_.GetAddress();
|
||||
Address shard_address = our_address;
|
||||
shard_address.unique_id = source;
|
||||
|
||||
shard_worker::RouteMessage shard_worker_message = {
|
||||
.message = split_request_3,
|
||||
.request_id = 0,
|
||||
.to = shard_address,
|
||||
.from = our_address,
|
||||
};
|
||||
|
||||
SendToWorkerByUuid(source, shard_worker_message);
|
||||
|
||||
if (serialize_shard_splits_for_determinisim_) {
|
||||
// This is only serialized during simulation for determinism
|
||||
// purposes, and has been tested for correctness without the
|
||||
// imposed determinism.
|
||||
size_t worker_index = UuidToWorkerIndex(source);
|
||||
workers_[worker_index].BlockOnQuiescence();
|
||||
}
|
||||
} else {
|
||||
MG_ASSERT(false, "bad split source: {}", source);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <optional>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
#include "common/errors.hpp"
|
||||
#include "parser/opencypher/parser.hpp"
|
||||
@@ -60,8 +61,6 @@ using conversions::FromMap;
|
||||
using conversions::FromPropertyValueToValue;
|
||||
using conversions::ToMsgsVertexId;
|
||||
using conversions::ToPropertyValue;
|
||||
using io::messages::ShardManagerMessages;
|
||||
using msgs::SplitResponse;
|
||||
|
||||
auto CreateErrorResponse(const ShardError &shard_error, const auto transaction_id, const std::string_view action) {
|
||||
msgs::ShardError message_shard_error{shard_error.code, shard_error.message};
|
||||
@@ -71,16 +70,6 @@ auto CreateErrorResponse(const ShardError &shard_error, const auto transaction_i
|
||||
}
|
||||
|
||||
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) {
|
||||
if (req.shard_map_version < shard_->Version()) {
|
||||
spdlog::debug("ShardRsm Rejecting client request with stale ShardMap version, so that they retry");
|
||||
return msgs::CreateVerticesResponse{
|
||||
.error = msgs::ShardError{
|
||||
.code = common::ErrorCode::STALE_SHARD_MAP,
|
||||
.message = "Shard has a higher version than the requestor's ShardMap due to a Shard split or merge, so the "
|
||||
"requestor must refresh their ShardMap and try again",
|
||||
}};
|
||||
}
|
||||
|
||||
auto acc = shard_->Access(req.transaction_id);
|
||||
|
||||
std::optional<msgs::ShardError> shard_error;
|
||||
@@ -101,12 +90,10 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateVerticesRequest &&req) {
|
||||
PrimaryKey transformed_pk;
|
||||
std::transform(new_vertex.primary_key.begin(), new_vertex.primary_key.end(), std::back_inserter(transformed_pk),
|
||||
[](msgs::Value &val) { return ToPropertyValue(std::move(val)); });
|
||||
auto result_schema = acc.CreateVertexAndValidate(converted_label_ids, transformed_pk, converted_property_map);
|
||||
|
||||
auto error_opt = acc.CreateVertexAndValidate(new_vertex.idempotency_token, converted_label_ids, transformed_pk,
|
||||
converted_property_map);
|
||||
|
||||
if (error_opt) {
|
||||
shard_error.emplace(CreateErrorResponse(error_opt.value(), req.transaction_id, "creating vertices"));
|
||||
if (result_schema.HasError()) {
|
||||
shard_error.emplace(CreateErrorResponse(result_schema.GetError(), req.transaction_id, "creating vertices"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -212,8 +199,7 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::CreateExpandRequest &&req) {
|
||||
break;
|
||||
}
|
||||
|
||||
auto edge_acc = acc.CreateEdge(from_vertex_id, to_vertex_id, new_expand.type.id, Gid::FromUint(new_expand.id.gid),
|
||||
new_expand.idempotency_token);
|
||||
auto edge_acc = acc.CreateEdge(from_vertex_id, to_vertex_id, new_expand.type.id, Gid::FromUint(new_expand.id.gid));
|
||||
if (edge_acc.HasValue()) {
|
||||
auto edge = edge_acc.GetValue();
|
||||
if (!new_expand.properties.empty()) {
|
||||
@@ -329,32 +315,7 @@ msgs::WriteResponses ShardRsm::ApplyWrite(msgs::UpdateEdgesRequest &&req) {
|
||||
return msgs::UpdateEdgesResponse{std::move(shard_error)};
|
||||
}
|
||||
|
||||
msgs::WriteResponses ShardRsm::ApplyWrite(msgs::SplitRequest &&req) {
|
||||
auto converted_primary_key = conversions::ConvertPropertyVector(req.split_key);
|
||||
auto new_shard_split_data = shard_->PerformSplit(converted_primary_key, req.old_shard_version,
|
||||
req.new_lhs_shard_version, req.new_rhs_shard_version);
|
||||
|
||||
if (new_shard_split_data) {
|
||||
spdlog::debug("ShardRsm performed split from version {} to versions {} and {}", req.old_shard_version,
|
||||
req.new_lhs_shard_version, req.new_rhs_shard_version);
|
||||
msgs::InitializeSplitShard msg{.shard = Shard::FromSplitData(std::move(*new_shard_split_data)),
|
||||
.uuid_mapping = req.uuid_mapping};
|
||||
shard_manager_sender_.Send(std::move(msg));
|
||||
}
|
||||
|
||||
return SplitResponse{};
|
||||
}
|
||||
|
||||
msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) {
|
||||
if (req.shard_map_version < shard_->Version()) {
|
||||
spdlog::debug("ShardRsm Rejecting client request with stale ShardMap version, so that they retry");
|
||||
return msgs::ScanVerticesResponse{
|
||||
.error = msgs::ShardError{
|
||||
.code = common::ErrorCode::STALE_SHARD_MAP,
|
||||
.message = "Shard has a higher version than the requestor's ShardMap due to a Shard split or merge, so the "
|
||||
"requestor must refresh their ShardMap and try again",
|
||||
}};
|
||||
}
|
||||
auto acc = shard_->Access(req.transaction_id);
|
||||
std::optional<msgs::ShardError> shard_error;
|
||||
|
||||
@@ -448,9 +409,6 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ScanVerticesRequest &&req) {
|
||||
resp.results = std::move(results);
|
||||
}
|
||||
|
||||
spdlog::trace("Shard version {} returning {} results for ScanVertices", shard_->Version().logical_id,
|
||||
resp.results.size());
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,20 +12,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <variant>
|
||||
|
||||
#include <openssl/ec.h>
|
||||
#include "io/messages.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
class ShardRsm {
|
||||
std::shared_ptr<Shard> shard_;
|
||||
utils::Sender<msgs::InitializeSplitShard> shard_manager_sender_;
|
||||
std::unique_ptr<Shard> shard_;
|
||||
|
||||
msgs::ReadResponses HandleRead(msgs::ExpandOneRequest &&req);
|
||||
msgs::ReadResponses HandleRead(msgs::GetPropertiesRequest &&req);
|
||||
@@ -39,26 +36,10 @@ class ShardRsm {
|
||||
msgs::WriteResponses ApplyWrite(msgs::DeleteEdgesRequest &&req);
|
||||
msgs::WriteResponses ApplyWrite(msgs::UpdateEdgesRequest &&req);
|
||||
|
||||
msgs::WriteResponses ApplyWrite(msgs::SplitRequest &&req);
|
||||
|
||||
msgs::WriteResponses ApplyWrite(msgs::CommitRequest &&req);
|
||||
|
||||
public:
|
||||
ShardRsm(std::shared_ptr<Shard> &&shard, utils::Sender<msgs::InitializeSplitShard> shard_manager_sender)
|
||||
: shard_(std::move(shard)), shard_manager_sender_{std::move(shard_manager_sender)} {};
|
||||
|
||||
std::optional<msgs::SuggestedSplitInfo> ShouldSplit() const noexcept {
|
||||
auto split_info = shard_->ShouldSplit();
|
||||
if (split_info) {
|
||||
return msgs::SuggestedSplitInfo{
|
||||
.shard_to_split_uuid = split_info->shard_to_split_uuid,
|
||||
.label_id = split_info->label_id,
|
||||
.splitting_shard_low_key = conversions::ConvertValueVector(split_info->splitting_shard_low_key),
|
||||
.split_key = conversions::ConvertValueVector(split_info->split_key),
|
||||
.shard_version = split_info->shard_version};
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
explicit ShardRsm(std::unique_ptr<Shard> &&shard) : shard_(std::move(shard)){};
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
msgs::ReadResponses Read(msgs::ReadRequests requests) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -17,11 +17,9 @@
|
||||
#include <queue>
|
||||
#include <variant>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/future.hpp"
|
||||
@@ -48,7 +46,6 @@ using io::RequestId;
|
||||
using io::Time;
|
||||
using io::messages::ShardMessages;
|
||||
using io::rsm::Raft;
|
||||
using msgs::InitializeSplitShardByUUID;
|
||||
using msgs::ReadRequests;
|
||||
using msgs::ReadResponses;
|
||||
using msgs::WriteRequests;
|
||||
@@ -69,7 +66,7 @@ struct RouteMessage {
|
||||
Address from;
|
||||
};
|
||||
|
||||
using Message = std::variant<ShutDown, Cron, ShardToInitialize, RouteMessage, InitializeSplitShardByUUID>;
|
||||
using Message = std::variant<ShutDown, Cron, ShardToInitialize, RouteMessage>;
|
||||
|
||||
struct QueueInner {
|
||||
std::mutex mu{};
|
||||
@@ -158,54 +155,9 @@ class ShardWorker {
|
||||
}
|
||||
|
||||
bool Process(RouteMessage &&route_message) {
|
||||
if (rsm_map_.contains(route_message.to.unique_id)) {
|
||||
spdlog::info("ShardWorker routing message to rsm {}", route_message.to.unique_id);
|
||||
auto &rsm = rsm_map_.at(route_message.to.unique_id);
|
||||
auto &rsm = rsm_map_.at(route_message.to.unique_id);
|
||||
|
||||
rsm.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
|
||||
} else {
|
||||
auto type_info = std::visit([&](const auto &msg) { return io::TypeInfoFor(msg); }, route_message.message);
|
||||
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
spdlog::debug(
|
||||
"ShardWorker received {} message for rsm {} which does not exist on our system (possibly due to a split not "
|
||||
"having been applied locally yet)",
|
||||
demangled_name, route_message.to.unique_id);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Process(InitializeSplitShardByUUID &&initialize_split_shard) {
|
||||
if (rsm_map_.contains(initialize_split_shard.shard_uuid)) {
|
||||
// it's not a bug for the coordinator to send us UUIDs that we have
|
||||
// already created, because there may have been lag that caused
|
||||
// the coordinator not to hear back from us.
|
||||
return true;
|
||||
}
|
||||
|
||||
auto rsm_io = io_.ForkLocal(initialize_split_shard.shard_uuid);
|
||||
|
||||
// TODO(tyler) get peers from Coordinator in HeartbeatResponse
|
||||
std::vector<Address> rsm_peers = {};
|
||||
|
||||
Address local_shard_manager_address = io_.GetAddress().ForkLocalShardManager();
|
||||
utils::Sender<msgs::InitializeSplitShard> local_shard_manager_sender =
|
||||
io_.template GetSender<msgs::InitializeSplitShard>(local_shard_manager_address);
|
||||
|
||||
// TODO(tyler) pass this local_shard_manager_sender to the Shard so that it can communicate back to the local
|
||||
// manager from split code
|
||||
ShardRsm rsm_state{std::move(initialize_split_shard.shard), std::move(local_shard_manager_sender)};
|
||||
|
||||
ShardRaft<IoImpl> rsm{std::move(rsm_io), rsm_peers, std::move(rsm_state)};
|
||||
|
||||
spdlog::info("ShardWorker created a new shard with UUID {}", initialize_split_shard.shard_uuid);
|
||||
|
||||
// perform an initial Cron call for the new RSM
|
||||
Time next_cron = rsm.Cron();
|
||||
cron_schedule_.push(std::make_pair(next_cron, initialize_split_shard.shard_uuid));
|
||||
|
||||
rsm_map_.emplace(initialize_split_shard.shard_uuid, std::move(rsm));
|
||||
rsm.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
|
||||
|
||||
return true;
|
||||
}
|
||||
@@ -221,13 +173,6 @@ class ShardWorker {
|
||||
auto &rsm = rsm_map_.at(uuid);
|
||||
Time next_for_uuid = rsm.Cron();
|
||||
|
||||
// Check if shard should split
|
||||
if (auto split_info = rsm.ShouldSplit(); split_info) {
|
||||
split_info->shard_to_split_uuid = uuid;
|
||||
const auto shard_manager_addr = io_.GetAddress().ForkLocalShardManager();
|
||||
io_.Send(shard_manager_addr, 0, *split_info);
|
||||
}
|
||||
|
||||
cron_schedule_.pop();
|
||||
cron_schedule_.push(std::make_pair(next_for_uuid, uuid));
|
||||
} else {
|
||||
@@ -243,7 +188,6 @@ class ShardWorker {
|
||||
// it's not a bug for the coordinator to send us UUIDs that we have
|
||||
// already created, because there may have been lag that caused
|
||||
// the coordinator not to hear back from us.
|
||||
spdlog::debug("ShardWorker told to initialize already-existing shard with UUID {} - skipping", to_init.uuid);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -252,22 +196,14 @@ class ShardWorker {
|
||||
// TODO(tyler) get peers from Coordinator in HeartbeatResponse
|
||||
std::vector<Address> rsm_peers = {};
|
||||
|
||||
Address local_shard_manager_address = io_.GetAddress().ForkLocalShardManager();
|
||||
utils::Sender<msgs::InitializeSplitShard> local_shard_manager_sender =
|
||||
io_.template GetSender<msgs::InitializeSplitShard>(local_shard_manager_address);
|
||||
std::unique_ptr<Shard> shard = std::make_unique<Shard>(to_init.label_id, to_init.min_key, to_init.max_key,
|
||||
to_init.schema, to_init.config, to_init.id_to_names);
|
||||
|
||||
// TODO(tyler) pass this local_shard_manager_sender to the Shard so that it can communicate back to the local
|
||||
// manager from split code
|
||||
|
||||
std::unique_ptr<Shard> shard =
|
||||
std::make_unique<Shard>(to_init.label_id, to_init.min_key, to_init.max_key, to_init.schema,
|
||||
to_init.new_shard_version, to_init.config, to_init.id_to_names);
|
||||
|
||||
ShardRsm rsm_state{std::move(shard), std::move(local_shard_manager_sender)};
|
||||
ShardRsm rsm_state{std::move(shard)};
|
||||
|
||||
ShardRaft<IoImpl> rsm{std::move(rsm_io), rsm_peers, std::move(rsm_state)};
|
||||
|
||||
spdlog::debug("ShardWorker created a new shard with UUID {}", to_init.uuid);
|
||||
spdlog::info("SM created a new shard with UUID {}", to_init.uuid);
|
||||
|
||||
// perform an initial Cron call for the new RSM
|
||||
Time next_cron = rsm.Cron();
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "storage/v3/splitter.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/delta.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/indices.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
Splitter::Splitter(const LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
|
||||
const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper)
|
||||
: primary_label_(primary_label),
|
||||
vertices_(vertices),
|
||||
edges_(edges),
|
||||
start_logical_id_to_transaction_(start_logical_id_to_transaction),
|
||||
indices_(indices),
|
||||
config_(config),
|
||||
schema_(schema),
|
||||
name_id_mapper_(name_id_mapper) {}
|
||||
|
||||
SplitData Splitter::SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
|
||||
const Hlc shard_version) {
|
||||
SplitData data{.primary_label = primary_label_,
|
||||
.min_primary_key = split_key,
|
||||
.max_primary_key = max_primary_key,
|
||||
.schema = schema_,
|
||||
.config = config_,
|
||||
.id_to_name = name_id_mapper_.GetIdToNameMap(),
|
||||
.shard_version = shard_version};
|
||||
|
||||
std::set<uint64_t> collected_transactions_;
|
||||
data.vertices = CollectVertices(data, collected_transactions_, split_key);
|
||||
data.edges = CollectEdges(collected_transactions_, data.vertices, split_key);
|
||||
data.transactions = CollectTransactions(collected_transactions_, data.vertices, *data.edges, split_key);
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
void Splitter::ScanDeltas(std::set<uint64_t> &collected_transactions_, Delta *delta) {
|
||||
while (delta != nullptr) {
|
||||
collected_transactions_.insert(delta->commit_info->start_or_commit_timestamp.logical_id);
|
||||
delta = delta->next;
|
||||
}
|
||||
}
|
||||
|
||||
VertexContainer Splitter::CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_,
|
||||
const PrimaryKey &split_key) {
|
||||
data.label_indices = indices_.label_index.SplitIndexEntries(split_key);
|
||||
data.label_property_indices = indices_.label_property_index.SplitIndexEntries(split_key);
|
||||
|
||||
VertexContainer splitted_data;
|
||||
auto split_key_it = vertices_.find(split_key);
|
||||
while (split_key_it != vertices_.end()) {
|
||||
// Go through deltas and pick up transactions start_id/commit_id
|
||||
ScanDeltas(collected_transactions_, split_key_it->second.delta);
|
||||
|
||||
auto next_it = std::next(split_key_it);
|
||||
|
||||
const auto &[splitted_vertex_it, inserted, node] = splitted_data.insert(vertices_.extract(split_key_it->first));
|
||||
MG_ASSERT(inserted, "Failed to extract vertex!");
|
||||
|
||||
split_key_it = next_it;
|
||||
}
|
||||
return splitted_data;
|
||||
}
|
||||
|
||||
std::optional<EdgeContainer> Splitter::CollectEdges(std::set<uint64_t> &collected_transactions_,
|
||||
const VertexContainer &split_vertices,
|
||||
const PrimaryKey &split_key) {
|
||||
if (!config_.items.properties_on_edges) {
|
||||
return std::nullopt;
|
||||
}
|
||||
EdgeContainer splitted_edges;
|
||||
const auto split_vertex_edges = [&](const auto &edges_ref) {
|
||||
// This is safe since if properties_on_edges is true, the this must be a ptr
|
||||
for (const auto &edge_ref : edges_ref) {
|
||||
auto *edge = std::get<2>(edge_ref).ptr;
|
||||
const auto &other_vtx = std::get<1>(edge_ref);
|
||||
ScanDeltas(collected_transactions_, edge->delta);
|
||||
// Check if src and dest edge are both on splitted shard so we know if we
|
||||
// should remove orphan edge, or make a clone
|
||||
if (other_vtx.primary_key >= split_key) {
|
||||
// Remove edge from shard
|
||||
splitted_edges.insert(edges_.extract(edge->gid));
|
||||
} else {
|
||||
splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto &vertex : split_vertices) {
|
||||
split_vertex_edges(vertex.second.in_edges);
|
||||
split_vertex_edges(vertex.second.out_edges);
|
||||
}
|
||||
return splitted_edges;
|
||||
}
|
||||
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> Splitter::CollectTransactions(
|
||||
const std::set<uint64_t> &collected_transactions_, VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
|
||||
const PrimaryKey &split_key) {
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
|
||||
|
||||
for (const auto &[commit_start, transaction] : start_logical_id_to_transaction_) {
|
||||
// We need all transaction whose deltas need to be resolved for any of the
|
||||
// entities
|
||||
if (collected_transactions_.contains(transaction->commit_info->start_or_commit_timestamp.logical_id)) {
|
||||
transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()});
|
||||
}
|
||||
}
|
||||
|
||||
// It is necessary to clone all the transactions first so we have new addresses
|
||||
// for deltas, before doing alignment of deltas and prev_ptr
|
||||
AdjustClonedTransactions(transactions, cloned_vertices, cloned_edges, split_key);
|
||||
return transactions;
|
||||
}
|
||||
|
||||
void PruneDeltas(Transaction &cloned_transaction, std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
const PrimaryKey &split_key) {
|
||||
// Remove delta chains that don't point to objects on splitted shard
|
||||
auto cloned_delta_it = cloned_transaction.deltas.begin();
|
||||
|
||||
while (cloned_delta_it != cloned_transaction.deltas.end()) {
|
||||
const auto prev = cloned_delta_it->prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::DELTA:
|
||||
case PreviousPtr::Type::NULLPTR:
|
||||
++cloned_delta_it;
|
||||
break;
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
if (prev.vertex->first < split_key) {
|
||||
// We can remove this delta chain
|
||||
auto *current_next_delta = cloned_delta_it->next;
|
||||
cloned_delta_it = cloned_transaction.deltas.erase(cloned_delta_it);
|
||||
|
||||
while (current_next_delta != nullptr) {
|
||||
auto *next_delta = current_next_delta->next;
|
||||
// Find next delta transaction delta list
|
||||
auto current_transaction_it = std::ranges::find_if(
|
||||
cloned_transactions,
|
||||
[&start_or_commit_timestamp =
|
||||
current_next_delta->commit_info->start_or_commit_timestamp](const auto &transaction) {
|
||||
return transaction.second->start_timestamp == start_or_commit_timestamp ||
|
||||
transaction.second->commit_info->start_or_commit_timestamp == start_or_commit_timestamp;
|
||||
});
|
||||
MG_ASSERT(current_transaction_it != cloned_transactions.end(), "Error when pruning deltas!");
|
||||
// Remove it
|
||||
current_transaction_it->second->deltas.remove_if(
|
||||
[¤t_next_delta = *current_next_delta](const auto &delta) { return delta == current_next_delta; });
|
||||
|
||||
current_next_delta = next_delta;
|
||||
}
|
||||
} else {
|
||||
++cloned_delta_it;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::EDGE:
|
||||
++cloned_delta_it;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Splitter::AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
|
||||
const PrimaryKey &split_key) {
|
||||
for (auto &[commit_start, cloned_transaction] : cloned_transactions) {
|
||||
AdjustClonedTransaction(*cloned_transaction, *start_logical_id_to_transaction_[commit_start], cloned_transactions,
|
||||
cloned_vertices, cloned_edges, split_key);
|
||||
}
|
||||
// Prune deltas whose delta chain points to vertex/edge that should not belong on that shard
|
||||
// Prune must be after ajdust, since next, and prev are not set and we cannot follow the chain
|
||||
for (auto &[commit_start, cloned_transaction] : cloned_transactions) {
|
||||
PruneDeltas(*cloned_transaction, cloned_transactions, split_key);
|
||||
}
|
||||
}
|
||||
|
||||
inline bool IsDeltaHeadOfChain(const PreviousPtr::Type &delta_type) {
|
||||
return delta_type == PreviousPtr::Type::VERTEX || delta_type == PreviousPtr::Type::EDGE;
|
||||
}
|
||||
|
||||
bool DoesPrevPtrPointsToSplittedData(const PreviousPtr::Pointer &prev_ptr, const PrimaryKey &split_key) {
|
||||
return prev_ptr.type == PreviousPtr::Type::VERTEX && prev_ptr.vertex->first < split_key;
|
||||
}
|
||||
|
||||
void Splitter::AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
|
||||
const PrimaryKey & /*split_key*/) {
|
||||
auto delta_it = transaction.deltas.begin();
|
||||
auto cloned_delta_it = cloned_transaction.deltas.begin();
|
||||
|
||||
while (delta_it != transaction.deltas.end()) {
|
||||
// We can safely ignore deltas which are not head of delta chain
|
||||
// Dont' adjust delta chain that points to irrelevant data vertices/edges
|
||||
if (const auto delta_prev = delta_it->prev.Get(); !IsDeltaHeadOfChain(delta_prev.type)) {
|
||||
++delta_it;
|
||||
++cloned_delta_it;
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto *delta = &*delta_it;
|
||||
auto *cloned_delta = &*cloned_delta_it;
|
||||
Delta *cloned_delta_prev_ptr = cloned_delta;
|
||||
while (delta->next != nullptr) {
|
||||
AdjustEdgeRef(*cloned_delta, cloned_edges);
|
||||
|
||||
// Align next ptr
|
||||
AdjustDeltaNext(*delta, *cloned_delta, cloned_transactions);
|
||||
|
||||
// Align prev ptr
|
||||
if (cloned_delta_prev_ptr != nullptr) {
|
||||
AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_vertices, cloned_edges);
|
||||
}
|
||||
|
||||
// TODO Next delta might not belong to the cloned transaction and thats
|
||||
// why we skip this delta of the delta chain
|
||||
if (cloned_delta->next != nullptr) {
|
||||
cloned_delta = cloned_delta->next;
|
||||
cloned_delta_prev_ptr = cloned_delta;
|
||||
} else {
|
||||
cloned_delta_prev_ptr = nullptr;
|
||||
}
|
||||
delta = delta->next;
|
||||
}
|
||||
// Align prev ptr
|
||||
if (cloned_delta_prev_ptr != nullptr) {
|
||||
AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_vertices, cloned_edges);
|
||||
}
|
||||
|
||||
++delta_it;
|
||||
++cloned_delta_it;
|
||||
}
|
||||
MG_ASSERT(delta_it == transaction.deltas.end() && cloned_delta_it == cloned_transaction.deltas.end(),
|
||||
"Both iterators must be exhausted!");
|
||||
}
|
||||
|
||||
void Splitter::AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const {
|
||||
switch (cloned_delta.action) {
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
// Find edge
|
||||
if (config_.items.properties_on_edges) {
|
||||
// Only case when not finding is when the edge is not on splitted shard
|
||||
// TODO Do this after prune an move condition into assert
|
||||
if (const auto cloned_edge_it =
|
||||
std::ranges::find_if(cloned_edges, [edge_ptr = cloned_delta.vertex_edge.edge.ptr](
|
||||
const auto &elem) { return elem.second.gid == edge_ptr->gid; });
|
||||
cloned_edge_it != cloned_edges.end()) {
|
||||
cloned_delta.vertex_edge.edge = EdgeRef{&cloned_edge_it->second};
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
// noop
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Splitter::AdjustDeltaNext(const Delta &original, Delta &cloned,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions) {
|
||||
// Get cloned_delta->next transaction, using delta->next original transaction
|
||||
auto cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&original](const auto &elem) {
|
||||
return elem.second->start_timestamp == original.next->commit_info->start_or_commit_timestamp ||
|
||||
elem.second->commit_info->start_or_commit_timestamp == original.next->commit_info->start_or_commit_timestamp;
|
||||
});
|
||||
// TODO(jbajic) What if next in delta chain does not belong to cloned transaction?
|
||||
// MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
|
||||
if (cloned_transaction_it == cloned_transactions.end()) return;
|
||||
// Find cloned delta in delta list of cloned transaction
|
||||
auto found_cloned_delta_it = std::ranges::find_if(
|
||||
cloned_transaction_it->second->deltas, [&original](const auto &elem) { return elem.id == original.next->id; });
|
||||
MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(), "Delta with given uuid must exist!");
|
||||
cloned.next = &*found_cloned_delta_it;
|
||||
}
|
||||
|
||||
void Splitter::AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer & /*cloned_vertices*/, EdgeContainer &cloned_edges) {
|
||||
auto ptr = original.prev.Get();
|
||||
switch (ptr.type) {
|
||||
case PreviousPtr::Type::NULLPTR: {
|
||||
// noop
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::DELTA: {
|
||||
// Same as for deltas except don't align next but prev
|
||||
auto cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&ptr](const auto &elem) {
|
||||
return elem.second->start_timestamp == ptr.delta->commit_info->start_or_commit_timestamp ||
|
||||
elem.second->commit_info->start_or_commit_timestamp == ptr.delta->commit_info->start_or_commit_timestamp;
|
||||
});
|
||||
MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
|
||||
// Find cloned delta in delta list of cloned transaction
|
||||
auto found_cloned_delta_it =
|
||||
std::ranges::find_if(cloned_transaction_it->second->deltas,
|
||||
[delta = ptr.delta](const auto &elem) { return elem.id == delta->id; });
|
||||
MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(),
|
||||
"Delta with given id must exist!");
|
||||
|
||||
cloned.prev.Set(&*found_cloned_delta_it);
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
// The vertex was extracted and it is safe to reuse address
|
||||
cloned.prev.Set(ptr.vertex);
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::EDGE: {
|
||||
// We can never be here if we have properties on edge disabled
|
||||
auto *cloned_edge = &*cloned_edges.find(ptr.edge->gid);
|
||||
cloned.prev.Set(&cloned_edge->second);
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
@@ -1,108 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/delta.hpp"
|
||||
#include "storage/v3/edge.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/indices.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
// If edge properties-on-edges is false then we don't need to send edges but
|
||||
// only vertices, since they will contain those edges
|
||||
struct SplitData {
|
||||
LabelId primary_label;
|
||||
PrimaryKey min_primary_key;
|
||||
std::optional<PrimaryKey> max_primary_key;
|
||||
std::vector<SchemaProperty> schema;
|
||||
Config config;
|
||||
std::unordered_map<uint64_t, std::string> id_to_name;
|
||||
coordinator::Hlc shard_version;
|
||||
|
||||
VertexContainer vertices;
|
||||
std::optional<EdgeContainer> edges;
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
|
||||
std::map<LabelId, LabelIndex::IndexContainer> label_indices;
|
||||
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndex::IndexContainer> label_property_indices;
|
||||
};
|
||||
|
||||
class Splitter final {
|
||||
public:
|
||||
Splitter(LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
|
||||
const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper_);
|
||||
|
||||
Splitter(const Splitter &) = delete;
|
||||
Splitter(Splitter &&) noexcept = delete;
|
||||
Splitter &operator=(const Splitter &) = delete;
|
||||
Splitter operator=(Splitter &&) noexcept = delete;
|
||||
~Splitter() = default;
|
||||
|
||||
SplitData SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
|
||||
coordinator::Hlc shard_version);
|
||||
|
||||
private:
|
||||
VertexContainer CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_start_id,
|
||||
const PrimaryKey &split_key);
|
||||
|
||||
std::optional<EdgeContainer> CollectEdges(std::set<uint64_t> &collected_transactions_start_id,
|
||||
const VertexContainer &split_vertices, const PrimaryKey &split_key);
|
||||
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> CollectTransactions(
|
||||
const std::set<uint64_t> &collected_transactions_start_id, VertexContainer &cloned_vertices,
|
||||
EdgeContainer &cloned_edges, const PrimaryKey &split_key);
|
||||
|
||||
static void ScanDeltas(std::set<uint64_t> &collected_transactions_start_id, Delta *delta);
|
||||
|
||||
void AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
|
||||
const PrimaryKey &split_key);
|
||||
|
||||
void AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer &cloned_vertices, EdgeContainer &cloned_edges,
|
||||
const PrimaryKey &split_key);
|
||||
|
||||
void AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const;
|
||||
|
||||
static void AdjustDeltaNext(const Delta &original, Delta &cloned,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions);
|
||||
|
||||
static void AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
|
||||
VertexContainer &cloned_vertices, EdgeContainer &cloned_edges);
|
||||
|
||||
const LabelId primary_label_;
|
||||
VertexContainer &vertices_;
|
||||
EdgeContainer &edges_;
|
||||
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction_;
|
||||
Indices &indices_;
|
||||
const Config &config_;
|
||||
const std::vector<SchemaProperty> schema_;
|
||||
const NameIdMapper &name_id_mapper_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -31,16 +31,6 @@ struct CommitInfo {
|
||||
};
|
||||
|
||||
struct Transaction {
|
||||
Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, std::list<Delta> deltas,
|
||||
uint64_t command_id, bool must_abort, bool is_aborted, IsolationLevel isolation_level)
|
||||
: start_timestamp{start_timestamp},
|
||||
commit_info{std::make_unique<CommitInfo>(new_commit_info)},
|
||||
command_id(command_id),
|
||||
deltas(std::move(deltas)),
|
||||
must_abort(must_abort),
|
||||
is_aborted(is_aborted),
|
||||
isolation_level(isolation_level){};
|
||||
|
||||
Transaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level)
|
||||
: start_timestamp(start_timestamp),
|
||||
commit_info(std::make_unique<CommitInfo>(CommitInfo{false, {start_timestamp}})),
|
||||
@@ -64,55 +54,6 @@ struct Transaction {
|
||||
|
||||
~Transaction() {}
|
||||
|
||||
std::list<Delta> CopyDeltas(CommitInfo *commit_info) const {
|
||||
std::list<Delta> copied_deltas;
|
||||
for (const auto &delta : deltas) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
copied_deltas.emplace_back(Delta::DeleteObjectTag{}, commit_info, delta.id, command_id,
|
||||
delta.idempotency_token);
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
copied_deltas.emplace_back(Delta::RecreateObjectTag{}, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::ADD_LABEL:
|
||||
copied_deltas.emplace_back(Delta::AddLabelTag{}, delta.label, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
copied_deltas.emplace_back(Delta::RemoveLabelTag{}, delta.label, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
copied_deltas.emplace_back(Delta::AddInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
|
||||
delta.vertex_edge.edge, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
copied_deltas.emplace_back(Delta::AddOutEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
|
||||
delta.vertex_edge.edge, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
copied_deltas.emplace_back(Delta::RemoveInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
|
||||
delta.vertex_edge.edge, commit_info, delta.id, command_id);
|
||||
break;
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
copied_deltas.emplace_back(Delta::RemoveOutEdgeTag{}, delta.vertex_edge.edge_type,
|
||||
delta.vertex_edge.vertex_id, delta.vertex_edge.edge, commit_info, delta.id,
|
||||
command_id);
|
||||
break;
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
copied_deltas.emplace_back(Delta::SetPropertyTag{}, delta.property.key, delta.property.value, commit_info,
|
||||
delta.id, command_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return copied_deltas;
|
||||
}
|
||||
|
||||
// This does not solve the whole problem of copying deltas
|
||||
std::unique_ptr<Transaction> Clone() const {
|
||||
return std::make_unique<Transaction>(start_timestamp, *commit_info, CopyDeltas(commit_info.get()), command_id,
|
||||
must_abort, is_aborted, isolation_level);
|
||||
}
|
||||
|
||||
coordinator::Hlc start_timestamp;
|
||||
std::unique_ptr<CommitInfo> commit_info;
|
||||
uint64_t command_id;
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <concepts>
|
||||
#include <functional>
|
||||
#include <type_traits>
|
||||
|
||||
namespace memgraph::utils {
|
||||
|
||||
// TODO(tyler) ensure that Message continues to represent
|
||||
// reasonable constraints around message types over time,
|
||||
// as we adapt things to use Thrift-generated message types.
|
||||
template <typename T>
|
||||
concept Message = std::movable<T> && std::copyable<T>;
|
||||
|
||||
/// This is a concrete type that allows one message type to be
|
||||
/// sent to a single address. Initially intended to be used by the
|
||||
/// Shard to send messages to the local ShardManager.
|
||||
template <Message M>
|
||||
class Sender {
|
||||
public:
|
||||
Sender() = default;
|
||||
|
||||
explicit Sender(std::function<void(M)> sender) : sender_(sender) {}
|
||||
|
||||
void Send(M &&message) { sender_(std::forward<M>(message)); }
|
||||
|
||||
private:
|
||||
std::function<void(M)> sender_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::utils
|
||||
@@ -79,6 +79,3 @@ target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage
|
||||
|
||||
add_benchmark(data_structures_remove.cpp)
|
||||
target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3)
|
||||
|
||||
add_benchmark(storage_v3_split.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v3_split mg-storage-v3 mg-query-v2)
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include <benchmark/benchmark.h>
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
|
||||
namespace memgraph::benchmark {
|
||||
|
||||
class ShardSplitBenchmark : public ::benchmark::Fixture {
|
||||
protected:
|
||||
using PrimaryKey = storage::v3::PrimaryKey;
|
||||
using PropertyId = storage::v3::PropertyId;
|
||||
using PropertyValue = storage::v3::PropertyValue;
|
||||
using LabelId = storage::v3::LabelId;
|
||||
using EdgeTypeId = storage::v3::EdgeTypeId;
|
||||
using Shard = storage::v3::Shard;
|
||||
using VertexId = storage::v3::VertexId;
|
||||
using Gid = storage::v3::Gid;
|
||||
|
||||
void SetUp(const ::benchmark::State &state) override {
|
||||
storage.emplace(primary_label, min_pk, std::nullopt, schema_property_vector, last_hlc);
|
||||
storage->StoreMapping(
|
||||
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
|
||||
}
|
||||
|
||||
void TearDown(const ::benchmark::State &) override { storage = std::nullopt; }
|
||||
|
||||
const PropertyId primary_property{PropertyId::FromUint(2)};
|
||||
const PropertyId secondary_property{PropertyId::FromUint(5)};
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
|
||||
const LabelId primary_label{LabelId::FromUint(1)};
|
||||
const LabelId secondary_label{LabelId::FromUint(4)};
|
||||
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
|
||||
std::optional<Shard> storage;
|
||||
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
|
||||
coordinator::Hlc GetNextHlc() {
|
||||
++last_hlc.logical_id;
|
||||
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
|
||||
return last_hlc;
|
||||
}
|
||||
};
|
||||
|
||||
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplit)(::benchmark::State &state) {
|
||||
const auto number_of_vertices{state.range(0)};
|
||||
std::random_device r;
|
||||
std::default_random_engine e1(r());
|
||||
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
|
||||
|
||||
for (int64_t i{0}; i < number_of_vertices; ++i) {
|
||||
auto acc = storage->Access(GetNextHlc());
|
||||
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
|
||||
{{secondary_property, PropertyValue(i)}})
|
||||
.HasValue(),
|
||||
"Failed creating with pk {}", i);
|
||||
if (i > 1) {
|
||||
const auto vtx1 = uniform_dist(e1) % i;
|
||||
const auto vtx2 = uniform_dist(e1) % i;
|
||||
|
||||
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
|
||||
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
|
||||
.HasValue(),
|
||||
"Failed on {} and {}", vtx1, vtx2);
|
||||
}
|
||||
acc.Commit(GetNextHlc());
|
||||
}
|
||||
for (auto _ : state) {
|
||||
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithGc)(::benchmark::State &state) {
|
||||
const auto number_of_vertices{state.range(0)};
|
||||
std::random_device r;
|
||||
std::default_random_engine e1(r());
|
||||
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
|
||||
|
||||
for (int64_t i{0}; i < number_of_vertices; ++i) {
|
||||
auto acc = storage->Access(GetNextHlc());
|
||||
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
|
||||
{{secondary_property, PropertyValue(i)}})
|
||||
.HasValue(),
|
||||
"Failed creating with pk {}", i);
|
||||
if (i > 1) {
|
||||
const auto vtx1 = uniform_dist(e1) % i;
|
||||
const auto vtx2 = uniform_dist(e1) % i;
|
||||
|
||||
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
|
||||
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
|
||||
.HasValue(),
|
||||
"Failed on {} and {}", vtx1, vtx2);
|
||||
}
|
||||
acc.Commit(GetNextHlc());
|
||||
}
|
||||
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
for (auto _ : state) {
|
||||
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
|
||||
}
|
||||
}
|
||||
|
||||
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithFewTransactions)(::benchmark::State &state) {
|
||||
const auto number_of_vertices = state.range(0);
|
||||
const auto number_of_edges = state.range(1);
|
||||
const auto number_of_transactions = state.range(2);
|
||||
std::random_device r;
|
||||
std::default_random_engine e1(r());
|
||||
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
|
||||
|
||||
const auto max_transactions_needed = std::max(number_of_vertices, number_of_edges);
|
||||
for (int64_t vertex_counter{number_of_vertices}, edge_counter{number_of_edges}, i{0};
|
||||
vertex_counter > 0 || edge_counter > 0; --vertex_counter, --edge_counter, ++i) {
|
||||
auto acc = storage->Access(GetNextHlc());
|
||||
if (vertex_counter > 0) {
|
||||
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
|
||||
{{secondary_property, PropertyValue(i)}})
|
||||
.HasValue(),
|
||||
"Failed creating with pk {}", i);
|
||||
}
|
||||
if (edge_counter > 0 && i > 1) {
|
||||
const auto vtx1 = uniform_dist(e1) % std::min(i, number_of_vertices);
|
||||
const auto vtx2 = uniform_dist(e1) % std::min(i, number_of_vertices);
|
||||
|
||||
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
|
||||
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
|
||||
.HasValue(),
|
||||
"Failed on {} and {}", vtx1, vtx2);
|
||||
}
|
||||
|
||||
acc.Commit(GetNextHlc());
|
||||
if (i == max_transactions_needed - number_of_transactions) {
|
||||
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto _ : state) {
|
||||
// Don't create shard since shard deallocation can take some time as well
|
||||
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, last_hlc, GetNextHlc());
|
||||
}
|
||||
}
|
||||
|
||||
// Range:
|
||||
// Number of vertices
|
||||
// This run is pessimistic, number of vertices corresponds with number if transactions
|
||||
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplit)
|
||||
->RangeMultiplier(10)
|
||||
->Range(100'000, 100'000)
|
||||
->Unit(::benchmark::kMillisecond);
|
||||
|
||||
// Range:
|
||||
// Number of vertices
|
||||
// This run is optimistic, in this run there are no transactions
|
||||
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithGc)
|
||||
->RangeMultiplier(10)
|
||||
->Range(100'000, 1'000'000)
|
||||
->Unit(::benchmark::kMillisecond);
|
||||
|
||||
// Args:
|
||||
// Number of vertices
|
||||
// Number of edges
|
||||
// Number of transaction
|
||||
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactions)
|
||||
->Args({100'000, 100'000, 1'000})
|
||||
->Args({100'000, 100'000, 10'000})
|
||||
->Args({1'000'000, 100'000, 1'000})
|
||||
->Args({1'000'000, 100'000, 10'000})
|
||||
->Args({100'000, 1'000'000, 1'000})
|
||||
->Args({1'000'000, 1'00'000, 10'000})
|
||||
->Unit(::benchmark::kMillisecond);
|
||||
|
||||
} // namespace memgraph::benchmark
|
||||
|
||||
BENCHMARK_MAIN();
|
||||
@@ -31,7 +31,6 @@ add_simulation_test(raft.cpp)
|
||||
add_simulation_test(trial_query_storage/query_storage_test.cpp)
|
||||
add_simulation_test(sharded_map.cpp)
|
||||
add_simulation_test(shard_rsm.cpp)
|
||||
add_simulation_test(bulk_load.cpp)
|
||||
add_simulation_test(random_workload.cpp)
|
||||
add_simulation_test(cluster_property_test.cpp)
|
||||
add_simulation_test(cluster_property_test_cypher_queries.cpp)
|
||||
add_simulation_test(request_router.cpp)
|
||||
add_simulation_test(property_test_demo.cpp)
|
||||
|
||||
@@ -20,11 +20,11 @@ namespace memgraph::tests::simulation {
|
||||
struct ClusterConfig {
|
||||
int servers;
|
||||
int replication_factor;
|
||||
int split_threshold;
|
||||
int shards;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const ClusterConfig &cluster) {
|
||||
in << "ClusterConfig { servers: " << cluster.servers << ", replication_factor: " << cluster.replication_factor
|
||||
<< " }";
|
||||
<< ", shards: " << cluster.shards << " }";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
@@ -44,7 +44,7 @@ struct Arbitrary<ClusterConfig> {
|
||||
gen::set(&ClusterConfig::servers, gen::inRange(kMinimumServers, kMaximumServers)),
|
||||
gen::set(&ClusterConfig::replication_factor,
|
||||
gen::inRange(kMinimumReplicationFactor, kMaximumReplicationFactor)),
|
||||
gen::set(&ClusterConfig::split_threshold, gen::inRange(kMinimumSplitThreshold, kMaximumSplitThreshold)));
|
||||
gen::set(&ClusterConfig::shards, gen::inRange(kMinimumShards, kMaximumShards)));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -33,13 +33,12 @@ using io::Time;
|
||||
using io::simulator::SimulatorConfig;
|
||||
using storage::v3::kMaximumCronInterval;
|
||||
|
||||
RC_GTEST_PROP(RandomClusterConfig, RandomWorkload,
|
||||
(ClusterConfig cluster_config, NonEmptyOpVec ops, uint64_t rng_seed)) {
|
||||
RC_GTEST_PROP(RandomClusterConfig, HappyPath, (ClusterConfig cluster_config, NonEmptyOpVec ops, uint64_t rng_seed)) {
|
||||
spdlog::cfg::load_env_levels();
|
||||
|
||||
SimulatorConfig sim_config{
|
||||
.drop_percent = 0,
|
||||
.perform_timeouts = true,
|
||||
.perform_timeouts = false,
|
||||
.scramble_messages = true,
|
||||
.rng_seed = rng_seed,
|
||||
.start_time = Time::min(),
|
||||
@@ -50,7 +49,7 @@ RC_GTEST_PROP(RandomClusterConfig, RandomWorkload,
|
||||
auto [sim_stats_1, latency_stats_1] = RunClusterSimulation(sim_config, cluster_config, ops.ops);
|
||||
auto [sim_stats_2, latency_stats_2] = RunClusterSimulation(sim_config, cluster_config, ops.ops);
|
||||
|
||||
if (latency_stats_1 != latency_stats_2 || sim_stats_1 != sim_stats_2) {
|
||||
if (latency_stats_1 != latency_stats_2) {
|
||||
spdlog::error("simulator stats diverged across runs");
|
||||
spdlog::error("run 1 simulator stats: {}", sim_stats_1);
|
||||
spdlog::error("run 2 simulator stats: {}", sim_stats_2);
|
||||
@@ -59,8 +58,6 @@ RC_GTEST_PROP(RandomClusterConfig, RandomWorkload,
|
||||
RC_ASSERT(latency_stats_1 == latency_stats_2);
|
||||
RC_ASSERT(sim_stats_1 == sim_stats_2);
|
||||
}
|
||||
|
||||
spdlog::trace("passed stats comparison - all good!");
|
||||
}
|
||||
|
||||
} // namespace memgraph::tests::simulation
|
||||
@@ -33,41 +33,24 @@ using io::Time;
|
||||
using io::simulator::SimulatorConfig;
|
||||
using storage::v3::kMaximumCronInterval;
|
||||
|
||||
RC_GTEST_PROP(RandomClusterConfig, BulkLoadAndSplit,
|
||||
(ClusterConfig cluster_config, uint8_t inserts, uint64_t rng_seed)) {
|
||||
RC_GTEST_PROP(RandomClusterConfig, HappyPath, (ClusterConfig cluster_config, NonEmptyOpVec ops, uint64_t rng_seed)) {
|
||||
spdlog::cfg::load_env_levels();
|
||||
|
||||
// This is a static workload that just inserts vertices and reads them back, which implicitly triggers concurrent
|
||||
// splits
|
||||
std::vector<Op> ops{};
|
||||
|
||||
for (int key = 0; key < inserts; key++) {
|
||||
Op op1 = {.inner = AssertShardsSplit{}};
|
||||
ops.emplace_back(std::move(op1));
|
||||
|
||||
Op op2 = {.inner = CreateVertex{.first = 0, .second = key}};
|
||||
ops.emplace_back(std::move(op2));
|
||||
}
|
||||
|
||||
Op op1 = {.inner = AssertShardsSplit{}};
|
||||
ops.emplace_back(std::move(op1));
|
||||
|
||||
ops.emplace_back(Op{.inner = ScanAll{}});
|
||||
|
||||
SimulatorConfig sim_config{
|
||||
.drop_percent = 0,
|
||||
.perform_timeouts = true,
|
||||
.perform_timeouts = false,
|
||||
.scramble_messages = true,
|
||||
.rng_seed = rng_seed,
|
||||
.start_time = Time::min(),
|
||||
// TODO(tyler) set abort_time to something more restrictive than Time::max()
|
||||
.abort_time = Time::max(),
|
||||
};
|
||||
|
||||
auto [sim_stats_1, latency_stats_1] = RunClusterSimulation(sim_config, cluster_config, ops);
|
||||
auto [sim_stats_2, latency_stats_2] = RunClusterSimulation(sim_config, cluster_config, ops);
|
||||
std::vector<std::string> queries = {"CREATE (n:test_label{property_1: 0, property_2: 0});", "MATCH (n) RETURN n;"};
|
||||
|
||||
if (latency_stats_1 != latency_stats_2 || sim_stats_1 != sim_stats_2) {
|
||||
auto [sim_stats_1, latency_stats_1] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries);
|
||||
auto [sim_stats_2, latency_stats_2] = RunClusterSimulationWithQueries(sim_config, cluster_config, queries);
|
||||
|
||||
if (latency_stats_1 != latency_stats_2) {
|
||||
spdlog::error("simulator stats diverged across runs");
|
||||
spdlog::error("run 1 simulator stats: {}", sim_stats_1);
|
||||
spdlog::error("run 2 simulator stats: {}", sim_stats_2);
|
||||
@@ -76,8 +59,6 @@ RC_GTEST_PROP(RandomClusterConfig, BulkLoadAndSplit,
|
||||
RC_ASSERT(latency_stats_1 == latency_stats_2);
|
||||
RC_ASSERT(sim_stats_1 == sim_stats_2);
|
||||
}
|
||||
|
||||
spdlog::trace("passed stats comparison - all good!");
|
||||
}
|
||||
|
||||
} // namespace memgraph::tests::simulation
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -130,7 +130,6 @@ class MockedShardRsm {
|
||||
msgs::DeleteEdgesResponse ApplyImpl(msgs::DeleteEdgesRequest rqst) { return {}; }
|
||||
msgs::UpdateEdgesResponse ApplyImpl(msgs::UpdateEdgesRequest rqst) { return {}; }
|
||||
msgs::CommitResponse ApplyImpl(msgs::CommitRequest rqst) { return {}; }
|
||||
msgs::CommitResponse ApplyImpl(msgs::SplitRequest rqst) { return {}; }
|
||||
|
||||
WriteResponses Apply(WriteRequests write_requests) {
|
||||
return {std::visit([this]<typename T>(T &&request) { return WriteResponses{ApplyImpl(std::forward<T>(request))}; },
|
||||
|
||||
@@ -35,20 +35,13 @@ struct CreateVertex {
|
||||
};
|
||||
|
||||
struct ScanAll {
|
||||
friend std::ostream &operator<<(std::ostream &in, const ScanAll & /* unused */) {
|
||||
friend std::ostream &operator<<(std::ostream &in, const ScanAll &get) {
|
||||
in << "ScanAll {}";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
struct AssertShardsSplit {
|
||||
friend std::ostream &operator<<(std::ostream &in, const AssertShardsSplit & /* unused */) {
|
||||
in << "AssertShardsSplit {}";
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
using OpVariant = std::variant<CreateVertex, ScanAll, AssertShardsSplit>;
|
||||
using OpVariant = std::variant<CreateVertex, ScanAll>;
|
||||
|
||||
struct Op {
|
||||
OpVariant inner;
|
||||
@@ -88,8 +81,8 @@ using namespace memgraph::tests::simulation;
|
||||
template <>
|
||||
struct Arbitrary<CreateVertex> {
|
||||
static Gen<CreateVertex> arbitrary() {
|
||||
return gen::build<CreateVertex>(gen::set(&CreateVertex::first, gen::inRange(0, kMaximumKey)),
|
||||
gen::set(&CreateVertex::second, gen::inRange(0, kMaximumKey)));
|
||||
return gen::build<CreateVertex>(gen::set(&CreateVertex::first, gen::inRange(0, kMaximumShards + 1)),
|
||||
gen::set(&CreateVertex::second, gen::inRange(0, kMaximumShards + 1)));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -98,23 +91,15 @@ struct Arbitrary<ScanAll> {
|
||||
static Gen<ScanAll> arbitrary() { return gen::just(ScanAll{}); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Arbitrary<AssertShardsSplit> {
|
||||
static Gen<AssertShardsSplit> arbitrary() { return gen::just(AssertShardsSplit{}); }
|
||||
};
|
||||
|
||||
OpVariant opHoist(ScanAll op) { return op; }
|
||||
OpVariant opHoist(CreateVertex op) { return op; }
|
||||
OpVariant opHoist(AssertShardsSplit op) { return op; }
|
||||
|
||||
template <>
|
||||
struct ::rc::Arbitrary<Op> {
|
||||
static Gen<Op> arbitrary() {
|
||||
return gen::build<Op>(gen::set(
|
||||
&Op::inner,
|
||||
gen::oneOf(gen::map(gen::arbitrary<CreateVertex>(), [](CreateVertex op) { return opHoist(op); }),
|
||||
gen::map(gen::arbitrary<ScanAll>(), [](ScanAll op) { return opHoist(op); }),
|
||||
gen::map(gen::arbitrary<AssertShardsSplit>(), [](AssertShardsSplit op) { return opHoist(op); }))));
|
||||
&Op::inner, gen::oneOf(gen::map(gen::arbitrary<CreateVertex>(), [](CreateVertex op) { return opHoist(op); }),
|
||||
gen::map(gen::arbitrary<ScanAll>(), [](ScanAll op) { return opHoist(op); }))));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
// This test serves as an example of a property-based model test.
|
||||
// It generates a cluster configuration and a set of operations to
|
||||
// apply against both the real system and a greatly simplified model.
|
||||
|
||||
#include <chrono>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <rapidcheck.h>
|
||||
#include <rapidcheck/gtest.h>
|
||||
#include <spdlog/cfg/env.h>
|
||||
|
||||
#include "generated_operations.hpp"
|
||||
|
||||
namespace memgraph::tests::simulation {
|
||||
|
||||
void my_code(NonEmptyOpVec input) { RC_ASSERT(input.ops.size() < 3); }
|
||||
|
||||
RC_GTEST_PROP(PropertyTestDemo, Demo1, (NonEmptyOpVec ops, uint64_t rng_seed)) {
|
||||
spdlog::cfg::load_env_levels();
|
||||
|
||||
my_code(ops);
|
||||
|
||||
spdlog::trace("passed stats comparison - all good!");
|
||||
}
|
||||
|
||||
} // namespace memgraph::tests::simulation
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
@@ -40,7 +39,7 @@
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::query::v2::tests {
|
||||
using coordinator::PeerMetadata;
|
||||
using coordinator::AddressAndStatus;
|
||||
using CompoundKey = coordinator::PrimaryKey;
|
||||
using coordinator::Coordinator;
|
||||
using coordinator::CoordinatorClient;
|
||||
@@ -101,8 +100,7 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
|
||||
SchemaProperty{.property_id = property_id_2, .type = type_2},
|
||||
};
|
||||
|
||||
const auto split_threshold = 4;
|
||||
auto label_success = sm.InitializeNewLabel(label_name, schema, 1, split_threshold, sm.shard_map_version);
|
||||
auto label_success = sm.InitializeNewLabel(label_name, schema, 1, sm.shard_map_version);
|
||||
MG_ASSERT(label_success);
|
||||
|
||||
const LabelId label_id = sm.labels.at(label_name);
|
||||
@@ -111,11 +109,11 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
|
||||
shards_for_label.clear();
|
||||
|
||||
// add first shard at [0, 0]
|
||||
PeerMetadata aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
|
||||
ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = {1}};
|
||||
ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
|
||||
|
||||
auto key1 = storage::v3::PropertyValue(0);
|
||||
auto key2 = storage::v3::PropertyValue(0);
|
||||
@@ -123,11 +121,11 @@ ShardMap CreateDummyShardmap(coordinator::Address a_io_1, coordinator::Address a
|
||||
shards_for_label[compound_key_1] = shard1;
|
||||
|
||||
// add second shard at [12, 13]
|
||||
PeerMetadata aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
|
||||
ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = {1}};
|
||||
ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
|
||||
|
||||
auto key3 = storage::v3::PropertyValue(12);
|
||||
auto key4 = storage::v3::PropertyValue(13);
|
||||
|
||||
@@ -1496,13 +1496,9 @@ int TestMessages() {
|
||||
|
||||
std::vector<SchemaProperty> schema_prop = {get_schema_property()};
|
||||
|
||||
const coordinator::Hlc shard_version{0};
|
||||
auto shard_ptr1 =
|
||||
std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
|
||||
auto shard_ptr2 =
|
||||
std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
|
||||
auto shard_ptr3 =
|
||||
std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop, shard_version);
|
||||
auto shard_ptr1 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
|
||||
auto shard_ptr2 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
|
||||
auto shard_ptr3 = std::make_unique<Shard>(get_primary_label(), min_prim_key, max_prim_key, schema_prop);
|
||||
|
||||
shard_ptr1->StoreMapping(
|
||||
{{1, "label"}, {2, "prop1"}, {3, "label1"}, {4, "prop2"}, {5, "prop3"}, {6, "prop4"}, {7, "e_prop"}});
|
||||
@@ -1515,13 +1511,9 @@ int TestMessages() {
|
||||
std::vector<Address> address_for_2{shard_server_1_address, shard_server_3_address};
|
||||
std::vector<Address> address_for_3{shard_server_1_address, shard_server_2_address};
|
||||
|
||||
utils::Sender<msgs::InitializeSplitShard> local_shard_manager_sender{[](const auto &elem) {}};
|
||||
ConcreteShardRsm shard_server1(std::move(shard_server_io_1), address_for_1,
|
||||
ShardRsm(std::move(shard_ptr1), local_shard_manager_sender));
|
||||
ConcreteShardRsm shard_server2(std::move(shard_server_io_2), address_for_2,
|
||||
ShardRsm(std::move(shard_ptr2), local_shard_manager_sender));
|
||||
ConcreteShardRsm shard_server3(std::move(shard_server_io_3), address_for_3,
|
||||
ShardRsm(std::move(shard_ptr3), local_shard_manager_sender));
|
||||
ConcreteShardRsm shard_server1(std::move(shard_server_io_1), address_for_1, ShardRsm(std::move(shard_ptr1)));
|
||||
ConcreteShardRsm shard_server2(std::move(shard_server_io_2), address_for_2, ShardRsm(std::move(shard_ptr2)));
|
||||
ConcreteShardRsm shard_server3(std::move(shard_server_io_3), address_for_3, ShardRsm(std::move(shard_ptr3)));
|
||||
|
||||
auto server_thread1 = std::jthread([&shard_server1]() { shard_server1.Run(); });
|
||||
simulator.IncrementServerCountAndWaitForQuiescentState(shard_server_1_address);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -33,12 +33,12 @@
|
||||
#include "utils/result.hpp"
|
||||
|
||||
using memgraph::common::SchemaType;
|
||||
using memgraph::coordinator::AddressAndStatus;
|
||||
using memgraph::coordinator::Coordinator;
|
||||
using memgraph::coordinator::CoordinatorClient;
|
||||
using memgraph::coordinator::CoordinatorRsm;
|
||||
using memgraph::coordinator::HlcRequest;
|
||||
using memgraph::coordinator::HlcResponse;
|
||||
using memgraph::coordinator::PeerMetadata;
|
||||
using memgraph::coordinator::PrimaryKey;
|
||||
using memgraph::coordinator::ShardMap;
|
||||
using memgraph::coordinator::ShardMetadata;
|
||||
@@ -94,10 +94,9 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
|
||||
SchemaProperty{.property_id = property_id_1, .type = type_1},
|
||||
SchemaProperty{.property_id = property_id_2, .type = type_2},
|
||||
};
|
||||
const size_t replication_factor = 3;
|
||||
const auto split_threshold = 999;
|
||||
size_t replication_factor = 3;
|
||||
std::optional<LabelId> label_id_opt =
|
||||
sm.InitializeNewLabel(label_name, schema, replication_factor, split_threshold, sm.shard_map_version);
|
||||
sm.InitializeNewLabel(label_name, schema, replication_factor, sm.shard_map_version);
|
||||
MG_ASSERT(label_id_opt.has_value());
|
||||
|
||||
const LabelId label_id = label_id_opt.value();
|
||||
@@ -106,9 +105,9 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
|
||||
shards_for_label.clear();
|
||||
|
||||
// add first shard at [0, 0]
|
||||
PeerMetadata aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_1{.address = a_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_2{.address = a_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas1_3{.address = a_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
|
||||
ShardMetadata shard1 = ShardMetadata{.peers = {aas1_1, aas1_2, aas1_3}, .version = 1};
|
||||
|
||||
@@ -118,9 +117,9 @@ ShardMap CreateDummyShardmap(Address a_io_1, Address a_io_2, Address a_io_3, Add
|
||||
shards_for_label.emplace(compound_key_1, shard1);
|
||||
|
||||
// add second shard at [12, 13]
|
||||
PeerMetadata aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
PeerMetadata aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_1{.address = b_io_1, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_2{.address = b_io_2, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
AddressAndStatus aas2_3{.address = b_io_3, .status = Status::CONSENSUS_PARTICIPANT};
|
||||
|
||||
ShardMetadata shard2 = ShardMetadata{.peers = {aas2_1, aas2_2, aas2_3}, .version = 1};
|
||||
|
||||
|
||||
@@ -27,18 +27,9 @@
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
|
||||
class FlatStream {
|
||||
std::vector<query::v2::TypedValue> all_results_;
|
||||
|
||||
public:
|
||||
void Result(const std::vector<query::v2::TypedValue> &new_values) {
|
||||
all_results_.insert(all_results_.end(), new_values.begin(), new_values.end());
|
||||
}
|
||||
|
||||
std::vector<query::v2::TypedValue> &&TakeResults() { return std::move(all_results_); }
|
||||
};
|
||||
|
||||
class SimulatedInterpreter {
|
||||
using ResultStream = query::v2::DiscardValueResultStream;
|
||||
|
||||
public:
|
||||
explicit SimulatedInterpreter(std::unique_ptr<query::v2::InterpreterContext> interpreter_context)
|
||||
: interpreter_context_(std::move(interpreter_context)) {
|
||||
@@ -51,8 +42,23 @@ class SimulatedInterpreter {
|
||||
SimulatedInterpreter &operator=(SimulatedInterpreter &&) = delete;
|
||||
~SimulatedInterpreter() = default;
|
||||
|
||||
std::vector<query::v2::TypedValue> RunQuery(const std::string &query) {
|
||||
FlatStream stream;
|
||||
void InstallSimulatorTicker(Simulator &simulator) {
|
||||
interpreter_->InstallSimulatorTicker(simulator.GetSimulatorTickClosure());
|
||||
}
|
||||
|
||||
std::vector<ResultStream> RunQueries(const std::vector<std::string> &queries) {
|
||||
std::vector<ResultStream> results;
|
||||
results.reserve(queries.size());
|
||||
|
||||
for (const auto &query : queries) {
|
||||
results.emplace_back(RunQuery(query));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private:
|
||||
ResultStream RunQuery(const std::string &query) {
|
||||
ResultStream stream;
|
||||
|
||||
std::map<std::string, memgraph::storage::v3::PropertyValue> params;
|
||||
const std::string *username = nullptr;
|
||||
@@ -60,10 +66,9 @@ class SimulatedInterpreter {
|
||||
interpreter_->Prepare(query, params, username);
|
||||
interpreter_->PullAll(&stream);
|
||||
|
||||
return stream.TakeResults();
|
||||
return stream;
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<query::v2::InterpreterContext> interpreter_context_;
|
||||
std::unique_ptr<query::v2::Interpreter> interpreter_;
|
||||
};
|
||||
|
||||
@@ -36,7 +36,6 @@
|
||||
#include "utils/print_helpers.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
#include "generated_operations.hpp"
|
||||
#include "simulation_interpreter.hpp"
|
||||
|
||||
namespace memgraph::tests::simulation {
|
||||
@@ -72,13 +71,6 @@ using storage::v3::SchemaProperty;
|
||||
using CompoundKey = std::pair<int, int>;
|
||||
using ShardClient = RsmClient<SimulatorTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
|
||||
|
||||
struct SimClientContext {
|
||||
CoordinatorClient<SimulatorTransport> coordinator_client;
|
||||
const ClusterConfig &cluster_config;
|
||||
io::simulator::SimulatedInterpreter interpreter;
|
||||
std::set<CompoundKey> correctness_model;
|
||||
};
|
||||
|
||||
MachineManager<SimulatorTransport> MkMm(Simulator &simulator, std::vector<Address> coordinator_addresses, Address addr,
|
||||
ShardMap shard_map) {
|
||||
MachineConfig config{
|
||||
@@ -124,8 +116,9 @@ void WaitForShardsToInitialize(CoordinatorClient<SimulatorTransport> &coordinato
|
||||
}
|
||||
}
|
||||
|
||||
ShardMap TestShardMap(const ClusterConfig &cluster_config) {
|
||||
ShardMap TestShardMap(int n_splits, int replication_factor) {
|
||||
ShardMap sm{};
|
||||
|
||||
const std::string label_name = std::string("test_label");
|
||||
|
||||
// register new properties
|
||||
@@ -142,100 +135,65 @@ ShardMap TestShardMap(const ClusterConfig &cluster_config) {
|
||||
SchemaProperty{.property_id = property_id_2, .type = type_2},
|
||||
};
|
||||
|
||||
std::optional<LabelId> label_id = sm.InitializeNewLabel(label_name, schema, cluster_config.replication_factor,
|
||||
cluster_config.split_threshold, sm.shard_map_version);
|
||||
std::optional<LabelId> label_id = sm.InitializeNewLabel(label_name, schema, replication_factor, sm.shard_map_version);
|
||||
RC_ASSERT(label_id.has_value());
|
||||
|
||||
// split the shard at N split points
|
||||
for (int64_t i = 1; i < n_splits; ++i) {
|
||||
const auto key1 = memgraph::storage::v3::PropertyValue(i);
|
||||
const auto key2 = memgraph::storage::v3::PropertyValue(0);
|
||||
|
||||
const auto split_point = {key1, key2};
|
||||
|
||||
const bool split_success = sm.SplitShard(sm.shard_map_version, label_id.value(), split_point);
|
||||
|
||||
RC_ASSERT(split_success);
|
||||
}
|
||||
|
||||
return sm;
|
||||
}
|
||||
|
||||
void ExecuteOp(SimClientContext &context, CreateVertex create_vertex) {
|
||||
void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std::set<CompoundKey> &correctness_model,
|
||||
CreateVertex create_vertex) {
|
||||
const auto key1 = memgraph::storage::v3::PropertyValue(create_vertex.first);
|
||||
const auto key2 = memgraph::storage::v3::PropertyValue(create_vertex.second);
|
||||
|
||||
std::vector<msgs::Value> primary_key = {msgs::Value(int64_t(create_vertex.first)),
|
||||
msgs::Value(int64_t(create_vertex.second))};
|
||||
|
||||
if (context.correctness_model.contains(std::make_pair(create_vertex.first, create_vertex.second))) {
|
||||
if (correctness_model.contains(std::make_pair(create_vertex.first, create_vertex.second))) {
|
||||
// TODO(tyler) remove this early-return when we have properly handled setting non-unique vertexes
|
||||
return;
|
||||
}
|
||||
|
||||
std::string query = fmt::format("CREATE (n:test_label{{property_1: {}, property_2: {}}});", create_vertex.first,
|
||||
create_vertex.second);
|
||||
auto label_id = request_router.NameToLabel("test_label");
|
||||
|
||||
auto result_stream = context.interpreter.RunQuery(query);
|
||||
msgs::NewVertex nv{.primary_key = primary_key};
|
||||
nv.label_ids.push_back({label_id});
|
||||
|
||||
// TODO(tyler) is there a better way to assert the actual success of the operation?
|
||||
RC_ASSERT(result_stream.size() == 0);
|
||||
std::vector<msgs::NewVertex> new_vertices;
|
||||
new_vertices.push_back(std::move(nv));
|
||||
|
||||
// RC_ASSERT(!result_stream[0].error.has_value());
|
||||
auto result = request_router.CreateVertices(std::move(new_vertices));
|
||||
|
||||
context.correctness_model.emplace(std::make_pair(create_vertex.first, create_vertex.second));
|
||||
RC_ASSERT(result.size() == 1);
|
||||
RC_ASSERT(!result[0].error.has_value());
|
||||
|
||||
correctness_model.emplace(std::make_pair(create_vertex.first, create_vertex.second));
|
||||
}
|
||||
|
||||
void ExecuteOp(SimClientContext &context, ScanAll scan_all) {
|
||||
auto results = context.interpreter.RunQuery("MATCH (n) RETURN n;");
|
||||
void ExecuteOp(query::v2::RequestRouter<SimulatorTransport> &request_router, std::set<CompoundKey> &correctness_model,
|
||||
ScanAll scan_all) {
|
||||
auto results = request_router.ScanVertices("test_label");
|
||||
|
||||
auto result_set = std::set<CompoundKey>{};
|
||||
RC_ASSERT(results.size() == correctness_model.size());
|
||||
|
||||
for (const auto &typed_value : results) {
|
||||
auto &vertex = typed_value.ValueVertex();
|
||||
auto &pk = vertex.Properties();
|
||||
auto &[l1, v1] = pk[0];
|
||||
auto &[l2, v2] = pk[1];
|
||||
auto compound_key = std::make_pair(v1.int_v, v2.int_v);
|
||||
result_set.emplace(compound_key);
|
||||
for (const auto &vertex_accessor : results) {
|
||||
const auto properties = vertex_accessor.Properties();
|
||||
const auto primary_key = vertex_accessor.Id().second;
|
||||
const CompoundKey model_key = std::make_pair(primary_key[0].int_v, primary_key[1].int_v);
|
||||
RC_ASSERT(correctness_model.contains(model_key));
|
||||
}
|
||||
|
||||
// TODO(tyler) we are getting more results back than expected, due to
|
||||
// race conditions during shard splits causing multiple shards to be
|
||||
// queried by the client. The std::set above papers-over this, but
|
||||
// it's an issue that needs to be addressed eventually.
|
||||
RC_ASSERT(result_set == context.correctness_model);
|
||||
}
|
||||
|
||||
void ExecuteOp(SimClientContext &context, AssertShardsSplit assert_shards_split) {
|
||||
// 1 -> 1
|
||||
// 2 -> 1
|
||||
// 3 -> 1
|
||||
// 4 -> 2
|
||||
// 5 -> 2
|
||||
// 6 -> 2
|
||||
// 7 -> 3
|
||||
const int max_shard_size = context.cluster_config.split_threshold - 1;
|
||||
const int min_shards = (std::max((size_t)1, context.correctness_model.size()) - 1) / max_shard_size;
|
||||
const int minimum_expected_shards = min_shards + 1;
|
||||
// TODO(tyler) make this a higher number of retries
|
||||
const int maximum_attempts = 100;
|
||||
size_t initialized_shards;
|
||||
|
||||
for (int i = 0; i < maximum_attempts; i++) {
|
||||
GetShardMapRequest req{};
|
||||
CoordinatorReadRequests read_req = req;
|
||||
auto read_res = context.coordinator_client.SendReadRequest(read_req);
|
||||
if (read_res.HasError()) {
|
||||
// timed out
|
||||
continue;
|
||||
}
|
||||
auto response_result = read_res.GetValue();
|
||||
auto response = std::get<GetShardMapResponse>(response_result);
|
||||
auto shard_map = response.shard_map;
|
||||
|
||||
initialized_shards = shard_map.InitializedShards();
|
||||
|
||||
if (initialized_shards >= minimum_expected_shards) {
|
||||
spdlog::info(
|
||||
"AssertShardsSplit returning after we see {} initialized shards ({} minimum expected with model size {} and "
|
||||
"split threshold {})",
|
||||
initialized_shards, minimum_expected_shards, context.correctness_model.size(),
|
||||
context.cluster_config.split_threshold);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::error("expected {} shards, but we only have {}", minimum_expected_shards, initialized_shards);
|
||||
MG_ASSERT(false, "exceeded maximum attempts for waiting for the expected number of shard splits to occur");
|
||||
}
|
||||
|
||||
/// This struct exists as a way of detaching
|
||||
@@ -264,17 +222,15 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
|
||||
auto machine_1_addr = Address::TestAddress(1);
|
||||
auto cli_addr = Address::TestAddress(2);
|
||||
auto cli_addr_2 = Address::TestAddress(3);
|
||||
auto cli_addr_3 = Address::TestAddress(4);
|
||||
|
||||
Io<SimulatorTransport> cli_io = simulator.Register(cli_addr);
|
||||
Io<SimulatorTransport> cli_io_2 = simulator.Register(cli_addr_2);
|
||||
Io<SimulatorTransport> cli_io_3 = simulator.Register(cli_addr_3);
|
||||
|
||||
auto coordinator_addresses = std::vector{
|
||||
machine_1_addr,
|
||||
};
|
||||
|
||||
ShardMap initialization_sm = TestShardMap(cluster_config);
|
||||
ShardMap initialization_sm = TestShardMap(cluster_config.shards - 1, cluster_config.replication_factor);
|
||||
|
||||
auto mm_1 = MkMm(simulator, coordinator_addresses, machine_1_addr, initialization_sm);
|
||||
Address coordinator_address = mm_1.CoordinatorAddress();
|
||||
@@ -286,19 +242,19 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
|
||||
|
||||
// TODO(tyler) clarify addresses of coordinator etc... as it's a mess
|
||||
|
||||
CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, coordinator_address, {coordinator_address});
|
||||
WaitForShardsToInitialize(coordinator_client);
|
||||
|
||||
query::v2::RequestRouter<SimulatorTransport> request_router(std::move(coordinator_client), std::move(cli_io));
|
||||
std::function<bool()> tick_simulator = simulator.GetSimulatorTickClosure();
|
||||
request_router.InstallSimulatorTicker(tick_simulator);
|
||||
|
||||
request_router.StartTransaction();
|
||||
|
||||
auto correctness_model = std::set<CompoundKey>{};
|
||||
|
||||
SimClientContext context{
|
||||
.coordinator_client = CoordinatorClient<SimulatorTransport>(cli_io, coordinator_address, {coordinator_address}),
|
||||
.cluster_config = cluster_config,
|
||||
.interpreter = io::simulator::SetUpInterpreter(coordinator_address, simulator),
|
||||
.correctness_model = correctness_model,
|
||||
};
|
||||
|
||||
WaitForShardsToInitialize(context.coordinator_client);
|
||||
|
||||
for (const Op &op : ops) {
|
||||
std::visit([&](auto &o) { ExecuteOp(context, o); }, op.inner);
|
||||
std::visit([&](auto &o) { ExecuteOp(request_router, correctness_model, o); }, op.inner);
|
||||
}
|
||||
|
||||
// We have now completed our workload without failing any assertions, so we can
|
||||
@@ -325,4 +281,65 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulation(const
|
||||
return std::make_pair(stats, histo);
|
||||
}
|
||||
|
||||
std::pair<SimulatorStats, LatencyHistogramSummaries> RunClusterSimulationWithQueries(
|
||||
const SimulatorConfig &sim_config, const ClusterConfig &cluster_config, const std::vector<std::string> &queries) {
|
||||
spdlog::info("========================== NEW SIMULATION ==========================");
|
||||
|
||||
auto simulator = Simulator(sim_config);
|
||||
|
||||
auto machine_1_addr = Address::TestAddress(1);
|
||||
auto cli_addr = Address::TestAddress(2);
|
||||
auto cli_addr_2 = Address::TestAddress(3);
|
||||
|
||||
Io<SimulatorTransport> cli_io = simulator.Register(cli_addr);
|
||||
Io<SimulatorTransport> cli_io_2 = simulator.Register(cli_addr_2);
|
||||
|
||||
auto coordinator_addresses = std::vector{
|
||||
machine_1_addr,
|
||||
};
|
||||
|
||||
ShardMap initialization_sm = TestShardMap(cluster_config.shards - 1, cluster_config.replication_factor);
|
||||
|
||||
auto mm_1 = MkMm(simulator, coordinator_addresses, machine_1_addr, initialization_sm);
|
||||
Address coordinator_address = mm_1.CoordinatorAddress();
|
||||
|
||||
auto mm_thread_1 = std::jthread(RunMachine, std::move(mm_1));
|
||||
simulator.IncrementServerCountAndWaitForQuiescentState(machine_1_addr);
|
||||
|
||||
auto detach_on_error = DetachIfDropped{.handle = mm_thread_1};
|
||||
|
||||
// TODO(tyler) clarify addresses of coordinator etc... as it's a mess
|
||||
|
||||
CoordinatorClient<SimulatorTransport> coordinator_client(cli_io, coordinator_address, {coordinator_address});
|
||||
WaitForShardsToInitialize(coordinator_client);
|
||||
|
||||
auto simulated_interpreter = io::simulator::SetUpInterpreter(coordinator_address, simulator);
|
||||
simulated_interpreter.InstallSimulatorTicker(simulator);
|
||||
|
||||
auto query_results = simulated_interpreter.RunQueries(queries);
|
||||
|
||||
// We have now completed our workload without failing any assertions, so we can
|
||||
// disable detaching the worker thread, which will cause the mm_thread_1 jthread
|
||||
// to be joined when this function returns.
|
||||
detach_on_error.detach = false;
|
||||
|
||||
simulator.ShutDown();
|
||||
|
||||
mm_thread_1.join();
|
||||
|
||||
SimulatorStats stats = simulator.Stats();
|
||||
|
||||
spdlog::info("total messages: {}", stats.total_messages);
|
||||
spdlog::info("dropped messages: {}", stats.dropped_messages);
|
||||
spdlog::info("timed out requests: {}", stats.timed_out_requests);
|
||||
spdlog::info("total requests: {}", stats.total_requests);
|
||||
spdlog::info("total responses: {}", stats.total_responses);
|
||||
spdlog::info("simulator ticks: {}", stats.simulator_ticks);
|
||||
|
||||
auto histo = cli_io_2.ResponseLatencies();
|
||||
|
||||
spdlog::info("========================== SUCCESS :) ==========================");
|
||||
return std::make_pair(stats, histo);
|
||||
}
|
||||
|
||||
} // namespace memgraph::tests::simulation
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
|
||||
namespace memgraph::tests::simulation {
|
||||
|
||||
static constexpr auto kMaximumKey = 5;
|
||||
// TODO(tyler) increase this when we start standing up multiple machines in cluster tests
|
||||
static constexpr auto kMinimumShards = 1;
|
||||
static constexpr auto kMaximumShards = kMinimumShards + 10;
|
||||
|
||||
// TODO(tyler) increase this when we start standing up multiple machines in cluster tests
|
||||
static constexpr auto kMinimumServers = 1;
|
||||
@@ -23,7 +25,4 @@ static constexpr auto kMaximumServers = kMinimumServers + 1;
|
||||
static constexpr auto kMinimumReplicationFactor = 1;
|
||||
static constexpr auto kMaximumReplicationFactor = kMinimumReplicationFactor + 1;
|
||||
|
||||
static constexpr auto kMinimumSplitThreshold = 4;
|
||||
static constexpr auto kMaximumSplitThreshold = 5;
|
||||
|
||||
} // namespace memgraph::tests::simulation
|
||||
|
||||
@@ -294,9 +294,6 @@ target_link_libraries(${test_prefix}storage_v3_expr mg-storage-v3 mg-expr)
|
||||
add_unit_test(storage_v3_schema.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3)
|
||||
|
||||
add_unit_test(storage_v3_shard_split.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v3_shard_split mg-storage-v3 mg-query-v2)
|
||||
|
||||
# Test mg-query-v2
|
||||
# These are commented out because of the new TypedValue in the query engine
|
||||
# add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp)
|
||||
|
||||
@@ -124,7 +124,7 @@ void WaitForShardsToInitialize(CoordinatorClient<LocalTransport> &coordinator_cl
|
||||
}
|
||||
}
|
||||
|
||||
ShardMap TestShardMap(int shards, int replication_factor, int split_threshold) {
|
||||
ShardMap TestShardMap(int shards, int replication_factor, int gap_between_shards) {
|
||||
ShardMap sm{};
|
||||
|
||||
const auto label_name = std::string("test_label");
|
||||
@@ -143,10 +143,21 @@ ShardMap TestShardMap(int shards, int replication_factor, int split_threshold) {
|
||||
SchemaProperty{.property_id = property_id_2, .type = type_2},
|
||||
};
|
||||
|
||||
std::optional<LabelId> label_id =
|
||||
sm.InitializeNewLabel(label_name, schema, replication_factor, split_threshold, sm.shard_map_version);
|
||||
std::optional<LabelId> label_id = sm.InitializeNewLabel(label_name, schema, replication_factor, sm.shard_map_version);
|
||||
MG_ASSERT(label_id.has_value());
|
||||
|
||||
// split the shard at N split points
|
||||
for (int64_t i = 1; i < shards; ++i) {
|
||||
const auto key1 = memgraph::storage::v3::PropertyValue(i * gap_between_shards);
|
||||
const auto key2 = memgraph::storage::v3::PropertyValue(0);
|
||||
|
||||
const auto split_point = {key1, key2};
|
||||
|
||||
const bool split_success = sm.SplitShard(sm.shard_map_version, label_id.value(), split_point);
|
||||
|
||||
MG_ASSERT(split_success);
|
||||
}
|
||||
|
||||
return sm;
|
||||
}
|
||||
|
||||
@@ -194,15 +205,15 @@ void ExecuteOp(query::v2::RequestRouter<LocalTransport> &request_router, std::se
|
||||
}
|
||||
}
|
||||
|
||||
void RunWorkload(int shards, int replication_factor, int split_threshold, int create_ops, int scan_ops,
|
||||
int shard_worker_threads, int gap_between_shards) {
|
||||
void RunWorkload(int shards, int replication_factor, int create_ops, int scan_ops, int shard_worker_threads,
|
||||
int gap_between_shards) {
|
||||
spdlog::info("======================== NEW TEST ========================");
|
||||
spdlog::info("shards: ", shards);
|
||||
spdlog::info("replication factor: ", replication_factor);
|
||||
spdlog::info("create ops: ", create_ops);
|
||||
spdlog::info("scan all ops: ", scan_ops);
|
||||
spdlog::info("shard worker threads: ", shard_worker_threads);
|
||||
spdlog::info("split threshold: ", split_threshold);
|
||||
spdlog::info("gap between shards: ", gap_between_shards);
|
||||
|
||||
LocalSystem local_system;
|
||||
|
||||
@@ -217,7 +228,7 @@ void RunWorkload(int shards, int replication_factor, int split_threshold, int cr
|
||||
};
|
||||
|
||||
auto time_before_shard_map_creation = cli_io_2.Now();
|
||||
ShardMap initialization_sm = TestShardMap(shards, replication_factor, split_threshold);
|
||||
ShardMap initialization_sm = TestShardMap(shards, replication_factor, gap_between_shards);
|
||||
auto time_after_shard_map_creation = cli_io_2.Now();
|
||||
|
||||
auto mm_1 = MkMm(local_system, coordinator_addresses, machine_1_addr, initialization_sm, shard_worker_threads);
|
||||
@@ -270,20 +281,19 @@ void RunWorkload(int shards, int replication_factor, int split_threshold, int cr
|
||||
}
|
||||
|
||||
TEST(MachineManager, ManyShards) {
|
||||
const auto shards_attempts = {1, 64};
|
||||
const auto shard_worker_thread_attempts = {1, 32};
|
||||
const auto replication_factor = 1;
|
||||
const auto create_ops = 128;
|
||||
const auto scan_ops = 1;
|
||||
const auto shards = 1;
|
||||
const auto split_threshold = create_ops / shards;
|
||||
auto shards_attempts = {1, 64};
|
||||
auto shard_worker_thread_attempts = {1, 32};
|
||||
auto replication_factor = 1;
|
||||
auto create_ops = 128;
|
||||
auto scan_ops = 1;
|
||||
|
||||
std::cout << "splits threads scan_all_microseconds\n";
|
||||
|
||||
for (const auto shards : shards_attempts) {
|
||||
auto gap_between_shards = create_ops / shards;
|
||||
|
||||
for (const auto shard_worker_threads : shard_worker_thread_attempts) {
|
||||
RunWorkload(shards, replication_factor, split_threshold, create_ops, scan_ops, shard_worker_threads,
|
||||
split_threshold);
|
||||
RunWorkload(shards, replication_factor, create_ops, scan_ops, shard_worker_threads, gap_between_shards);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,13 +82,29 @@ ShardMap TestShardMap() {
|
||||
};
|
||||
|
||||
const size_t replication_factor = 1;
|
||||
const size_t split_threshold = 4;
|
||||
|
||||
const auto label_id =
|
||||
sm.InitializeNewLabel(kLabelName, schema, replication_factor, split_threshold, sm.shard_map_version);
|
||||
const auto label_id = sm.InitializeNewLabel(kLabelName, schema, replication_factor, sm.shard_map_version);
|
||||
EXPECT_TRUE(label_id.has_value());
|
||||
|
||||
sm.AllocateEdgeTypeIds(std::vector<std::string>{"edge_type"});
|
||||
// split the shard at N split points
|
||||
// NB: this is the logic that should be provided by the "split file"
|
||||
// TODO(tyler) split points should account for signedness
|
||||
const size_t n_splits = 16;
|
||||
const auto split_interval = std::numeric_limits<int64_t>::max() / n_splits;
|
||||
|
||||
for (int64_t i = 0; i < n_splits; ++i) {
|
||||
const int64_t value = i * split_interval;
|
||||
|
||||
const auto key1 = memgraph::storage::v3::PropertyValue(value);
|
||||
const auto key2 = memgraph::storage::v3::PropertyValue(0);
|
||||
|
||||
const CompoundKey split_point = {key1, key2};
|
||||
|
||||
const auto split_success = sm.SplitShard(sm.shard_map_version, label_id.value(), split_point);
|
||||
|
||||
EXPECT_TRUE(split_success);
|
||||
}
|
||||
|
||||
return sm;
|
||||
}
|
||||
|
||||
@@ -188,8 +188,7 @@ ShardMap CreateDummyShardmap() {
|
||||
SchemaProperty{.property_id = property_id_7, .type = type_1},
|
||||
};
|
||||
|
||||
const auto split_threshold = 4;
|
||||
auto label_success = sm.InitializeNewLabel(label_name, schema, 1, split_threshold, sm.shard_map_version);
|
||||
auto label_success = sm.InitializeNewLabel(label_name, schema, 1, sm.shard_map_version);
|
||||
MG_ASSERT(label_success);
|
||||
|
||||
const LabelId label_id = sm.labels.at(label_name);
|
||||
|
||||
@@ -78,9 +78,9 @@ class StorageV3 : public ::testing::TestWithParam<bool> {
|
||||
const PropertyId primary_property{PropertyId::FromUint(2)};
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
Shard store{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector,
|
||||
Config{.gc = {.reclamation_interval = reclamation_interval}}};
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard store{primary_label, min_pk, std::nullopt /*max_primary_key*/,
|
||||
schema_property_vector, last_hlc, Config{.gc = {.reclamation_interval = reclamation_interval}}};
|
||||
};
|
||||
INSTANTIATE_TEST_SUITE_P(WithGc, StorageV3, ::testing::Values(true));
|
||||
INSTANTIATE_TEST_SUITE_P(WithoutGc, StorageV3, ::testing::Values(false));
|
||||
@@ -2650,7 +2650,7 @@ TEST_P(StorageV3, TestCreateVertexAndValidate) {
|
||||
(std::map<PropertyId, PropertyValue>{{prop1, PropertyValue(111)}}));
|
||||
}
|
||||
{
|
||||
Shard store(primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc);
|
||||
Shard store(primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector);
|
||||
auto acc = store.Access(GetNextHlc());
|
||||
auto vertex1 = acc.CreateVertexAndValidate({}, {PropertyValue{0}}, {});
|
||||
auto vertex2 = acc.CreateVertexAndValidate({}, {PropertyValue{0}}, {});
|
||||
|
||||
@@ -54,9 +54,10 @@ class StorageEdgeTest : public ::testing::TestWithParam<bool> {
|
||||
const PropertyId primary_property{PropertyId::FromUint(2)};
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
Shard store{primary_label, min_pk, max_pk, schema_property_vector,
|
||||
Config{.items = {.properties_on_edges = GetParam()}}};
|
||||
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard store{primary_label, min_pk, max_pk,
|
||||
schema_property_vector, last_hlc, Config{.items = {.properties_on_edges = GetParam()}}};
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(EdgesWithProperties, StorageEdgeTest, ::testing::Values(true));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -136,8 +136,7 @@ class ExpressionEvaluatorTest : public ::testing::Test {
|
||||
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard db{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
|
||||
Shard db{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
|
||||
|
||||
Shard::Accessor storage_dba{db.Access(GetNextHlc())};
|
||||
DbAccessor dba{&storage_dba};
|
||||
@@ -150,6 +149,8 @@ class ExpressionEvaluatorTest : public ::testing::Test {
|
||||
Frame frame{128};
|
||||
ExpressionEvaluator eval{&frame, symbol_table, ctx, &dba, View::OLD};
|
||||
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
|
||||
void SetUp() override { db.StoreMapping({{1, "label"}, {2, "property"}}); }
|
||||
|
||||
std::vector<PropertyId> NamesToProperties(const std::vector<std::string> &property_names) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -44,8 +44,7 @@ class IndexTest : public testing::Test {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
|
||||
const LabelId primary_label{LabelId::FromUint(1)};
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
|
||||
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
|
||||
|
||||
const PropertyId prop_id{PropertyId::FromUint(5)};
|
||||
const PropertyId prop_val{PropertyId::FromUint(6)};
|
||||
@@ -56,6 +55,7 @@ class IndexTest : public testing::Test {
|
||||
static constexpr io::Duration one_time_unit{1};
|
||||
int primary_key_id{0};
|
||||
int vertex_id{0};
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
|
||||
LabelId NameToLabelId(std::string_view label_name) { return storage.NameToLabel(label_name); }
|
||||
|
||||
|
||||
@@ -82,9 +82,8 @@ TEST_P(StorageIsolationLevelTest, Visibility) {
|
||||
|
||||
for (auto override_isolation_level_index{0U}; override_isolation_level_index < isolation_levels.size();
|
||||
++override_isolation_level_index) {
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard store{primary_label, min_pk, max_pk,
|
||||
schema_property_vector, last_hlc, Config{.transaction = {.isolation_level = default_isolation_level}}};
|
||||
Shard store{primary_label, min_pk, max_pk, schema_property_vector,
|
||||
Config{.transaction = {.isolation_level = default_isolation_level}}};
|
||||
const auto override_isolation_level = isolation_levels[override_isolation_level_index];
|
||||
auto creator = store.Access(GetNextHlc());
|
||||
auto default_isolation_level_reader = store.Access(GetNextHlc());
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "common/types.hpp"
|
||||
@@ -57,20 +58,15 @@ class ShardRSMTest : public testing::Test {
|
||||
PropertyValue max_pk(static_cast<int64_t>(10000000));
|
||||
std::vector<PropertyValue> max_prim_key = {max_pk};
|
||||
|
||||
coordinator::Hlc shard_version{GetTransactionId()};
|
||||
auto shard_ptr1 =
|
||||
std::make_unique<Shard>(primary_label, min_prim_key, max_prim_key, std::vector{schema_prop}, shard_version);
|
||||
auto shard_ptr1 = std::make_unique<Shard>(primary_label, min_prim_key, max_prim_key, std::vector{schema_prop});
|
||||
shard_ptr1->StoreMapping({{1, "primary_label"},
|
||||
{2, "primary_label2"},
|
||||
{3, "label"},
|
||||
{4, "primary_prop1"},
|
||||
{5, "primary_prop2"},
|
||||
{6, "prop"}});
|
||||
coordinator::Address local_shard_manager_address = coordinator::Address();
|
||||
utils::Sender<msgs::InitializeSplitShard> local_shard_manager_sender{[](const auto &elem) {}};
|
||||
|
||||
shard_ptr1->CreateSchema(primary_label2, {{primary_property2, SchemaType::INT}});
|
||||
shard_rsm = std::make_unique<ShardRsm>(std::move(shard_ptr1), std::move(local_shard_manager_sender));
|
||||
shard_rsm = std::make_unique<ShardRsm>(std::move(shard_ptr1));
|
||||
}
|
||||
|
||||
LabelId NameToLabel(const std::string &name) { return LabelId::FromUint(id_mapper_.NameToId(name)); }
|
||||
|
||||
@@ -1,500 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include <gmock/gmock-matchers.h>
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/delta.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/mvcc.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
|
||||
using testing::Pair;
|
||||
using testing::UnorderedElementsAre;
|
||||
|
||||
namespace memgraph::storage::v3::tests {
|
||||
|
||||
class ShardSplitTest : public testing::Test {
|
||||
protected:
|
||||
void SetUp() override {
|
||||
storage.StoreMapping(
|
||||
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
|
||||
}
|
||||
|
||||
const PropertyId primary_property{PropertyId::FromUint(2)};
|
||||
const PropertyId secondary_property{PropertyId::FromUint(5)};
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
|
||||
const LabelId primary_label{LabelId::FromUint(1)};
|
||||
const LabelId secondary_label{LabelId::FromUint(4)};
|
||||
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
|
||||
|
||||
coordinator::Hlc GetNextHlc() {
|
||||
++last_hlc.logical_id;
|
||||
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
|
||||
return last_hlc;
|
||||
}
|
||||
|
||||
void AssertShardState(auto &shard, const int split_min, const int split_max) {
|
||||
auto acc = shard.Access(GetNextHlc());
|
||||
for (int i{0}; i < split_min; ++i) {
|
||||
EXPECT_FALSE(acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD).has_value());
|
||||
}
|
||||
for (int i{split_min}; i < split_max; ++i) {
|
||||
const auto vtx = acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD);
|
||||
ASSERT_TRUE(vtx.has_value());
|
||||
EXPECT_TRUE(vtx->InEdges(View::OLD)->size() == 1 || vtx->OutEdges(View::OLD)->size() == 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void AssertEqVertexContainer(const VertexContainer &actual, const VertexContainer &expected) {
|
||||
ASSERT_EQ(actual.size(), expected.size());
|
||||
|
||||
auto expected_it = expected.begin();
|
||||
auto actual_it = actual.begin();
|
||||
while (expected_it != expected.end()) {
|
||||
EXPECT_EQ(actual_it->first, expected_it->first);
|
||||
EXPECT_EQ(actual_it->second.deleted, expected_it->second.deleted);
|
||||
EXPECT_EQ(actual_it->second.labels, expected_it->second.labels);
|
||||
|
||||
auto *expected_delta = expected_it->second.delta;
|
||||
auto *actual_delta = actual_it->second.delta;
|
||||
// This asserts delta chain
|
||||
while (expected_delta != nullptr) {
|
||||
EXPECT_EQ(actual_delta->action, expected_delta->action);
|
||||
EXPECT_EQ(actual_delta->id, expected_delta->id);
|
||||
|
||||
switch (expected_delta->action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
EXPECT_EQ(actual_delta->label, expected_delta->label);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
EXPECT_EQ(actual_delta->property.key, expected_delta->property.key);
|
||||
EXPECT_EQ(actual_delta->property.value, expected_delta->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const auto expected_prev = expected_delta->prev.Get();
|
||||
const auto actual_prev = actual_delta->prev.Get();
|
||||
switch (expected_prev.type) {
|
||||
case PreviousPtr::Type::NULLPTR: {
|
||||
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::NULLPTR) << "Expected type is nullptr!";
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::DELTA: {
|
||||
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::DELTA) << "Expected type is delta!";
|
||||
EXPECT_EQ(actual_prev.delta->action, expected_prev.delta->action);
|
||||
EXPECT_EQ(actual_prev.delta->id, expected_prev.delta->id);
|
||||
break;
|
||||
}
|
||||
case v3::PreviousPtr::Type::EDGE: {
|
||||
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::EDGE) << "Expected type is edge!";
|
||||
EXPECT_EQ(actual_prev.edge->gid, expected_prev.edge->gid);
|
||||
break;
|
||||
}
|
||||
case v3::PreviousPtr::Type::VERTEX: {
|
||||
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::VERTEX) << "Expected type is vertex!";
|
||||
EXPECT_EQ(actual_prev.vertex->first, expected_prev.vertex->first);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
expected_delta = expected_delta->next;
|
||||
actual_delta = actual_delta->next;
|
||||
}
|
||||
EXPECT_EQ(expected_delta, nullptr);
|
||||
EXPECT_EQ(actual_delta, nullptr);
|
||||
++expected_it;
|
||||
++actual_it;
|
||||
}
|
||||
}
|
||||
|
||||
void AssertEqDeltaLists(const std::list<Delta> &actual, const std::list<Delta> &expected) {
|
||||
EXPECT_EQ(actual.size(), expected.size());
|
||||
auto actual_it = actual.begin();
|
||||
auto expected_it = expected.begin();
|
||||
while (actual_it != actual.end()) {
|
||||
EXPECT_EQ(actual_it->id, expected_it->id);
|
||||
EXPECT_EQ(actual_it->action, expected_it->action);
|
||||
++actual_it;
|
||||
++expected_it;
|
||||
}
|
||||
}
|
||||
|
||||
void AddDeltaToDeltaChain(Vertex *object, Delta *new_delta) {
|
||||
auto *delta_holder = GetDeltaHolder(object);
|
||||
|
||||
new_delta->next = delta_holder->delta;
|
||||
new_delta->prev.Set(object);
|
||||
if (delta_holder->delta) {
|
||||
delta_holder->delta->prev.Set(new_delta);
|
||||
}
|
||||
delta_holder->delta = new_delta;
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitWithVertices) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(
|
||||
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(121)}})
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
auto current_hlc = GetNextHlc();
|
||||
acc.Commit(current_hlc);
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 0);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
|
||||
|
||||
CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
|
||||
Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 4, 1};
|
||||
Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 5, 2};
|
||||
Delta delta_remove_label{Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4};
|
||||
Delta delta_set_property{Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4};
|
||||
Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 8, 3};
|
||||
|
||||
VertexContainer expected_vertices;
|
||||
auto [it_4, inserted1] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
|
||||
delta_delete1.prev.Set(&*it_4);
|
||||
auto [it_5, inserted2] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
|
||||
delta_delete2.prev.Set(&*it_5);
|
||||
auto [it_6, inserted3] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
|
||||
delta_delete3.prev.Set(&*it_6);
|
||||
it_5->second.labels.push_back(secondary_label);
|
||||
AddDeltaToDeltaChain(&*it_5, &delta_set_property);
|
||||
AddDeltaToDeltaChain(&*it_5, &delta_remove_label);
|
||||
|
||||
AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
|
||||
|
||||
// This is to ensure that the transaction that we have don't point to invalid
|
||||
// object on the other shard
|
||||
std::list<Delta> expected_deltas;
|
||||
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 4, 1);
|
||||
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 5, 2);
|
||||
expected_deltas.emplace_back(Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4);
|
||||
expected_deltas.emplace_back(Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4);
|
||||
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 8, 3);
|
||||
AssertEqDeltaLists(splitted_data.transactions.begin()->second->deltas, expected_deltas);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
|
||||
.HasError());
|
||||
|
||||
auto current_hlc = GetNextHlc();
|
||||
acc.Commit(current_hlc);
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 2);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
|
||||
|
||||
CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
|
||||
Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 12, 1};
|
||||
Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 13, 1};
|
||||
Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 14, 1};
|
||||
Delta delta_add_in_edge1{Delta::RemoveInEdgeTag{},
|
||||
edge_type_id,
|
||||
VertexId{primary_label, {PropertyValue(1)}},
|
||||
EdgeRef{Gid::FromUint(1)},
|
||||
&commit_info,
|
||||
17,
|
||||
1};
|
||||
Delta delta_add_out_edge2{Delta::RemoveOutEdgeTag{},
|
||||
edge_type_id,
|
||||
VertexId{primary_label, {PropertyValue(6)}},
|
||||
EdgeRef{Gid::FromUint(2)},
|
||||
&commit_info,
|
||||
19,
|
||||
1};
|
||||
Delta delta_add_in_edge2{Delta::RemoveInEdgeTag{},
|
||||
edge_type_id,
|
||||
VertexId{primary_label, {PropertyValue(4)}},
|
||||
EdgeRef{Gid::FromUint(2)},
|
||||
&commit_info,
|
||||
20,
|
||||
1};
|
||||
VertexContainer expected_vertices;
|
||||
auto [vtx4, inserted4] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
|
||||
auto [vtx5, inserted5] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
|
||||
auto [vtx6, inserted6] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
|
||||
AddDeltaToDeltaChain(&*vtx4, &delta_add_out_edge2);
|
||||
AddDeltaToDeltaChain(&*vtx5, &delta_add_in_edge1);
|
||||
AddDeltaToDeltaChain(&*vtx6, &delta_add_in_edge2);
|
||||
|
||||
AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitBeforeCommit) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
|
||||
.HasError());
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 2);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitWithCommitedAndOngoingTransactions) {
|
||||
{
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
|
||||
acc.Commit(GetNextHlc());
|
||||
}
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
|
||||
.HasError());
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 2);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 2);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitWithLabelIndex) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {}).HasError());
|
||||
acc.Commit(GetNextHlc());
|
||||
storage.CreateIndex(secondary_label);
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 0);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBasicSplitWithLabelPropertyIndex) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(
|
||||
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {{secondary_property, PropertyValue(1)}})
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(
|
||||
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(21)}})
|
||||
.HasError());
|
||||
EXPECT_FALSE(
|
||||
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {{secondary_property, PropertyValue(22)}})
|
||||
.HasError());
|
||||
acc.Commit(GetNextHlc());
|
||||
storage.CreateIndex(secondary_label, secondary_property);
|
||||
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 0);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroyOriginalShard) {
|
||||
const auto split_value{4};
|
||||
PrimaryKey splitted_value{{PropertyValue(4)}};
|
||||
std::unique_ptr<Shard> splitted_shard;
|
||||
|
||||
{
|
||||
Shard storage2{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, GetNextHlc()};
|
||||
auto acc = storage2.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
|
||||
.HasError());
|
||||
acc.Commit(GetNextHlc());
|
||||
|
||||
auto splitted_data = storage2.PerformSplit({PropertyValue(split_value)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data->vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data->edges->size(), 2);
|
||||
EXPECT_EQ(splitted_data->transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data->label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data->label_property_indices.size(), 0);
|
||||
|
||||
// Create a new shard
|
||||
splitted_shard = Shard::FromSplitData(std::move(*splitted_data));
|
||||
// Call gc on old shard
|
||||
storage2.CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
// Destroy original
|
||||
}
|
||||
|
||||
splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
AssertShardState(*splitted_shard, 4, 6);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroySplittedShard) {
|
||||
PrimaryKey splitted_value{{PropertyValue(4)}};
|
||||
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
|
||||
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
|
||||
.HasError());
|
||||
acc.Commit(GetNextHlc());
|
||||
|
||||
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, last_hlc, GetNextHlc());
|
||||
EXPECT_EQ(splitted_data->vertices.size(), 3);
|
||||
EXPECT_EQ(splitted_data->edges->size(), 2);
|
||||
EXPECT_EQ(splitted_data->transactions.size(), 1);
|
||||
EXPECT_EQ(splitted_data->label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data->label_property_indices.size(), 0);
|
||||
|
||||
{
|
||||
// Create a new shard
|
||||
auto splitted_shard = Shard::FromSplitData(std::move(*splitted_data));
|
||||
// Call gc on new shard
|
||||
splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
// Destroy splitted shard
|
||||
}
|
||||
|
||||
storage.CollectGarbage(GetNextHlc().coordinator_wall_clock);
|
||||
AssertShardState(storage, 1, 3);
|
||||
}
|
||||
|
||||
TEST_F(ShardSplitTest, TestBigSplit) {
|
||||
int pk{0};
|
||||
for (int64_t i{0}; i < 10'000; ++i) {
|
||||
auto acc = storage.Access(GetNextHlc());
|
||||
EXPECT_FALSE(
|
||||
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(pk++)}, {{secondary_property, PropertyValue(i)}})
|
||||
.HasError());
|
||||
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(pk++)}, {}).HasError());
|
||||
|
||||
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(pk - 2)}},
|
||||
VertexId{primary_label, PrimaryKey{PropertyValue(pk - 1)}}, edge_type_id,
|
||||
Gid::FromUint(pk))
|
||||
.HasError());
|
||||
acc.Commit(GetNextHlc());
|
||||
}
|
||||
storage.CreateIndex(secondary_label, secondary_property);
|
||||
|
||||
const auto split_value = pk / 2;
|
||||
auto splitted_data = *storage.PerformSplit({PropertyValue(split_value)}, last_hlc, GetNextHlc());
|
||||
|
||||
EXPECT_EQ(splitted_data.vertices.size(), 10000);
|
||||
EXPECT_EQ(splitted_data.edges->size(), 5000);
|
||||
EXPECT_EQ(splitted_data.transactions.size(), 5000);
|
||||
EXPECT_EQ(splitted_data.label_indices.size(), 0);
|
||||
EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
|
||||
|
||||
auto shard = Shard::FromSplitData(std::move(splitted_data));
|
||||
AssertShardState(*shard, split_value, split_value * 2);
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage::v3::tests
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -57,8 +57,9 @@ class StorageV3Accessor : public ::testing::Test {
|
||||
const PropertyId primary_property{PropertyId::FromUint(2)};
|
||||
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
|
||||
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
|
||||
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
|
||||
|
||||
coordinator::Hlc last_hlc{0, io::Time{}};
|
||||
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector, last_hlc};
|
||||
};
|
||||
|
||||
TEST_F(StorageV3Accessor, TestPrimaryLabel) {
|
||||
|
||||
2
tools/plot/.gitignore
vendored
Normal file
2
tools/plot/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*.dat
|
||||
*.out
|
||||
7
tools/plot/pool_size_batch_size_query_latency.gnuplot
Normal file
7
tools/plot/pool_size_batch_size_query_latency.gnuplot
Normal file
@@ -0,0 +1,7 @@
|
||||
set dgrid3d 30,30
|
||||
set hidden3d
|
||||
set label "pool size" at 8, 11000, 0
|
||||
set label "multiframe size" at 20,6000,0
|
||||
set label "execution time (ms)" at 14,0,5000
|
||||
splot "frames1.dat" u 1:2:3 with lines
|
||||
pause mouse close
|
||||
6
tools/plot/thread_shard_scanall_latency.gnuplot
Normal file
6
tools/plot/thread_shard_scanall_latency.gnuplot
Normal file
@@ -0,0 +1,6 @@
|
||||
set dgrid3d 30,30
|
||||
set hidden3d
|
||||
set label "shards" at 20,-1,10000
|
||||
set label "threads" at -10,5,10000
|
||||
splot "data.dat" u 1:2:3 with lines
|
||||
pause mouse close
|
||||
Reference in New Issue
Block a user