Compare commits

..

3 Commits

Author SHA1 Message Date
gvolfing
e70c1c1ae2 Set start-up flag to a fix value for now
The FLAGS_query_vertex_count_to_expand_existing flag has specific
meaning for v2 related things. In v3 we king of hijacked the semantic
mening of indexing beacuse of the label-based indexing.
This caused the rule based planner to pick a not so efficient plan for
execution. with this quick fix the generated plan is correct, however
the flag is useless at this form. This will be reworked in subsequent
commits.
2023-03-09 10:20:00 +01:00
gvolfing
ebdf7344d8 Make a split_threshold stub 2023-03-02 11:07:40 +01:00
gvolfing
f2c4376a44 Add glue-code between vertex_count_cache and the coordinator 2023-03-01 08:31:35 +01:00
76 changed files with 545 additions and 3343 deletions

View File

@@ -271,30 +271,3 @@ jobs:
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
- name: Run query performance tests
run: |
cd tests/manual
./query_performance_runner.py
- name: Get branch name (merge)
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
- name: Get branch name (pull request)
if: github.event_name == 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "query_performance" \
--benchmark-results-path "../../build/tests/manual/query_performance_benchmark/summary.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -18,7 +18,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_r
// add this storage engine to any under-replicated shards that it is not already a part of
return shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
auto initializing_rsms_for_shard_manager =
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
return HeartbeatResponse{
.shards_to_initialize = initializing_rsms_for_shard_manager,
};
}
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
@@ -62,8 +67,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
}
return res;
}

View File

@@ -121,6 +121,15 @@ struct InitializeLabelResponse {
std::optional<ShardMap> fresher_shard_map;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
};
using CoordinatorWriteRequests =
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,17 +23,17 @@ namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc final {
uint64_t logical_id{0};
struct Hlc {
uint64_t logical_id = 0;
Time coordinator_wall_clock = Time::min();
auto operator<=>(const Hlc &other) const noexcept { return logical_id <=> other.logical_id; }
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
bool operator==(const Hlc &other) const noexcept = default;
bool operator<(const Hlc &other) const noexcept = default;
bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
bool operator==(const Hlc &other) const = default;
bool operator<(const Hlc &other) const = default;
bool operator==(const uint64_t other) const { return logical_id == other; }
bool operator<(const uint64_t other) const { return logical_id < other; }
bool operator>=(const uint64_t other) const { return logical_id >= other; }
friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);

View File

@@ -9,7 +9,6 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <map>
#include <optional>
#include <unordered_map>
#include <vector>
@@ -268,8 +267,9 @@ boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
static_cast<unsigned char>(shard_id)};
}
HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
HeartbeatResponse ret{};
std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
std::set<boost::uuids::uuid> initialized) {
std::vector<ShardToInitialize> ret{};
bool mutated = false;
@@ -281,74 +281,48 @@ HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost
high_key = next_it->first;
}
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool shard_assigned_to_machine = false;
bool machine_contains_shard = false;
for (auto &peer_metadata : shard.peers) {
const bool same_machine = peer_metadata.address.last_known_ip == storage_manager.last_known_ip &&
peer_metadata.address.last_known_port == storage_manager.last_known_port;
if (initialized.contains(peer_metadata.address.unique_id)) {
shard_assigned_to_machine = true;
if (!same_machine) {
// set the last known ip and port to the storage manager that has heartbeated it just now
for (auto &aas : shard.peers) {
if (initialized.contains(aas.address.unique_id)) {
machine_contains_shard = true;
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
mutated = true;
peer_metadata.address.last_known_ip = storage_manager.last_known_ip;
peer_metadata.address.last_known_port = storage_manager.last_known_port;
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
aas.status = Status::CONSENSUS_PARTICIPANT;
}
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
mutated = true;
spdlog::info("marking shard as full consensus participant: {}", peer_metadata.address.unique_id);
peer_metadata.status = Status::CONSENSUS_PARTICIPANT;
}
} else if (same_machine && peer_metadata.status == Status::INITIALIZING) {
// we are expecting this shard to be initialized (rather than split) on this machine,
// so send it an initialization request
shard_assigned_to_machine = true;
spdlog::info("reminding shard manager that they should begin participating in shard");
ret.shards_to_initialize.push_back(ShardToInitialize{
.uuid = peer_metadata.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = high_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
} else if (same_machine && peer_metadata.status == Status::PENDING_SPLIT) {
// we are expecting this shard to be split, so send it a split request
ret.shards_to_split.push_back(ShardToSplit{
.shard_to_split_uuid = peer_metadata.split_from,
.new_right_side_uuid = peer_metadata.address.unique_id,
.split_requested_at = shard.version,
.label_id = label_id,
.split_key = low_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
} else {
MG_ASSERT(
!same_machine,
"failed to properly handle a new Status type in the heartbeat management and shard assignment code");
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
aas.address.last_known_port == storage_manager.last_known_port;
if (same_machine) {
machine_contains_shard = true;
spdlog::info("reminding shard manager that they should begin participating in shard");
ret.push_back(ShardToInitialize{
.uuid = aas.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = high_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
}
}
}
if (!shard_assigned_to_machine && shard.peers.size() < label_space.replication_factor) {
Address address = storage_manager;
// NB: increment version for each new uuid for deterministic creation
if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
// increment version for each new uuid for deterministic creation
IncrementShardMapVersion();
Address address = storage_manager;
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
address.unique_id = NewShardUuid(shard_map_version.logical_id);
spdlog::info("assigning shard manager to shard");
ret.shards_to_initialize.push_back(ShardToInitialize{
ret.push_back(ShardToInitialize{
.uuid = address.unique_id,
.label_id = label_id,
.min_key = low_key,
@@ -358,12 +332,12 @@ HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost
.id_to_names = IdToNames(),
});
PeerMetadata peer_metadata = {
AddressAndStatus aas = {
.address = address,
.status = Status::INITIALIZING,
};
shard.peers.emplace_back(peer_metadata);
shard.peers.emplace_back(aas);
}
}
}
@@ -371,12 +345,11 @@ HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
if (previous_shard_map_version != shard_map_version || !label_spaces.contains(label_id)) {
if (previous_shard_map_version != shard_map_version) {
return false;
}
@@ -385,26 +358,11 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons
MG_ASSERT(!shards_in_map.empty());
MG_ASSERT(!shards_in_map.contains(key));
MG_ASSERT(label_spaces.contains(label_id));
// Finding the ShardMetadata that the new PrimaryKey should map to.
ShardMetadata duplicated_shard = GetShardForKey(label_id, key);
std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
for (auto &peer_metadata : duplicated_shard.peers) {
peer_metadata.status = Status::PENDING_SPLIT;
peer_metadata.split_from = peer_metadata.address.unique_id;
// NB: increment version for each new uuid for deterministic creation
IncrementShardMapVersion();
auto new_uuid = NewShardUuid(shard_map_version.logical_id);
// store new uuid for the right side of each shard
split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
peer_metadata.address.unique_id = new_uuid;
}
auto prev = std::prev(shards_in_map.upper_bound(key));
ShardMetadata duplicated_shard = prev->second;
// Apply the split
shards_in_map[key] = duplicated_shard;
@@ -603,8 +561,8 @@ bool ShardMap::ClusterInitialized() const {
return false;
}
for (const auto &peer_metadata : shard.peers) {
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
for (const auto &aas : shard.peers) {
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
return false;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -47,45 +47,38 @@ using memgraph::storage::v3::SchemaProperty;
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
PENDING_SPLIT,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct PeerMetadata {
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
boost::uuids::uuid split_from;
friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
in << "PeerMetadata { address: ";
in << peer_metadata.address;
if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
in << ", status: CONSENSUS_PARTICIPANT";
} else if (peer_metadata.status == Status::INITIALIZING) {
in << ", status: INITIALIZING";
} else if (peer_metadata.status == Status::PENDING_SPLIT) {
in << ", status: PENDING_SPLIT";
friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
in << "AddressAndStatus { address: ";
in << address_and_status.address;
if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
in << ", status: CONSENSUS_PARTICIPANT }";
} else {
MG_ASSERT(false, "failed to update the operator<< implementation for Status");
in << ", status: INITIALIZING }";
}
in << ", split_from: " << peer_metadata.split_from << " }";
return in;
}
friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
return lhs.address == rhs.address;
}
};
using PrimaryKey = std::vector<PropertyValue>;
struct ShardMetadata {
std::vector<PeerMetadata> peers;
std::vector<AddressAndStatus> peers;
uint64_t version;
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
@@ -128,27 +121,6 @@ struct ShardToInitialize {
std::unordered_map<uint64_t, std::string> id_to_names;
};
struct ShardToSplit {
boost::uuids::uuid shard_to_split_uuid;
boost::uuids::uuid new_right_side_uuid;
Hlc split_requested_at;
LabelId label_id;
PrimaryKey split_key;
std::vector<SchemaProperty> schema;
Config config;
std::unordered_map<uint64_t, std::string> id_to_names;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
std::vector<ShardToSplit> shards_to_split;
};
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
struct LabelSpace {
@@ -156,6 +128,9 @@ struct LabelSpace {
// Maps between the smallest primary key stored in the shard and the shard
std::map<PrimaryKey, ShardMetadata> shards;
size_t replication_factor;
// TODO
// Stub value. Should be replaced once the shard-split logic is in place.
int64_t split_threshold{10000};
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
using utils::print_helpers::operator<<;
@@ -196,7 +171,7 @@ struct ShardMap {
std::unordered_map<uint64_t, std::string> IdToNames();
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
HeartbeatResponse AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ constexpr char kId[] = "ID";
namespace MG_INJECTED_NAMESPACE_NAME {
namespace detail {
using antlropencypher::v2::MemgraphCypher;
using antlropencypher::MemgraphCypher;
template <typename TVisitor>
std::optional<std::pair<Expression *, size_t>> VisitMemoryLimit(MemgraphCypher::MemoryLimitContext *memory_limit_ctx,
@@ -211,13 +211,13 @@ inline std::string_view ToString(const PulsarConfigKey key) {
}
} // namespace detail
using antlropencypher::v2::MemgraphCypher;
using antlropencypher::MemgraphCypher;
struct ParsingContext {
bool is_query_cached = false;
};
class CypherMainVisitor : public antlropencypher::v2::MemgraphCypherBaseVisitor {
class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
public:
explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -47,12 +47,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TypedValue Visit(NamedExpression &named_expression) override {
const auto &symbol = symbol_table_->at(named_expression);
auto value = named_expression.expression_->Accept(*this);
frame_->At(symbol) = value;
frame_->at(symbol) = value;
return value;
}
TypedValue Visit(Identifier &ident) override {
return TypedValue(frame_->At(symbol_table_->at(ident)), ctx_->memory);
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
@@ -470,7 +470,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->At(symbol_table_->at(aggregation)), ctx_->memory);
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
TypedValue Visit(Coalesce &coalesce) override {
@@ -528,8 +528,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
auto accumulator = reduce.initializer_->Accept(*this);
for (const auto &element : list) {
frame_->At(accumulator_symbol) = accumulator;
frame_->At(element_symbol) = element;
frame_->at(accumulator_symbol) = accumulator;
frame_->at(element_symbol) = element;
accumulator = reduce.expression_->Accept(*this);
}
return accumulator;
@@ -551,7 +551,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
if (element.IsNull()) {
result.emplace_back();
} else {
frame_->At(element_symbol) = element;
frame_->at(element_symbol) = element;
result.emplace_back(extract.expression_->Accept(*this));
}
}
@@ -571,7 +571,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
bool has_null_elements = false;
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = all.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
@@ -608,7 +608,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
bool has_value = false;
bool predicate_satisfied = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = single.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
@@ -645,7 +645,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &symbol = symbol_table_->at(*any.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = any.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
@@ -677,7 +677,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &symbol = symbol_table_->at(*none.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = none.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());

View File

@@ -30,18 +30,15 @@ class Frame {
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
TypedValue &At(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &At(const Symbol &symbol) const { return elems_.at(symbol.position()); }
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
uint64_t Id() const { return id_; }
void SetId(const uint64_t id) { id_ = id; }
const utils::pmr::vector<TypedValue> &Elems() const { return elems_; }
auto &elems() { return elems_; }
const auto &elems() const { return elems_; }
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
private:
uint64_t id_{0U};
utils::pmr::vector<TypedValue> elems_;
};

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -120,7 +120,7 @@ class Shared {
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!filled_, "Promise filled twice!");
item_ = std::move(item);
item_ = item;
filled_ = true;
} // lock released before condition variable notification
@@ -235,7 +235,7 @@ class Promise {
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(std::move(item));
shared_->Fill(item);
filled_or_moved_ = true;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,10 +30,10 @@ class LocalTransport {
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
: local_transport_handle_(std::move(local_transport_handle)) {}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
std::function<void()> fill_notifier, Duration timeout) {
return local_transport_handle_->template SubmitRequest<ResponseT, RequestT>(
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
to_address, from_address, std::move(request), timeout, fill_notifier);
}
@@ -43,8 +43,8 @@ class LocalTransport {
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return local_transport_handle_->Now(); }

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -104,10 +104,10 @@ class LocalTransportHandle {
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
auto type_info = TypeInfoFor(message);
std::any message_any(std::move(message));
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
@@ -138,14 +138,14 @@ class LocalTransportHandle {
cv_.notify_all();
}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
Duration timeout, std::function<void()> fill_notifier) {
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
// set null notifier for when the Future::Wait is called
nullptr,
// set notifier for when Promise::Fill is called
std::move(fill_notifier));
std::forward<std::function<void()>>(fill_notifier));
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
@@ -168,7 +168,7 @@ class LocalTransportHandle {
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send<RequestT>(to_address, from_address, request_id, std::move(request));
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
return std::move(future);
}

View File

@@ -13,7 +13,6 @@
#include <boost/core/demangle.hpp>
#include "io/time.hpp"
#include "io/transport.hpp"
#include "utils/type_info_ref.hpp"
@@ -39,7 +38,6 @@ struct OpaqueMessage {
uint64_t request_id;
std::any message;
utils::TypeInfoRef type_info;
Time deliverable_at;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,7 +19,6 @@
#include <map>
#include <set>
#include <thread>
#include <type_traits>
#include <vector>
#include <boost/core/demangle.hpp>
@@ -247,7 +246,7 @@ to a CAS operation.
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(std::move(r)) } -> std::same_as<ReadResponseValue>;
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
@@ -403,7 +402,7 @@ class Raft {
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
leader.pending_client_requests.erase(apply_index);
WriteResponse<WriteResponseValue> resp{
const WriteResponse<WriteResponseValue> resp{
.success = true,
.write_return = std::move(write_return),
.raft_index = apply_index,
@@ -555,7 +554,7 @@ class Raft {
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send(peer, request_id, VoteRequest{request});
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
@@ -625,12 +624,13 @@ class Raft {
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
io_.Send(from_address, request_id,
VoteResponse{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
});
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
@@ -718,10 +718,6 @@ class Raft {
.log_size = state_.log.size(),
};
static_assert(std::is_trivially_copyable_v<AppendResponse>,
"This function copies this message, therefore it is important to be trivially copyable. Otherwise it "
"should be moved");
if constexpr (std::is_same<ALL, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
@@ -740,7 +736,7 @@ class Raft {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
@@ -751,7 +747,7 @@ class Raft {
if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -812,7 +808,7 @@ class Raft {
Log("returning log_size of ", res.log_size);
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -863,17 +859,17 @@ class Raft {
auto type_info = TypeInfoFor(req);
std::string demangled_name = boost::core::demangle(type_info.get().name());
Log("handling ReadOperation<" + demangled_name + ">");
ReadOperation &read_operation = req.operation;
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(std::move(read_operation));
ReadResponseValue read_return = replicated_state_.Read(read_operation);
ReadResponse<ReadResponseValue> resp{
const ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, std::move(resp));
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
@@ -882,11 +878,11 @@ class Raft {
std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
ReadResponse<ReadResponseValue> res{
const ReadResponse<ReadResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
Cron();
@@ -898,12 +894,12 @@ class Raft {
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
ReadResponse<ReadResponseValue> res{
const ReadResponse<ReadResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -917,12 +913,12 @@ class Raft {
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
WriteResponse<WriteResponseValue> res{
const WriteResponse<WriteResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -931,11 +927,11 @@ class Raft {
RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
WriteResponse<WriteResponseValue> res{
const WriteResponse<WriteResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
Cron();

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -95,7 +95,7 @@ class RsmClient {
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncWriteRequest(std::move(req), notifier, readiness_token);
SendAsyncWriteRequest(req, notifier, readiness_token);
auto poll_result = AwaitAsyncWriteRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncWriteRequest(readiness_token);
@@ -106,7 +106,7 @@ class RsmClient {
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncReadRequest(std::move(req), notifier, readiness_token);
SendAsyncReadRequest(req, notifier, readiness_token);
auto poll_result = AwaitAsyncReadRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncReadRequest(readiness_token);
@@ -115,15 +115,15 @@ class RsmClient {
}
/// AsyncRead methods
void SendAsyncReadRequest(ReadRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
void SendAsyncReadRequest(const ReadRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
ReadRequest<ReadRequestT> read_req = {.operation = req};
AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
leader_, std::move(read_req), notifier, readiness_token),
.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
leader_, read_req, notifier, readiness_token),
};
async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
@@ -134,8 +134,8 @@ class RsmClient {
ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
async_request.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
leader_, std::move(read_req), async_request.notifier, readiness_token);
async_request.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
leader_, read_req, async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
@@ -184,15 +184,15 @@ class RsmClient {
}
/// AsyncWrite methods
void SendAsyncWriteRequest(WriteRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
void SendAsyncWriteRequest(const WriteRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
WriteRequest<WriteRequestT> write_req = {.operation = req};
AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
leader_, std::move(write_req), notifier, readiness_token),
.future = io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
leader_, write_req, notifier, readiness_token),
};
async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
@@ -204,8 +204,8 @@ class RsmClient {
WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
async_request.future =
io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
leader_, std::move(write_req), async_request.notifier, readiness_token);
io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
leader_, write_req, async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {

View File

@@ -26,6 +26,5 @@ struct SimulatorConfig {
uint64_t rng_seed = 0;
Time start_time = Time::min();
Time abort_time = Time::max();
Duration message_delay = std::chrono::microseconds(100);
};
}; // namespace memgraph::io::simulator

View File

@@ -175,8 +175,8 @@ bool SimulatorHandle::MaybeTickSimulator() {
spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
opaque_message.to_address.last_known_port);
const auto &[om_vec, inserted] =
can_receive_.try_emplace(to_address.ToPartialAddress(), std::deque<OpaqueMessage>());
om_vec->second.emplace_front(std::move(opaque_message));
can_receive_.try_emplace(to_address.ToPartialAddress(), std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
}
return true;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -46,7 +46,7 @@ class SimulatorHandle {
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::map<PartialAddress, std::deque<OpaqueMessage>> can_receive_;
std::map<PartialAddress, std::vector<OpaqueMessage>> can_receive_;
Time cluster_wide_time_microseconds_;
bool should_shut_down_ = false;
@@ -105,19 +105,19 @@ class SimulatorHandle {
bool ShouldShutDown() const;
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
Duration timeout, std::function<bool()> &&maybe_tick_simulator,
std::function<void()> &&fill_notifier) {
template <Message Request, Message Response>
ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
std::function<bool()> &&maybe_tick_simulator,
std::function<void()> &&fill_notifier) {
auto type_info = TypeInfoFor(request);
std::string demangled_name = boost::core::demangle(type_info.get().name());
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<Response>>(
// set notifier for when the Future::Wait is called
std::move(maybe_tick_simulator),
std::forward<std::function<bool()>>(maybe_tick_simulator),
// set notifier for when Promise::Fill is called
std::move(fill_notifier));
std::forward<std::function<void()>>(fill_notifier));
{
std::unique_lock<std::mutex> lock(mu_);
@@ -126,13 +126,12 @@ class SimulatorHandle {
const Time deadline = cluster_wide_time_microseconds_ + timeout;
std::any message(std::move(request));
std::any message(request);
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message),
.type_info = type_info,
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
.type_info = type_info};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
@@ -166,12 +165,8 @@ class SimulatorHandle {
while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
if (can_receive_.contains(partial_address)) {
std::deque<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
bool contains_items = !can_rx.empty();
bool can_receive = contains_items && can_rx.back().deliverable_at <= cluster_wide_time_microseconds_;
if (can_receive) {
std::vector<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
if (!can_rx.empty()) {
OpaqueMessage message = std::move(can_rx.back());
can_rx.pop_back();
@@ -182,12 +177,6 @@ class SimulatorHandle {
return std::move(m_opt).value();
}
if (contains_items) {
auto count = can_rx.back().deliverable_at.time_since_epoch().count();
auto now_count = cluster_wide_time_microseconds_.time_since_epoch().count();
spdlog::trace("can't receive message yet due to artificial latency. deliverable_at: {}, now: {}", count,
now_count);
}
}
if (!should_shut_down_) {
@@ -205,7 +194,7 @@ class SimulatorHandle {
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
auto type_info = TypeInfoFor(message);
{
@@ -215,8 +204,7 @@ class SimulatorHandle {
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any),
.type_info = type_info,
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
.type_info = type_info};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
stats_.total_messages++;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,14 +33,14 @@ class SimulatorTransport {
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
std::function<void()> notification, Duration timeout) {
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
return handle_copy->MaybeTickSimulator();
};
return simulator_handle_->template SubmitRequest<ResponseT, RequestT>(
return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
}
@@ -50,8 +50,8 @@ class SimulatorTransport {
}
template <Message M>
void Send(Address to_address, Address from_address, uint64_t request_id, RValueRef<M> message) {
return simulator_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
}
Time Now() const { return simulator_handle_->Now(); }

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,6 @@
#include "io/message_histogram_collector.hpp"
#include "io/notifier.hpp"
#include "io/time.hpp"
#include "utils/concepts.hpp"
#include "utils/result.hpp"
namespace memgraph::io {
@@ -33,15 +32,7 @@ using memgraph::utils::BasicResult;
// reasonable constraints around message types over time,
// as we adapt things to use Thrift-generated message types.
template <typename T>
concept Message = std::movable<T> && std::copyable<T>;
template <utils::Object T>
struct RValueRefEnforcer {
using Type = T &&;
};
template <typename T>
using RValueRef = typename RValueRefEnforcer<T>::Type;
concept Message = std::same_as<T, std::decay_t<T>>;
using RequestId = uint64_t;
@@ -91,44 +82,44 @@ class Io {
Duration GetDefaultTimeout() { return default_timeout_; }
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RValueRef<RequestT> request, Duration timeout) {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<ResponseT, RequestT>(address, from_address, std::move(request),
fill_notifier, timeout);
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, fill_notifier,
timeout);
}
/// Issue a request that times out after the default timeout. This tends
/// to be used by clients.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, RValueRef<RequestT> request) {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RValueRef<RequestT> request, Notifier notifier,
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
ReadinessToken readiness_token) {
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT &&request, Notifier notifier,
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
ReadinessToken readiness_token, Duration timeout) {
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<ResponseT>(to_address, from_address, std::forward<RequestT>(request),
fill_notifier, timeout);
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Wait for an explicit number of microseconds for a request of one of the
@@ -150,9 +141,9 @@ class Io {
/// responses are not necessarily expected, and for servers to respond to requests.
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
template <Message M>
void Send(Address to_address, RequestId request_id, M &&message) {
void Send(Address to_address, RequestId request_id, M message) {
Address from_address = address_;
return implementation_.template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
}
/// The current system time. This time source should be preferred over any other,

View File

@@ -23,7 +23,7 @@ add_custom_command(
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher::v2
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,10 +14,10 @@
#include <string>
#include "antlr4-runtime.h"
#include "utils/exceptions.hpp"
#include "parser/opencypher/generated/MemgraphCypher.h"
#include "parser/opencypher/generated/MemgraphCypherLexer.h"
#include "utils/concepts.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::frontend::opencypher {
@@ -32,9 +32,11 @@ class SyntaxException : public utils::BasicException {
* This thing must me a class since parser.cypher() returns pointer and there is
* no way for us to get ownership over the object.
*/
enum class ParserOpTag : uint8_t { CYPHER, EXPRESSION };
enum class ParserOpTag : uint8_t {
CYPHER, EXPRESSION
};
template <ParserOpTag Tag = ParserOpTag::CYPHER>
template<ParserOpTag Tag = ParserOpTag::CYPHER>
class Parser {
public:
/**
@@ -44,9 +46,10 @@ class Parser {
Parser(const std::string query) : query_(std::move(query)) {
parser_.removeErrorListeners();
parser_.addErrorListener(&error_listener_);
if constexpr (Tag == ParserOpTag::CYPHER) {
if constexpr(Tag == ParserOpTag::CYPHER) {
tree_ = parser_.cypher();
} else {
}
else {
tree_ = parser_.expression();
}
if (parser_.getNumberOfSyntaxErrors()) {
@@ -72,11 +75,11 @@ class Parser {
FirstMessageErrorListener error_listener_;
std::string query_;
antlr4::ANTLRInputStream input_{query_};
antlropencypher::v2::MemgraphCypherLexer lexer_{&input_};
antlropencypher::MemgraphCypherLexer lexer_{&input_};
antlr4::CommonTokenStream tokens_{&lexer_};
// generate ast
antlropencypher::v2::MemgraphCypher parser_{&tokens_};
antlropencypher::MemgraphCypher parser_{&tokens_};
antlr4::tree::ParseTree *tree_ = nullptr;
};
} // namespace memgraph::frontend::opencypher

View File

@@ -48,20 +48,18 @@ add_dependencies(mg-query generate_lcp_query)
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query dl cppitertools Boost::headers)
target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
endif()
target_link_libraries(mg-query Python3::Python)
# Generate Antlr openCypher parser
set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
set(opencypher_generated ${opencypher_frontend}/generated)
set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
@@ -84,15 +82,15 @@ add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
DEPENDS
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
add_custom_target(generate_opencypher_parser
DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,9 +14,9 @@
#include "query/v2/request_router.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_v2_cost_planner, true, "Use the cost-estimating query planner.");
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_v2_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace memgraph::query::v2 {
@@ -123,7 +123,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
auto vertex_counts = plan::MakeVertexCountCache(request_router);
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_v2_cost_planner);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
std::move(symbol_table));
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,9 +22,9 @@
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_v2_cost_planner);
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_v2_plan_cache_ttl);
DECLARE_int32(query_plan_cache_ttl);
namespace memgraph::query::v2 {
@@ -58,7 +58,7 @@ class CachedPlan {
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_v2_plan_cache_ttl);
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:

View File

@@ -338,31 +338,18 @@ bool Once::OnceCursor::Pull(Frame &, ExecutionContext &context) {
return false;
}
bool Once::OnceCursor::PullMultiple(MultiFrame &output_multi_frame, ExecutionContext &context) {
bool Once::OnceCursor::PullMultiple(MultiFrame &multi_frame, ExecutionContext &context) {
SCOPED_PROFILE_OP("OnceMF");
if (!did_pull_) {
auto &first_frame = multi_frame.GetFirstFrame();
first_frame.MakeValid();
did_pull_ = true;
if (pushed_down_multi_frame_.has_value()) {
auto pushed_down_consumer = pushed_down_multi_frame_->GetValidFramesConsumer();
auto output_populator = output_multi_frame.GetInvalidFramesPopulator();
auto consumer_it = pushed_down_consumer.begin();
auto populator_it = output_populator.begin();
for (; consumer_it != pushed_down_consumer.end(); ++consumer_it, ++populator_it) {
MG_ASSERT(populator_it != output_populator.end());
*populator_it = std::move(*consumer_it);
}
} else {
auto &first_frame = output_multi_frame.GetFirstFrame();
first_frame.MakeValid();
}
return true;
}
return false;
}
void Once::OnceCursor::PushDown(const MultiFrame &multi_frame) { pushed_down_multi_frame_.emplace(multi_frame); }
UniqueCursorPtr Once::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::OnceOperator);
@@ -521,7 +508,7 @@ class DistributedScanAllAndFilterCursor : public Cursor {
SCOPED_PROFILE_OP(op_name_);
if (!own_multi_frame_.has_value()) {
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
own_frames_it_ = own_frames_consumer_->begin();
@@ -586,8 +573,6 @@ class DistributedScanAllAndFilterCursor : public Cursor {
return populated_any;
};
void PushDown(const MultiFrame &multi_frame) override { input_cursor_->PushDown(multi_frame); }
void Shutdown() override { input_cursor_->Shutdown(); }
void ResetExecutionState() {
@@ -719,12 +704,12 @@ class DistributedScanByPrimaryKeyCursor : public Cursor {
void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
if (!own_multi_frame_.has_value()) {
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
own_frames_it_ = own_frames_consumer_->begin();
}
MG_ASSERT(output_multi_frame.GetFirstFrame().Elems().size() == own_multi_frame_->GetFirstFrame().Elems().size());
MG_ASSERT(output_multi_frame.GetFirstFrame().elems().size() == own_multi_frame_->GetFirstFrame().elems().size());
}
bool PullMultiple(MultiFrame &output_multi_frame, ExecutionContext &context) override {
@@ -2119,8 +2104,6 @@ bool Optional::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
return visitor.PostVisit(*this);
}
class OptionalCursor;
UniqueCursorPtr Optional::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::OptionalOperator);
@@ -2134,31 +2117,29 @@ std::vector<Symbol> Optional::ModifiedSymbols(const SymbolTable &table) const {
return symbols;
}
class OptionalCursor : public Cursor {
public:
OptionalCursor(const Optional &self, utils::MemoryResource *mem)
: self_(self), input_cursor_(self.input_->MakeCursor(mem)), optional_cursor_(self.optional_->MakeCursor(mem)) {}
Optional::OptionalCursor::OptionalCursor(const Optional &self, utils::MemoryResource *mem)
: self_(self), input_cursor_(self.input_->MakeCursor(mem)), optional_cursor_(self.optional_->MakeCursor(mem)) {}
bool Pull(Frame &frame, ExecutionContext &context) override {
SCOPED_PROFILE_OP("Optional");
bool Optional::OptionalCursor::Pull(Frame &frame, ExecutionContext &context) {
SCOPED_PROFILE_OP("Optional");
while (true) {
if (pull_input_) {
if (input_cursor_->Pull(frame, context)) {
// after a successful pull from the input
// reset optional_ (it's expand iterators maintain state)
optional_cursor_->Reset();
} else
// input is exhausted, we're done
return false;
}
while (true) {
if (pull_input_) {
if (input_cursor_->Pull(frame, context)) {
// after a successful input from the input
// reset optional_ (it's expand iterators maintain state)
optional_cursor_->Reset();
} else
// input is exhausted, we're done
return false;
}
// pull from the optional_ cursor
if (optional_cursor_->Pull(frame, context)) {
// if successful, next Pull from this should not pull_input_
pull_input_ = false;
return true;
}
// pull from the optional_ cursor
if (optional_cursor_->Pull(frame, context)) {
// if successful, next Pull from this should not pull_input_
pull_input_ = false;
return true;
} else {
// failed to Pull from the merge_match cursor
if (pull_input_) {
// if we have just now pulled from the input
@@ -2172,180 +2153,21 @@ class OptionalCursor : public Cursor {
// we have exhausted optional_cursor_ after 1 or more successful Pulls
// attempt next input_cursor_ pull
pull_input_ = true;
continue;
}
}
}
bool HandleReadyInput(InvalidFramesPopulator &output_populator, InvalidFramesPopulator::Iterator &output_frames_it,
ExecutionContext &context) {
bool populated_any = false;
while (true) {
switch (optional_state_) {
case State::Pull: {
if (!optional_cursor_->PullMultiple(*optional_multi_frame_, context)) {
optional_state_ = State::Exhausted;
optional_frames_consumer_.reset();
optional_frames_it_ = {};
if (populated_any) {
++own_frames_it_;
}
} else {
optional_frames_consumer_ = optional_multi_frame_->GetValidFramesConsumer();
optional_frames_it_ = optional_frames_consumer_->begin();
optional_state_ = State::Ready;
}
break;
}
case State::Ready: {
while (optional_frames_it_ != optional_frames_consumer_->end()) {
if (output_frames_it == output_populator.end()) {
return populated_any;
}
populated_any = true;
if (optional_frames_it_->Id() == own_frames_it_->Id()) {
// This might be a move, but then in we have to have special logic is EnsureOwnMultiFramesAreGood
*output_frames_it = *optional_frames_it_;
last_matched_frame_ = optional_frames_it_->Id();
optional_frames_it_->MakeInvalid();
++optional_frames_it_;
++output_frames_it;
if (optional_frames_it_ == optional_frames_consumer_->end()) {
optional_state_ = State::Pull;
}
} else if (last_matched_frame_ == own_frames_it_->Id()) {
++own_frames_it_;
} else {
// TODO(antaljanosbenjamin): Remove (or improve the message of) this assert
MG_ASSERT(optional_frames_it_->Id() > own_frames_it_->Id(), "This should be the case DELETE ME");
for (const auto &symbol : self_.optional_symbols_) {
spdlog::error("{}", symbol.name());
(*own_frames_it_)[symbol] = TypedValue(context.evaluation_context.memory);
}
// This might be a move, but then in we have to have special logic is EnsureOwnMultiFramesAreGood
*output_frames_it = *own_frames_it_;
last_matched_frame_ = own_frames_it_->Id();
own_frames_it_->MakeInvalid();
++own_frames_it_;
}
}
break;
}
case State::Exhausted: {
while (own_frames_it_ != own_frames_consumer_->end() && output_frames_it != output_populator.end()) {
MG_ASSERT(!optional_frames_consumer_.has_value(), "This should be the case DELETE ME");
for (const auto &symbol : self_.optional_symbols_) {
spdlog::error("{}", symbol.name());
(*own_frames_it_)[symbol] = TypedValue(context.evaluation_context.memory);
}
// This might be a move, but then in we have to have special logic is EnsureOwnMultiFramesAreGood
*output_frames_it = *own_frames_it_;
++own_frames_it_;
void Optional::OptionalCursor::Shutdown() {
input_cursor_->Shutdown();
optional_cursor_->Shutdown();
}
populated_any = true;
own_frames_it_->MakeInvalid();
++output_frames_it;
}
return populated_any;
}
}
}
return populated_any;
}
bool PullMultiple(MultiFrame &output_multi_frame, ExecutionContext &context) override {
SCOPED_PROFILE_OP("OptionalMF");
EnsureOwnMultiFramesAreGood(output_multi_frame);
auto populated_any{false};
auto output_frames_populator = output_multi_frame.GetInvalidFramesPopulator();
auto output_frames_it = output_frames_populator.begin();
while (true) {
switch (input_state_) {
case State::Pull: {
MG_ASSERT(optional_state_ != State::Ready, "Unexpected state");
if (!input_cursor_->PullMultiple(*own_multi_frame_, context)) {
input_state_ = State::Exhausted;
optional_state_ = State::Exhausted;
} else {
input_state_ = State::Ready;
optional_state_ = State::Pull;
uint64_t frame_id{0U};
for (auto &frame : own_multi_frame_->GetValidFramesModifier()) {
frame.SetId(frame_id++);
}
last_matched_frame_ = 0U;
optional_cursor_->Reset();
optional_cursor_->PushDown(*own_multi_frame_);
own_frames_consumer_ = own_multi_frame_->GetValidFramesConsumer();
own_frames_it_ = own_frames_consumer_->begin();
}
break;
}
case State::Ready: {
populated_any |= HandleReadyInput(output_frames_populator, output_frames_it, context);
if (output_frames_it == output_frames_populator.end()) {
return populated_any;
}
if (own_frames_it_ == own_frames_consumer_->end()) {
input_state_ = State::Pull;
}
break;
}
case State::Exhausted: {
MG_ASSERT(optional_state_ == State::Exhausted);
return populated_any;
}
}
}
}
void Shutdown() override {
input_cursor_->Shutdown();
optional_cursor_->Shutdown();
}
void Reset() override {
// TODO(antaljanosbenjamin)
input_cursor_->Reset();
optional_cursor_->Reset();
pull_input_ = true;
}
private:
enum class State { Pull, Ready, Exhausted };
void EnsureOwnMultiFramesAreGood(MultiFrame &output_multi_frame) {
if (!own_multi_frame_.has_value()) {
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
own_frames_it_ = own_frames_consumer_->begin();
optional_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
}
MG_ASSERT(output_multi_frame.GetFirstFrame().Elems().size() == own_multi_frame_->GetFirstFrame().Elems().size());
}
const Optional &self_;
const UniqueCursorPtr input_cursor_;
const UniqueCursorPtr optional_cursor_;
State input_state_{State::Pull};
State optional_state_{State::Pull};
std::optional<MultiFrame> own_multi_frame_;
std::optional<ValidFramesConsumer> own_frames_consumer_;
ValidFramesConsumer::Iterator own_frames_it_;
std::optional<MultiFrame> optional_multi_frame_;
std::optional<ValidFramesConsumer> optional_frames_consumer_;
ValidFramesConsumer::Iterator optional_frames_it_;
uint64_t last_matched_frame_{0U};
// indicates if the next Pull from this cursor should
// perform a Pull from the input_cursor_
// this is true when:
// - first pulling from this Cursor
// - previous Pull from this cursor exhausted the optional_cursor_
bool pull_input_{true};
};
void Optional::OptionalCursor::Reset() {
input_cursor_->Reset();
optional_cursor_->Reset();
pull_input_ = true;
}
Unwind::Unwind(const std::shared_ptr<LogicalOperator> &input, Expression *input_expression, Symbol output_symbol)
: input_(input ? input : std::make_shared<Once>()),
@@ -2390,7 +2212,7 @@ class UnwindCursor : public Cursor {
SCOPED_PROFILE_OP("UnwindMF");
if (!own_multi_frame_.has_value()) {
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
own_frames_it_ = own_frames_consumer_->begin();
@@ -2655,7 +2477,7 @@ class CartesianCursor : public Cursor {
if (!cartesian_pull_initialized_) {
// Pull all left_op frames.
while (left_op_cursor_->Pull(frame, context)) {
left_op_frames_.emplace_back(frame.Elems().begin(), frame.Elems().end());
left_op_frames_.emplace_back(frame.elems().begin(), frame.elems().end());
}
// We're setting the iterator to 'end' here so it pulls the right
@@ -2679,7 +2501,7 @@ class CartesianCursor : public Cursor {
// Advance right_op_cursor_.
if (!right_op_cursor_->Pull(frame, context)) return false;
right_op_frame_.assign(frame.Elems().begin(), frame.Elems().end());
right_op_frame_.assign(frame.elems().begin(), frame.elems().end());
left_op_frames_it_ = left_op_frames_.begin();
} else {
// Make sure right_op_cursor last pulled results are on frame.
@@ -3265,18 +3087,25 @@ class DistributedExpandCursor : public Cursor {
MG_ASSERT(direction != EdgeAtom::Direction::BOTH);
const auto &edge = frame[self_.common_.edge_symbol].ValueEdge();
static constexpr auto get_dst_vertex = [](const EdgeAccessor &edge,
const EdgeAtom::Direction direction) -> accessors::VertexAccessor {
const EdgeAtom::Direction direction) -> msgs::VertexId {
switch (direction) {
case EdgeAtom::Direction::IN:
return edge.From();
return edge.From().Id();
case EdgeAtom::Direction::OUT:
return edge.To();
return edge.To().Id();
case EdgeAtom::Direction::BOTH:
throw std::runtime_error("EdgeDirection Both not implemented");
}
};
frame[self_.common_.node_symbol] = get_dst_vertex(edge, direction);
msgs::GetPropertiesRequest request;
// to not fetch any properties of the edges
request.vertex_ids.push_back(get_dst_vertex(edge, direction));
auto result_rows = context.request_router->GetProperties(std::move(request));
MG_ASSERT(result_rows.size() == 1);
auto &result_row = result_rows.front();
frame[self_.common_.node_symbol] =
accessors::VertexAccessor(msgs::Vertex{result_row.vertex}, result_row.props, context.request_router);
}
bool InitEdges(Frame &frame, ExecutionContext &context) {
@@ -3300,8 +3129,6 @@ class DistributedExpandCursor : public Cursor {
// to not fetch any properties of the edges
request.edge_properties.emplace();
request.src_vertices.push_back(vertex.Id());
request.edge_properties.emplace();
request.src_vertex_properties.emplace();
auto result_rows = std::invoke([&context, &request]() mutable {
SCOPED_REQUEST_WAIT_PROFILE;
return context.request_router->ExpandOne(std::move(request));
@@ -3393,8 +3220,7 @@ class DistributedExpandCursor : public Cursor {
void InitEdgesMultiple() {
// This function won't work if any vertex id is duplicated in the input, because:
// 1. vertex_id_to_result_row is not a multimap
// 2. if self_.common_.existing_node is true, then we erase edges that might be necessary for the input
// vertex on a
// 2. if self_.common_.existing_node is true, then we erase edges that might be necessary for the input vertex on a
// later frame
const auto &frame = (*own_frames_it_);
const auto &vertex_value = frame[self_.input_symbol_];
@@ -3445,7 +3271,6 @@ class DistributedExpandCursor : public Cursor {
[](const storage::v3::EdgeTypeId edge_type_id) { return msgs::EdgeType{edge_type_id}; });
// to not fetch any properties of the edges
request.edge_properties.emplace();
request.src_vertex_properties.emplace();
for (const auto &frame : own_multi_frame_->GetValidFramesReader()) {
const auto &vertex_value = frame[self_.input_symbol_];
@@ -3554,16 +3379,14 @@ class DistributedExpandCursor : public Cursor {
return populated_any;
}
void PushDown(const MultiFrame &multi_frame) override { input_cursor_->PushDown(multi_frame); }
void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
if (!own_multi_frame_.has_value()) {
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().Elems().size(),
own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
own_frames_it_ = own_frames_consumer_->begin();
}
MG_ASSERT(output_multi_frame.GetFirstFrame().Elems().size() == own_multi_frame_->GetFirstFrame().Elems().size());
MG_ASSERT(output_multi_frame.GetFirstFrame().elems().size() == own_multi_frame_->GetFirstFrame().elems().size());
}
void Shutdown() override { input_cursor_->Shutdown(); }
@@ -3574,9 +3397,7 @@ class DistributedExpandCursor : public Cursor {
result_rows_.clear();
own_frames_it_ = ValidFramesConsumer::Iterator{};
own_frames_consumer_.reset();
if (own_multi_frame_.has_value()) {
own_multi_frame_->MakeAllFramesInvalid();
}
own_multi_frame_->MakeAllFramesInvalid();
state_ = State::PullInputAndEdges;
current_in_edges_.clear();

View File

@@ -88,8 +88,6 @@ class Cursor {
/// @throws QueryRuntimeException if something went wrong with execution
virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
virtual void PushDown(const MultiFrame&) { MG_ASSERT(false, "PushDownIsNotImplemented"); }
/// Resets the Cursor to its initial state.
virtual void Reset() = 0;
@@ -352,13 +350,11 @@ and false on every following Pull.")
public:
OnceCursor() {}
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void PushDown(const MultiFrame&) override;
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
private:
std::optional<MultiFrame> pushed_down_multi_frame_;
bool did_pull_{false};
};
cpp<#)
@@ -1971,6 +1967,27 @@ and returns true, once.")
input_ = input;
}
cpp<#)
(:private
#>cpp
class OptionalCursor : public Cursor {
public:
OptionalCursor(const Optional &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
private:
const Optional &self_;
const UniqueCursorPtr input_cursor_;
const UniqueCursorPtr optional_cursor_;
// indicates if the next Pull from this cursor should
// perform a Pull from the input_cursor_
// this is true when:
// - first pulling from this Cursor
// - previous Pull from this cursor exhausted the optional_cursor_
bool pull_input_{true};
};
cpp<#)
(:serialize (:slk))
(:clone))

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,8 +13,7 @@
#include "utils/flag_validation.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_HIDDEN_int64(query_v2_vertex_count_to_expand_existing, 10,
DEFINE_VALIDATED_HIDDEN_int64(query_vertex_count_to_expand_existing, 10,
"Maximum count of indexed vertices which provoke "
"indexed lookup and then expand to existing, instead of "
"a regular expand. Default is 10, to turn off use -1.",

View File

@@ -17,6 +17,7 @@
#pragma once
#include <algorithm>
#include <limits>
#include <memory>
#include <optional>
#include <unordered_map>
@@ -30,8 +31,7 @@
#include "query/v2/plan/preprocess.hpp"
#include "storage/v3/id_types.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int64(query_v2_vertex_count_to_expand_existing);
DECLARE_int64(query_vertex_count_to_expand_existing);
namespace memgraph::query::v2::plan {
@@ -101,7 +101,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
return true;
}
ScanAll dst_scan(expand.input(), expand.common_.node_symbol, expand.view_);
auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_v2_vertex_count_to_expand_existing);
auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing);
if (indexed_scan) {
expand.set_input(std::move(indexed_scan));
expand.common_.existing_node = true;
@@ -130,7 +130,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
// unconditionally creating an indexed scan.
indexed_scan = GenScanByIndex(dst_scan);
} else {
indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_v2_vertex_count_to_expand_existing);
indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing);
}
if (indexed_scan) {
expand.set_input(std::move(indexed_scan));
@@ -562,8 +562,12 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
// `max_vertex_count` controls, whether no operator should be created if the
// vertex count in the best index exceeds this number. In such a case,
// `nullptr` is returned and `input` is not chained.
std::unique_ptr<ScanAll> GenScanByIndex(const ScanAll &scan,
const std::optional<int64_t> &max_vertex_count = std::nullopt) {
// std::unique_ptr<ScanAll> GenScanByIndex(const ScanAll &scan, const std::optional<int64_t> &max_vertex_count =
// std::nullopt) {
std::unique_ptr<ScanAll> GenScanByIndex(const ScanAll &scan, std::optional<int64_t> max_vertex_count = std::nullopt) {
// debug (gvolfing)
max_vertex_count = std::numeric_limits<int64_t>::max();
const auto &input = scan.input();
const auto &node_symbol = scan.output_symbol_;
const auto &view = scan.view_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,8 +17,7 @@
#include "utils/flag_validation.hpp"
#include "utils/logging.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_HIDDEN_uint64(query_v2_max_plans, 1000U, "Maximum number of generated plans for a query.",
DEFINE_VALIDATED_HIDDEN_uint64(query_max_plans, 1000U, "Maximum number of generated plans for a query.",
FLAG_IN_RANGE(1, std::numeric_limits<std::uint64_t>::max()));
namespace memgraph::query::v2::plan::impl {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,8 +18,7 @@
#include "query/v2/plan/rule_based_planner.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint64(query_v2_max_plans);
DECLARE_uint64(query_max_plans);
namespace memgraph::query::v2::plan {
@@ -311,7 +310,7 @@ class VariableStartPlanner {
for (const auto &query_part : query_parts) {
alternative_query_parts.emplace_back(impl::VaryQueryPartMatching(query_part, symbol_table));
}
return iter::slice(MakeCartesianProduct(std::move(alternative_query_parts)), 0UL, FLAGS_query_v2_max_plans);
return iter::slice(MakeCartesianProduct(std::move(alternative_query_parts)), 0UL, FLAGS_query_max_plans);
}
public:

View File

@@ -38,12 +38,15 @@ class VertexCountCache {
auto NameToProperty(const std::string &name) { return request_router_->NameToProperty(name); }
auto NameToEdgeType(const std::string &name) { return request_router_->NameToEdgeType(name); }
int64_t VerticesCount() { return 1; }
int64_t VerticesCount() { return request_router_->GetApproximateVertexCount(); }
int64_t VerticesCount(storage::v3::LabelId /*label*/) { return 1; }
int64_t VerticesCount(storage::v3::LabelId label) { return request_router_->GetApproximateVertexCount(label); }
int64_t VerticesCount(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return 1; }
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) {
return request_router_->GetApproximateVertexCount(label, property);
}
// TODO(gvolfing) check if we actually use these overloads...
int64_t VerticesCount(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/,
const storage::v3::PropertyValue & /*value*/) {
return 1;

View File

@@ -122,6 +122,10 @@ class RequestRouterInterface {
virtual std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) = 0;
virtual void InstallSimulatorTicker(std::function<bool()> tick_simulator) = 0;
virtual const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const = 0;
virtual int64_t GetApproximateVertexCount() const = 0;
virtual int64_t GetApproximateVertexCount(storage::v3::LabelId label) const = 0;
virtual int64_t GetApproximateVertexCount(storage::v3::LabelId label, storage::v3::PropertyId property) const = 0;
};
// TODO(kostasrim)rename this class template
@@ -323,8 +327,8 @@ class RequestRouter : public RequestRouterInterface {
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::WriteRequests req = request.request;
storage_client.SendAsyncWriteRequest(std::move(req), notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), std::move(request));
storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
@@ -339,8 +343,7 @@ class RequestRouter : public RequestRouterInterface {
// must be fetched again with an ExpandOne(Edges.dst)
// create requests
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent =
RequestsForExpandOne(std::move(request));
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
// begin all requests in parallel
RunningRequests<msgs::ExpandOneRequest> running_requests = {};
@@ -350,8 +353,8 @@ class RequestRouter : public RequestRouterInterface {
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::ReadRequests req = request.request;
storage_client.SendAsyncReadRequest(std::move(req), notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), std::move(request));
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
@@ -387,8 +390,8 @@ class RequestRouter : public RequestRouterInterface {
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::ReadRequests req = request.request;
storage_client.SendAsyncReadRequest(std::move(req), notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), std::move(request));
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
@@ -416,6 +419,30 @@ class RequestRouter : public RequestRouterInterface {
return shards_map_.GetLabelId(name);
}
int64_t GetApproximateVertexCount() const override {
int64_t vertex_count = 0;
for (const auto &label_space : shards_map_.label_spaces) {
const auto split_threshold = label_space.second.split_threshold;
const auto shard_count = static_cast<int64_t>(label_space.second.shards.size());
vertex_count += split_threshold * shard_count;
}
return vertex_count;
}
int64_t GetApproximateVertexCount(storage::v3::LabelId label) const override {
const auto &label_space = shards_map_.label_spaces.at(label);
return label_space.split_threshold * label_space.shards.size();
}
int64_t GetApproximateVertexCount(storage::v3::LabelId label, storage::v3::PropertyId /*property*/) const override {
// TODO(gvolfing)
// Once we have reliable metadata to approximate the
// vertex count -based on properties- rework this function.
return GetApproximateVertexCount(label);
}
private:
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> RequestsForCreateVertices(
const std::vector<msgs::NewVertex> &new_vertices) {
@@ -504,7 +531,6 @@ class RequestRouter : public RequestRouterInterface {
msgs::ScanVerticesRequest request;
request.transaction_id = transaction_id_;
request.props_to_return.emplace();
request.start_id.second = storage::conversions::ConvertValueVector(key);
ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
@@ -519,7 +545,7 @@ class RequestRouter : public RequestRouterInterface {
return requests;
}
std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(msgs::ExpandOneRequest &&request) {
std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(const msgs::ExpandOneRequest &request) {
std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
msgs::ExpandOneRequest top_level_rqst_template = request;
top_level_rqst_template.transaction_id = transaction_id_;
@@ -531,7 +557,7 @@ class RequestRouter : public RequestRouterInterface {
if (!per_shard_request_table.contains(shard)) {
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
}
per_shard_request_table[shard].src_vertices.push_back(std::move(vertex));
per_shard_request_table[shard].src_vertices.push_back(vertex);
}
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests = {};
@@ -728,11 +754,11 @@ class RequestRouter : public RequestRouterInterface {
coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
ww.operation = std::move(requests);
auto resp = io_.template Request<io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>,
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>>(coordinator_address,
std::move(ww))
.Wait();
ww.operation = requests;
auto resp =
io_.template Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(coordinator_address, ww)
.Wait();
if (resp.HasValue()) {
const auto alloc_edge_id_reps =
std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);

View File

@@ -12,7 +12,6 @@
#pragma once
#include <chrono>
#include <cstdint>
#include <iostream>
#include <map>
#include <memory>
@@ -572,16 +571,6 @@ struct CommitResponse {
std::optional<ShardError> error;
};
struct SplitInfo {
PrimaryKey split_key;
uint64_t shard_version;
};
struct PerformSplitDataInfo {
PrimaryKey split_key;
uint64_t shard_version;
};
using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse>;

View File

@@ -18,7 +18,6 @@ set(storage_v3_src_files
bindings/typed_value.cpp
expr.cpp
vertex.cpp
splitter.cpp
request_helper.cpp)
# ######################

View File

@@ -30,10 +30,6 @@ struct Config {
io::Duration reclamation_interval{};
} gc;
struct Split {
uint64_t max_shard_vertex_size{500'000};
} split;
struct Items {
bool properties_on_edges{true};
} items;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,14 +13,12 @@
#include <cstdint>
#include <memory>
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
#include "utils/logging.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::storage::v3 {
@@ -29,11 +27,6 @@ struct Edge;
struct Delta;
struct CommitInfo;
inline uint64_t GetNextDeltaId() {
static utils::Synchronized<uint64_t, utils::SpinLock> delta_id{0};
return delta_id.WithLock([](auto &id) { return id++; });
}
// This class stores one of three pointers (`Delta`, `Vertex` and `Edge`)
// without using additional memory for storing the type. The type is stored in
// the pointer itself in the lower bits. All of those structures contain large
@@ -165,54 +158,46 @@ struct Delta {
struct RemoveInEdgeTag {};
struct RemoveOutEdgeTag {};
Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
: action(Action::DELETE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
Delta(DeleteObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
: action(Action::DELETE_OBJECT), commit_info(commit_info), command_id(command_id) {}
Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
: action(Action::RECREATE_OBJECT), id(delta_id), commit_info(commit_info), command_id(command_id) {}
Delta(RecreateObjectTag /*unused*/, CommitInfo *commit_info, uint64_t command_id)
: action(Action::RECREATE_OBJECT), commit_info(commit_info), command_id(command_id) {}
Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
: action(Action::ADD_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
Delta(AddLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
: action(Action::ADD_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t delta_id, uint64_t command_id)
: action(Action::REMOVE_LABEL), id(delta_id), commit_info(commit_info), command_id(command_id), label(label) {}
Delta(RemoveLabelTag /*unused*/, LabelId label, CommitInfo *commit_info, uint64_t command_id)
: action(Action::REMOVE_LABEL), commit_info(commit_info), command_id(command_id), label(label) {}
Delta(SetPropertyTag /*unused*/, PropertyId key, const PropertyValue &value, CommitInfo *commit_info,
uint64_t delta_id, uint64_t command_id)
: action(Action::SET_PROPERTY),
id(delta_id),
commit_info(commit_info),
command_id(command_id),
property({key, value}) {}
uint64_t command_id)
: action(Action::SET_PROPERTY), commit_info(commit_info), command_id(command_id), property({key, value}) {}
Delta(AddInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
uint64_t delta_id, uint64_t command_id)
uint64_t command_id)
: action(Action::ADD_IN_EDGE),
id(delta_id),
commit_info(commit_info),
command_id(command_id),
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(AddOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
uint64_t delta_id, uint64_t command_id)
uint64_t command_id)
: action(Action::ADD_OUT_EDGE),
id(delta_id),
commit_info(commit_info),
command_id(command_id),
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(RemoveInEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
uint64_t delta_id, uint64_t command_id)
uint64_t command_id)
: action(Action::REMOVE_IN_EDGE),
id(delta_id),
commit_info(commit_info),
command_id(command_id),
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
Delta(RemoveOutEdgeTag /*unused*/, EdgeTypeId edge_type, VertexId vertex_id, EdgeRef edge, CommitInfo *commit_info,
uint64_t delta_id, uint64_t command_id)
uint64_t command_id)
: action(Action::REMOVE_OUT_EDGE),
id(delta_id),
commit_info(commit_info),
command_id(command_id),
vertex_edge({edge_type, std::move(vertex_id), edge}) {}
@@ -241,10 +226,8 @@ struct Delta {
}
}
friend bool operator==(const Delta &lhs, const Delta &rhs) noexcept { return lhs.id == rhs.id; }
Action action;
uint64_t id;
// TODO: optimize with in-place copy
CommitInfo *commit_info;
uint64_t command_id;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -35,7 +35,7 @@ msgs::Value ConstructValueVertex(const VertexAccessor &acc, View view) {
memgraph::msgs::Label value_label{.id = prim_label};
auto prim_key = conversions::ConvertValueVector(acc.PrimaryKey(view).GetValue());
memgraph::msgs::VertexId vertex_id = std::make_pair(value_label, std::move(prim_key));
memgraph::msgs::VertexId vertex_id = std::make_pair(value_label, prim_key);
// Get the labels
auto vertex_labels = acc.Labels(view).GetValue();
@@ -45,7 +45,7 @@ msgs::Value ConstructValueVertex(const VertexAccessor &acc, View view) {
std::transform(vertex_labels.begin(), vertex_labels.end(), std::back_inserter(value_labels),
[](const auto &label) { return msgs::Label{.id = label}; });
return msgs::Value({.id = std::move(vertex_id), .labels = std::move(value_labels)});
return msgs::Value({.id = vertex_id, .labels = value_labels});
}
msgs::Value ConstructValueEdge(const EdgeAccessor &acc, View view) {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -325,7 +325,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
}
}
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -353,7 +353,7 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
}
}
LabelIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction,
LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: index_container_(&index_container),
label_(label),
@@ -465,7 +465,7 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
}
}
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, IndexContainer::iterator index_iterator)
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -526,7 +526,7 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
const PropertyValue kSmallestTemporalData =
PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
LabelPropertyIndex::Iterable::Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
Transaction *transaction, Indices *indices, Config::Items config,

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,8 +18,6 @@
#include <utility>
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex_accessor.hpp"
@@ -42,18 +40,12 @@ class LabelIndex {
bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
};
using IndexType = LabelId;
public:
using IndexContainer = std::set<Entry>;
using LabelIndexContainer = std::set<Entry>;
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator,
std::map<LabelId, IndexContainer> &data)
: index_{std::move(data)}, indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
/// @throw std::bad_alloc
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
@@ -71,12 +63,12 @@ class LabelIndex {
class Iterable {
public:
Iterable(IndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
Iterator(Iterable *self, IndexContainer::iterator index_iterator);
Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
VertexAccessor operator*() const { return current_vertex_accessor_; }
@@ -89,7 +81,7 @@ class LabelIndex {
void AdvanceUntilValid();
Iterable *self_;
IndexContainer::iterator index_iterator_;
LabelIndexContainer::iterator index_iterator_;
VertexAccessor current_vertex_accessor_;
Vertex *current_vertex_;
};
@@ -98,7 +90,7 @@ class LabelIndex {
Iterator end() { return {this, index_container_->end()}; }
private:
IndexContainer *index_container_;
LabelIndexContainer *index_container_;
LabelId label_;
View view_;
Transaction *transaction_;
@@ -122,29 +114,8 @@ class LabelIndex {
void Clear() { index_.clear(); }
std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
std::map<IndexType, IndexContainer> cloned_indices;
for (auto &[index_type_val, index] : index_) {
auto entry_it = index.begin();
auto &cloned_indices_container = cloned_indices[index_type_val];
while (entry_it != index.end()) {
// We need to save the next iterator since the current one will be
// invalidated after extract
auto next_entry_it = std::next(entry_it);
if (entry_it->vertex->first > split_key) {
[[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
cloned_indices_container.insert(index.extract(entry_it));
MG_ASSERT(inserted, "Failed to extract index entry!");
}
entry_it = next_entry_it;
}
}
return cloned_indices;
}
private:
std::map<LabelId, IndexContainer> index_;
std::map<LabelId, LabelIndexContainer> index_;
Indices *indices_;
Config::Items config_;
const VertexValidator *vertex_validator_;
@@ -162,10 +133,9 @@ class LabelPropertyIndex {
bool operator<(const PropertyValue &rhs) const;
bool operator==(const PropertyValue &rhs) const;
};
using IndexType = std::pair<LabelId, PropertyId>;
public:
using IndexContainer = std::set<Entry>;
using LabelPropertyIndexContainer = std::set<Entry>;
LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
@@ -189,14 +159,14 @@ class LabelPropertyIndex {
class Iterable {
public:
Iterable(IndexContainer &index_container, LabelId label, PropertyId property,
Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
Iterator(Iterable *self, IndexContainer::iterator index_iterator);
Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
VertexAccessor operator*() const { return current_vertex_accessor_; }
@@ -209,7 +179,7 @@ class LabelPropertyIndex {
void AdvanceUntilValid();
Iterable *self_;
IndexContainer::iterator index_iterator_;
LabelPropertyIndexContainer::iterator index_iterator_;
VertexAccessor current_vertex_accessor_;
Vertex *current_vertex_;
};
@@ -218,7 +188,7 @@ class LabelPropertyIndex {
Iterator end();
private:
IndexContainer *index_container_;
LabelPropertyIndexContainer *index_container_;
LabelId label_;
PropertyId property_;
std::optional<utils::Bound<PropertyValue>> lower_bound_;
@@ -259,29 +229,8 @@ class LabelPropertyIndex {
void Clear() { index_.clear(); }
std::map<IndexType, IndexContainer> SplitIndexEntries(const PrimaryKey &split_key) {
std::map<IndexType, IndexContainer> cloned_indices;
for (auto &[index_type_val, index] : index_) {
auto entry_it = index.begin();
auto &cloned_index_container = cloned_indices[index_type_val];
while (entry_it != index.end()) {
// We need to save the next iterator since the current one will be
// invalidated after extract
auto next_entry_it = std::next(entry_it);
if (entry_it->vertex->first > split_key) {
[[maybe_unused]] const auto &[inserted_entry_it, inserted, node] =
cloned_index_container.insert(index.extract(entry_it));
MG_ASSERT(inserted, "Failed to extract index entry!");
}
entry_it = next_entry_it;
}
}
return cloned_indices;
}
private:
std::map<std::pair<LabelId, PropertyId>, IndexContainer> index_;
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
Indices *indices_;
Config::Items config_;
const VertexValidator *vertex_validator_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -108,7 +108,7 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
/// a `DELETE_OBJECT` delta).
/// @throw std::bad_alloc
inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(), GetNextDeltaId(),
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_info.get(),
transaction->command_id);
}
@@ -119,7 +119,7 @@ template <typename TObj, class... Args>
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
GetNextDeltaId(), transaction->command_id);
transaction->command_id);
auto *delta_holder = GetDeltaHolder(object);
// The operations are written in such order so that both `next` and `prev`

View File

@@ -46,6 +46,7 @@ struct VertexIdCmpr {
std::optional<std::map<PropertyId, Value>> PrimaryKeysFromAccessor(const VertexAccessor &acc, View view,
const Schemas::Schema &schema) {
std::map<PropertyId, Value> ret;
auto props = acc.Properties(view);
auto maybe_pk = acc.PrimaryKey(view);
if (maybe_pk.HasError()) {
spdlog::debug("Encountered an error while trying to get vertex primary key.");
@@ -57,7 +58,7 @@ std::optional<std::map<PropertyId, Value>> PrimaryKeysFromAccessor(const VertexA
ret.emplace(schema.second[i].property_id, FromPropertyValueToValue(std::move(pk[i])));
}
return {std::move(ret)};
return ret;
}
ShardResult<std::vector<msgs::Label>> FillUpSourceVertexSecondaryLabels(const std::optional<VertexAccessor> &v_acc,
@@ -98,7 +99,7 @@ ShardResult<std::map<PropertyId, Value>> FillUpSourceVertexProperties(const std:
}
auto pks = PrimaryKeysFromAccessor(*v_acc, view, schema);
if (pks) {
src_vertex_properties.merge(std::move(*pks));
src_vertex_properties.merge(*pks);
}
} else if (req.src_vertex_properties.value().empty()) {
@@ -383,10 +384,13 @@ bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, con
}
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema) {
/// Fill up source vertex
const auto primary_key = ConvertPropertyVector(src_vertex.second);
auto v_acc = acc.FindVertex(primary_key, View::NEW);
msgs::Vertex source_vertex = {.id = src_vertex};
auto maybe_secondary_labels = FillUpSourceVertexSecondaryLabels(v_acc, req);
if (maybe_secondary_labels.HasError()) {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -233,7 +233,7 @@ ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesImpl(const TAccesso
[](std::pair<const PropertyId, PropertyValue> &pair) {
return std::make_pair(pair.first, conversions::FromPropertyValueToValue(std::move(pair.second)));
});
return {std::move(ret)};
return ret;
}
} // namespace impl
@@ -247,7 +247,7 @@ EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbo
EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req);
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,14 +18,14 @@
#include <memory>
#include <mutex>
#include <optional>
#include <variant>
#include <bits/ranges_algo.h>
#include <gflags/gflags.h>
#include <spdlog/spdlog.h>
#include "io/network/endpoint.hpp"
#include "io/time.hpp"
#include "storage/v3/delta.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
@@ -332,64 +332,16 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key,
vertex_validator_{schema_validator_, primary_label},
indices_{config.items, vertex_validator_},
isolation_level_{config.transaction.isolation_level},
config_{config} {
config_{config},
uuid_{utils::GenerateUUID()},
epoch_id_{utils::GenerateUUID()},
global_locker_{file_retainer_.AddLocker()} {
CreateSchema(primary_label_, schema);
StoreMapping(std::move(id_to_name));
}
Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
const std::unordered_map<uint64_t, std::string> &id_to_name, const uint64_t shard_version)
: primary_label_{primary_label},
min_primary_key_{min_primary_key},
max_primary_key_{max_primary_key},
vertices_(std::move(vertices)),
edges_(std::move(edges)),
shard_version_(shard_version),
schema_validator_{schemas_, name_id_mapper_},
vertex_validator_{schema_validator_, primary_label},
indices_{config.items, vertex_validator_},
isolation_level_{config.transaction.isolation_level},
config_{config},
start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)) {
CreateSchema(primary_label_, schema);
StoreMapping(id_to_name);
}
Shard::Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, VertexContainer &&vertices,
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
const std::unordered_map<uint64_t, std::string> &id_to_name, const uint64_t shard_version)
: primary_label_{primary_label},
min_primary_key_{min_primary_key},
max_primary_key_{max_primary_key},
vertices_(std::move(vertices)),
shard_version_(shard_version),
schema_validator_{schemas_, name_id_mapper_},
vertex_validator_{schema_validator_, primary_label},
indices_{config.items, vertex_validator_},
isolation_level_{config.transaction.isolation_level},
config_{config},
start_logical_id_to_transaction_(std::move(start_logical_id_to_transaction)) {
CreateSchema(primary_label_, schema);
StoreMapping(id_to_name);
}
Shard::~Shard() {}
std::unique_ptr<Shard> Shard::FromSplitData(SplitData &&split_data) {
if (split_data.config.items.properties_on_edges) [[likely]] {
return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
split_data.schema, std::move(split_data.vertices), std::move(*split_data.edges),
std::move(split_data.transactions), split_data.config, split_data.id_to_name,
split_data.shard_version);
}
return std::make_unique<Shard>(split_data.primary_label, split_data.min_primary_key, split_data.max_primary_key,
split_data.schema, std::move(split_data.vertices), std::move(split_data.transactions),
split_data.config, split_data.id_to_name, split_data.shard_version);
}
Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
: shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
@@ -484,7 +436,7 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
}
std::vector<EdgeAccessor> deleted_edges;
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)};
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)}; // TODO Replace
for (const auto &item : in_edges) {
auto [edge_type, from_vertex, edge] = item;
EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
@@ -1096,28 +1048,6 @@ void Shard::StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name) {
name_id_mapper_.StoreMapping(std::move(id_to_name));
}
std::optional<SplitInfo> Shard::ShouldSplit() const noexcept {
if (vertices_.size() > config_.split.max_shard_vertex_size) {
auto mid_elem = vertices_.begin();
// TODO(tyler) the first time we calculate the split point, we should store it so that we don't have to
// iterate over half of the entire index each time Cron is run until the split succeeds.
std::ranges::advance(mid_elem, static_cast<VertexContainer::difference_type>(vertices_.size() / 2));
return SplitInfo{mid_elem->first, shard_version_};
}
return std::nullopt;
}
SplitData Shard::PerformSplit(const PrimaryKey &split_key, const uint64_t shard_version) {
shard_version_ = shard_version;
const auto old_max_key = max_primary_key_;
max_primary_key_ = split_key;
const auto *schema = GetSchema(primary_label_);
MG_ASSERT(schema, "Shard must know about schema of primary label!");
Splitter shard_splitter(primary_label_, vertices_, edges_, start_logical_id_to_transaction_, indices_, config_,
schema->second, name_id_mapper_);
return shard_splitter.SplitShard(split_key, old_max_key, shard_version);
}
bool Shard::IsVertexBelongToShard(const VertexId &vertex_id) const {
return vertex_id.primary_label == primary_label_ && vertex_id.primary_key >= min_primary_key_ &&
(!max_primary_key_.has_value() || vertex_id.primary_key < *max_primary_key_);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,6 @@
#include <cstdint>
#include <filesystem>
#include <map>
#include <memory>
#include <numeric>
#include <optional>
#include <shared_mutex>
@@ -38,7 +37,6 @@
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/splitter.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
@@ -176,11 +174,6 @@ struct SchemasInfo {
Schemas::SchemasList schemas;
};
struct SplitInfo {
PrimaryKey split_point;
uint64_t shard_version;
};
/// Structure used to return information about the storage.
struct StorageInfo {
uint64_t vertex_count;
@@ -193,19 +186,9 @@ class Shard final {
public:
/// @throw std::system_error
/// @throw std::bad_alloc
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, Config config = Config(),
std::unordered_map<uint64_t, std::string> id_to_name = {});
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, VertexContainer &&vertices, EdgeContainer &&edges,
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
const std::unordered_map<uint64_t, std::string> &id_to_name, uint64_t shard_version);
Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, VertexContainer &&vertices,
std::map<uint64_t, std::unique_ptr<Transaction>> &&start_logical_id_to_transaction, const Config &config,
const std::unordered_map<uint64_t, std::string> &id_to_name, uint64_t shard_version);
explicit Shard(LabelId primary_label, PrimaryKey min_primary_key, std::optional<PrimaryKey> max_primary_key,
std::vector<SchemaProperty> schema, Config config = Config(),
std::unordered_map<uint64_t, std::string> id_to_name = {});
Shard(const Shard &) = delete;
Shard(Shard &&) noexcept = delete;
@@ -213,8 +196,6 @@ class Shard final {
Shard operator=(Shard &&) noexcept = delete;
~Shard();
static std::unique_ptr<Shard> FromSplitData(SplitData &&split_data);
class Accessor final {
private:
friend class Shard;
@@ -379,10 +360,6 @@ class Shard final {
void StoreMapping(std::unordered_map<uint64_t, std::string> id_to_name);
std::optional<SplitInfo> ShouldSplit() const noexcept;
SplitData PerformSplit(const PrimaryKey &split_key, uint64_t shard_version);
private:
Transaction &GetTransaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level);
@@ -400,7 +377,6 @@ class Shard final {
// list is used only when properties are enabled for edges. Because of that we
// keep a separate count of edges that is always updated.
uint64_t edge_count_{0};
uint64_t shard_version_{0};
SchemaValidator schema_validator_;
VertexValidator vertex_validator_;
@@ -420,6 +396,38 @@ class Shard final {
// storage.
std::list<Gid> deleted_edges_;
// UUID used to distinguish snapshots and to link snapshots to WALs
std::string uuid_;
// Sequence number used to keep track of the chain of WALs.
uint64_t wal_seq_num_{0};
// UUID to distinguish different main instance runs for replication process
// on SAME storage.
// Multiple instances can have same storage UUID and be MAIN at the same time.
// We cannot compare commit timestamps of those instances if one of them
// becomes the replica of the other so we use epoch_id_ as additional
// discriminating property.
// Example of this:
// We have 2 instances of the same storage, S1 and S2.
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
// At the moment when S1 commited a transaction with timestamp 20, and S2
// a different transaction with timestamp 15, we change S2's role to REPLICA
// and register it on S1.
// Without using the epoch_id, we don't know that S1 and S2 have completely
// different transactions, we think that the S2 is behind only by 5 commits.
std::string epoch_id_;
// History of the previous epoch ids.
// Each value consists of the epoch id along the last commit belonging to that
// epoch.
std::deque<std::pair<std::string, uint64_t>> epoch_history_;
uint64_t wal_unsynced_transactions_{0};
utils::FileRetainer file_retainer_;
// Global locker that is used for clients file locking
utils::FileRetainer::FileLocker global_locker_;
// Holds all of the (in progress, committed and aborted) transactions that are read or write to this shard, but
// haven't been cleaned up yet
std::map<uint64_t, std::unique_ptr<Transaction>> start_logical_id_to_transaction_{};

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -250,8 +250,8 @@ class ShardManager {
spdlog::info("SM sending heartbeat to coordinator {}", coordinator_leader_.ToString());
heartbeat_res_.emplace(std::move(
io_.template Request<WriteResponse<CoordinatorWriteResponses>, WriteRequest<CoordinatorWriteRequests>>(
coordinator_leader_, std::move(ww))));
io_.template Request<WriteRequest<CoordinatorWriteRequests>, WriteResponse<CoordinatorWriteResponses>>(
coordinator_leader_, ww)));
spdlog::info("SM sent heartbeat");
}

View File

@@ -472,8 +472,7 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ExpandOneRequest &&req) {
if (req.order_by_edges.empty()) {
const auto *schema = shard_->GetSchema(shard_->PrimaryLabel());
MG_ASSERT(schema);
return GetExpandOneResult(src_vertex_acc, std::move(src_vertex), req, maybe_filter_based_on_edge_uniqueness,
edge_filler, *schema);
return GetExpandOneResult(acc, src_vertex, req, maybe_filter_based_on_edge_uniqueness, edge_filler, *schema);
}
auto [in_edge_accessors, out_edge_accessors] = GetEdgesFromVertex(src_vertex_acc, req.direction);
const auto in_ordered_edges = OrderByEdges(dba, in_edge_accessors, req.order_by_edges, src_vertex_acc);
@@ -488,13 +487,12 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::ExpandOneRequest &&req) {
[](const auto &edge_element) { return edge_element.object_acc; });
const auto *schema = shard_->GetSchema(shard_->PrimaryLabel());
MG_ASSERT(schema);
return GetExpandOneResult(src_vertex_acc, std::move(src_vertex), req, std::move(in_edge_ordered_accessors),
std::move(out_edge_ordered_accessors), maybe_filter_based_on_edge_uniqueness,
edge_filler, *schema);
return GetExpandOneResult(src_vertex_acc, src_vertex, req, in_edge_ordered_accessors, out_edge_ordered_accessors,
maybe_filter_based_on_edge_uniqueness, edge_filler, *schema);
});
if (maybe_result.HasError()) {
shard_error.emplace(CreateErrorResponse(maybe_result.GetError(), req.transaction_id, "getting expand result"));
shard_error.emplace(CreateErrorResponse(primary_key.GetError(), req.transaction_id, "getting primary key"));
break;
}
@@ -583,12 +581,12 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest &&req) {
if (maybe_id.HasError()) {
return {maybe_id.GetError()};
}
auto &vertex_id = maybe_id.GetValue();
const auto &id = maybe_id.GetValue();
std::optional<msgs::EdgeId> e_id;
if (e_acc) {
e_id = msgs::EdgeId{e_acc->Gid().AsUint()};
}
msgs::VertexId v_id{msgs::Label{vertex_id.primary_label}, ConvertValueVector(std::move(vertex_id.primary_key))};
msgs::VertexId v_id{msgs::Label{id.primary_label}, ConvertValueVector(id.primary_key)};
auto maybe_props = collect_props(v_acc, e_acc);
if (maybe_props.HasError()) {
return {maybe_props.GetError()};

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,13 +12,11 @@
#pragma once
#include <memory>
#include <optional>
#include <variant>
#include <openssl/ec.h>
#include "query/v2/requests.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/value_conversions.hpp"
#include "storage/v3/vertex_accessor.hpp"
namespace memgraph::storage::v3 {
@@ -43,21 +41,8 @@ class ShardRsm {
public:
explicit ShardRsm(std::unique_ptr<Shard> &&shard) : shard_(std::move(shard)){};
std::optional<msgs::SplitInfo> ShouldSplit() const noexcept {
auto split_info = shard_->ShouldSplit();
if (split_info) {
return msgs::SplitInfo{conversions::ConvertValueVector(split_info->split_point), split_info->shard_version};
}
return std::nullopt;
}
std::unique_ptr<Shard> PerformSplit(msgs::PerformSplitDataInfo perform_split) const noexcept {
return Shard::FromSplitData(
shard_->PerformSplit(conversions::ConvertPropertyVector(perform_split.split_key), perform_split.shard_version));
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
msgs::ReadResponses Read(msgs::ReadRequests &&requests) {
msgs::ReadResponses Read(msgs::ReadRequests requests) {
return std::visit([&](auto &&request) mutable { return HandleRead(std::forward<decltype(request)>(request)); },
std::move(requests));
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -100,7 +100,7 @@ class Queue {
inner_->submitted++;
inner_->queue.emplace_back(std::move(message));
inner_->queue.emplace_back(std::forward<Message>(message));
} // lock dropped before notifying condition variable
inner_->cv.notify_all();

View File

@@ -1,411 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage/v3/splitter.hpp"
#include <algorithm>
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include "storage/v3/config.hpp"
#include "storage/v3/delta.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/logging.hpp"
namespace memgraph::storage::v3 {
Splitter::Splitter(const LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper)
: primary_label_(primary_label),
vertices_(vertices),
edges_(edges),
start_logical_id_to_transaction_(start_logical_id_to_transaction),
indices_(indices),
config_(config),
schema_(schema),
name_id_mapper_(name_id_mapper) {}
SplitData Splitter::SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
const uint64_t shard_version) {
SplitData data{.primary_label = primary_label_,
.min_primary_key = split_key,
.max_primary_key = max_primary_key,
.schema = schema_,
.config = config_,
.id_to_name = name_id_mapper_.GetIdToNameMap(),
.shard_version = shard_version};
std::set<uint64_t> collected_transactions_;
data.vertices = CollectVertices(data, collected_transactions_, split_key);
data.edges = CollectEdges(collected_transactions_, data.vertices, split_key);
data.transactions = CollectTransactions(collected_transactions_, *data.edges, split_key);
return data;
}
void Splitter::ScanDeltas(std::set<uint64_t> &collected_transactions_, const Delta *delta) {
while (delta != nullptr) {
collected_transactions_.insert(delta->commit_info->start_or_commit_timestamp.logical_id);
delta = delta->next;
}
}
VertexContainer Splitter::CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_,
const PrimaryKey &split_key) {
data.label_indices = indices_.label_index.SplitIndexEntries(split_key);
data.label_property_indices = indices_.label_property_index.SplitIndexEntries(split_key);
VertexContainer splitted_data;
auto split_key_it = vertices_.find(split_key);
while (split_key_it != vertices_.end()) {
// Go through deltas and pick up transactions start_id/commit_id
ScanDeltas(collected_transactions_, split_key_it->second.delta);
auto next_it = std::next(split_key_it);
const auto new_it = splitted_data.insert(splitted_data.end(), vertices_.extract(split_key_it));
MG_ASSERT(new_it != splitted_data.end(), "Failed to extract vertex!");
split_key_it = next_it;
}
return splitted_data;
}
std::optional<EdgeContainer> Splitter::CollectEdges(std::set<uint64_t> &collected_transactions_,
const VertexContainer &split_vertices,
const PrimaryKey &split_key) {
if (!config_.items.properties_on_edges) {
return std::nullopt;
}
EdgeContainer splitted_edges;
const auto split_vertex_edges = [&](const auto &edges_ref) {
// This is safe since if properties_on_edges is true, the this must be a ptr
for (const auto &edge_ref : edges_ref) {
auto *edge = std::get<2>(edge_ref).ptr;
const auto &other_vtx = std::get<1>(edge_ref);
ScanDeltas(collected_transactions_, edge->delta);
// Check if src and dest edge are both on splitted shard so we know if we
// should remove orphan edge, or make a clone
if (other_vtx.primary_key >= split_key) {
// Remove edge from shard
splitted_edges.insert(edges_.extract(edge->gid));
} else {
splitted_edges.insert({edge->gid, Edge{edge->gid, edge->delta}});
}
}
};
for (const auto &vertex : split_vertices) {
split_vertex_edges(vertex.second.in_edges);
split_vertex_edges(vertex.second.out_edges);
}
return splitted_edges;
}
std::map<uint64_t, std::unique_ptr<Transaction>> Splitter::CollectTransactions(
const std::set<uint64_t> &collected_transactions_, EdgeContainer &cloned_edges, const PrimaryKey &split_key) {
std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
for (const auto &[commit_start, transaction] : start_logical_id_to_transaction_) {
// We need all transaction whose deltas need to be resolved for any of the
// entities
if (collected_transactions_.contains(transaction->commit_info->start_or_commit_timestamp.logical_id)) {
transactions.insert({commit_start, start_logical_id_to_transaction_[commit_start]->Clone()});
}
}
// It is necessary to clone all the transactions first so we have new addresses
// for deltas, before doing alignment of deltas and prev_ptr
AdjustClonedTransactions(transactions, cloned_edges, split_key);
return transactions;
}
void EraseDeltaChain(auto &transaction, auto &transactions, auto &delta_head_it) {
auto *current_next_delta = delta_head_it->next;
// We need to keep track of delta_head_it in the delta list of current transaction
delta_head_it = transaction.deltas.erase(delta_head_it);
while (current_next_delta != nullptr) {
auto *next_delta = current_next_delta->next;
// Find next delta transaction delta list
auto current_transaction_it = std::ranges::find_if(
transactions, [&start_or_commit_timestamp =
current_next_delta->commit_info->start_or_commit_timestamp](const auto &transaction) {
return transaction.second->start_timestamp == start_or_commit_timestamp ||
transaction.second->commit_info->start_or_commit_timestamp == start_or_commit_timestamp;
});
MG_ASSERT(current_transaction_it != transactions.end(), "Error when pruning deltas!");
// Remove the delta
const auto delta_it =
std::ranges::find_if(current_transaction_it->second->deltas,
[current_next_delta](const auto &elem) { return elem.id == current_next_delta->id; });
if (delta_it != current_transaction_it->second->deltas.end()) {
// If the next delta is next in transaction list replace current_transaction_it
// with the next one
if (current_transaction_it->second->start_timestamp == transaction.start_timestamp &&
current_transaction_it == std::next(current_transaction_it)) {
delta_head_it = current_transaction_it->second->deltas.erase(delta_it);
} else {
current_transaction_it->second->deltas.erase(delta_it);
}
}
current_next_delta = next_delta;
}
}
void PruneDeltas(Transaction &cloned_transaction, std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
const PrimaryKey &split_key, EdgeContainer &cloned_edges) {
// Remove delta chains that don't point to objects on splitted shard
auto cloned_delta_it = cloned_transaction.deltas.begin();
while (cloned_delta_it != cloned_transaction.deltas.end()) {
const auto prev = cloned_delta_it->prev.Get();
switch (prev.type) {
case PreviousPtr::Type::DELTA:
case PreviousPtr::Type::NULLPTR:
++cloned_delta_it;
break;
case PreviousPtr::Type::VERTEX: {
if (prev.vertex->first < split_key) {
// We can remove this delta chain
EraseDeltaChain(cloned_transaction, cloned_transactions, cloned_delta_it);
} else {
++cloned_delta_it;
}
break;
}
case PreviousPtr::Type::EDGE: {
if (const auto edge_gid = prev.edge->gid; !cloned_edges.contains(edge_gid)) {
// We can remove this delta chain
EraseDeltaChain(cloned_transaction, cloned_transactions, cloned_delta_it);
} else {
++cloned_delta_it;
}
break;
}
}
}
}
void Splitter::PruneOriginalDeltas(Transaction &transaction,
std::map<uint64_t, std::unique_ptr<Transaction>> &transactions,
const PrimaryKey &split_key) {
// Remove delta chains that don't point to objects on splitted shard
auto delta_it = transaction.deltas.begin();
while (delta_it != transaction.deltas.end()) {
const auto prev = delta_it->prev.Get();
switch (prev.type) {
case PreviousPtr::Type::DELTA:
case PreviousPtr::Type::NULLPTR:
++delta_it;
break;
case PreviousPtr::Type::VERTEX: {
if (prev.vertex->first >= split_key) {
// We can remove this delta chain
EraseDeltaChain(transaction, transactions, delta_it);
} else {
++delta_it;
}
break;
}
case PreviousPtr::Type::EDGE: {
if (const auto edge_gid = prev.edge->gid; !edges_.contains(edge_gid)) {
// We can remove this delta chain
EraseDeltaChain(transaction, transactions, delta_it);
} else {
++delta_it;
}
break;
}
}
}
}
void Splitter::AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges, const PrimaryKey &split_key) {
for (auto &[start_id, cloned_transaction] : cloned_transactions) {
AdjustClonedTransaction(*cloned_transaction, *start_logical_id_to_transaction_[start_id], cloned_transactions,
cloned_edges);
}
// Prune deltas whose delta chain points to vertex/edge that should not belong on that shard
// Prune must be after adjust, since next, and prev are not set and we cannot follow the chain
for (auto &[start_id, cloned_transaction] : cloned_transactions) {
PruneDeltas(*cloned_transaction, cloned_transactions, split_key, cloned_edges);
}
// Also we need to remove deltas from original transactions
for (auto &[start_id, original_transaction] : start_logical_id_to_transaction_) {
PruneOriginalDeltas(*original_transaction, start_logical_id_to_transaction_, split_key);
}
}
inline bool IsDeltaHeadOfChain(const PreviousPtr::Type &delta_type) {
return delta_type == PreviousPtr::Type::VERTEX || delta_type == PreviousPtr::Type::EDGE;
}
bool DoesPrevPtrPointsToSplittedData(const PreviousPtr::Pointer &prev_ptr, const PrimaryKey &split_key) {
return prev_ptr.type == PreviousPtr::Type::VERTEX && prev_ptr.vertex->first < split_key;
}
void Splitter::AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges) {
auto delta_it = transaction.deltas.begin();
auto cloned_delta_it = cloned_transaction.deltas.begin();
while (delta_it != transaction.deltas.end()) {
// We can safely ignore deltas which are not head of delta chain
// Dont' adjust delta chain that points to irrelevant data vertices/edges
if (const auto delta_prev = delta_it->prev.Get(); !IsDeltaHeadOfChain(delta_prev.type)) {
++delta_it;
++cloned_delta_it;
continue;
}
const auto *delta = &*delta_it;
auto *cloned_delta = &*cloned_delta_it;
Delta *cloned_delta_prev_ptr = cloned_delta;
// The head of delta chains contain either vertex/edge as prev ptr so we adjust
// it just at the beginning of delta chain
AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_edges);
while (delta->next != nullptr) {
AdjustEdgeRef(*cloned_delta, cloned_edges);
// Align next ptr and prev ptr
AdjustDeltaNextAndPrev(*delta, *cloned_delta, cloned_transactions);
// Next delta might not belong to the cloned transaction and thats
// why we skip this delta of the delta chain
if (cloned_delta->next != nullptr) {
cloned_delta = cloned_delta->next;
cloned_delta_prev_ptr = cloned_delta;
} else {
cloned_delta_prev_ptr = nullptr;
}
delta = delta->next;
}
// Align prev ptr
if (cloned_delta_prev_ptr != nullptr) {
AdjustDeltaPrevPtr(*delta, *cloned_delta_prev_ptr, cloned_transactions, cloned_edges);
}
++delta_it;
++cloned_delta_it;
}
MG_ASSERT(delta_it == transaction.deltas.end() && cloned_delta_it == cloned_transaction.deltas.end(),
"Both iterators must be exhausted!");
}
void Splitter::AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const {
switch (cloned_delta.action) {
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::REMOVE_OUT_EDGE: {
// Find edge
if (config_.items.properties_on_edges) {
if (const auto cloned_edge_it = cloned_edges.find(cloned_delta.vertex_edge.edge.ptr->gid);
cloned_edge_it != cloned_edges.end()) {
cloned_delta.vertex_edge.edge = EdgeRef{&cloned_edge_it->second};
}
}
break;
}
case Delta::Action::DELETE_OBJECT:
case Delta::Action::RECREATE_OBJECT:
case Delta::Action::SET_PROPERTY:
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL: {
// noop
break;
}
}
}
void Splitter::AdjustDeltaNextAndPrev(const Delta &original, Delta &cloned,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions) {
// Get cloned_delta->next transaction, using delta->next original transaction
// cloned_transactions key is start_timestamp
auto cloned_transaction_it =
cloned_transactions.find(original.next->commit_info->start_or_commit_timestamp.logical_id);
if (cloned_transaction_it == cloned_transactions.end()) {
cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&original](const auto &elem) {
return elem.second->commit_info->start_or_commit_timestamp ==
original.next->commit_info->start_or_commit_timestamp;
});
}
// TODO(jbajic) What if next in delta chain does not belong to cloned transaction?
// MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
if (cloned_transaction_it == cloned_transactions.end()) return;
// Find cloned delta in delta list of cloned transaction
auto found_cloned_delta_it = std::ranges::find_if(
cloned_transaction_it->second->deltas, [&original](const auto &elem) { return elem.id == original.next->id; });
MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(), "Delta with given uuid must exist!");
cloned.next = &*found_cloned_delta_it;
found_cloned_delta_it->prev.Set(&cloned);
}
void Splitter::AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges) {
auto ptr = original.prev.Get();
switch (ptr.type) {
case PreviousPtr::Type::NULLPTR: {
MG_ASSERT(false, "PreviousPtr cannot be a nullptr!");
break;
}
case PreviousPtr::Type::DELTA: {
// Same as for deltas except don't align next but prev
auto cloned_transaction_it = std::ranges::find_if(cloned_transactions, [&ptr](const auto &elem) {
return elem.second->start_timestamp == ptr.delta->commit_info->start_or_commit_timestamp ||
elem.second->commit_info->start_or_commit_timestamp == ptr.delta->commit_info->start_or_commit_timestamp;
});
MG_ASSERT(cloned_transaction_it != cloned_transactions.end(), "Cloned transaction not found");
// Find cloned delta in delta list of cloned transaction
auto found_cloned_delta_it =
std::ranges::find_if(cloned_transaction_it->second->deltas,
[delta = ptr.delta](const auto &elem) { return elem.id == delta->id; });
MG_ASSERT(found_cloned_delta_it != cloned_transaction_it->second->deltas.end(),
"Delta with given id must exist!");
cloned.prev.Set(&*found_cloned_delta_it);
break;
}
case PreviousPtr::Type::VERTEX: {
// The vertex was extracted and it is safe to reuse address
cloned.prev.Set(ptr.vertex);
ptr.vertex->second.delta = &cloned;
break;
}
case PreviousPtr::Type::EDGE: {
// We can never be here if we have properties on edge disabled
auto *cloned_edge = &*cloned_edges.find(ptr.edge->gid);
ptr.edge->delta = &cloned;
cloned.prev.Set(&cloned_edge->second);
break;
}
};
}
} // namespace memgraph::storage::v3

View File

@@ -1,109 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include "storage/v3/config.hpp"
#include "storage/v3/delta.hpp"
#include "storage/v3/edge.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/indices.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/concepts.hpp"
namespace memgraph::storage::v3 {
// If edge properties-on-edges is false then we don't need to send edges but
// only vertices, since they will contain those edges
struct SplitData {
LabelId primary_label;
PrimaryKey min_primary_key;
std::optional<PrimaryKey> max_primary_key;
std::vector<SchemaProperty> schema;
Config config;
std::unordered_map<uint64_t, std::string> id_to_name;
uint64_t shard_version;
VertexContainer vertices;
std::optional<EdgeContainer> edges;
std::map<uint64_t, std::unique_ptr<Transaction>> transactions;
std::map<LabelId, LabelIndex::IndexContainer> label_indices;
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndex::IndexContainer> label_property_indices;
};
// TODO(jbajic) Handle deleted_vertices_ and deleted_edges_ after the finishing GC
class Splitter final {
public:
Splitter(LabelId primary_label, VertexContainer &vertices, EdgeContainer &edges,
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction, Indices &indices,
const Config &config, const std::vector<SchemaProperty> &schema, const NameIdMapper &name_id_mapper_);
Splitter(const Splitter &) = delete;
Splitter(Splitter &&) noexcept = delete;
Splitter &operator=(const Splitter &) = delete;
Splitter operator=(Splitter &&) noexcept = delete;
~Splitter() = default;
SplitData SplitShard(const PrimaryKey &split_key, const std::optional<PrimaryKey> &max_primary_key,
uint64_t shard_version);
private:
VertexContainer CollectVertices(SplitData &data, std::set<uint64_t> &collected_transactions_start_id,
const PrimaryKey &split_key);
std::optional<EdgeContainer> CollectEdges(std::set<uint64_t> &collected_transactions_start_id,
const VertexContainer &split_vertices, const PrimaryKey &split_key);
std::map<uint64_t, std::unique_ptr<Transaction>> CollectTransactions(
const std::set<uint64_t> &collected_transactions_start_id, EdgeContainer &cloned_edges,
const PrimaryKey &split_key);
static void ScanDeltas(std::set<uint64_t> &collected_transactions_start_id, const Delta *delta);
void PruneOriginalDeltas(Transaction &transaction, std::map<uint64_t, std::unique_ptr<Transaction>> &transactions,
const PrimaryKey &split_key);
void AdjustClonedTransaction(Transaction &cloned_transaction, const Transaction &transaction,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges);
void AdjustClonedTransactions(std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges, const PrimaryKey &split_key);
void AdjustEdgeRef(Delta &cloned_delta, EdgeContainer &cloned_edges) const;
static void AdjustDeltaNextAndPrev(const Delta &original, Delta &cloned,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions);
static void AdjustDeltaPrevPtr(const Delta &original, Delta &cloned,
std::map<uint64_t, std::unique_ptr<Transaction>> &cloned_transactions,
EdgeContainer &cloned_edges);
const LabelId primary_label_;
VertexContainer &vertices_;
EdgeContainer &edges_;
std::map<uint64_t, std::unique_ptr<Transaction>> &start_logical_id_to_transaction_;
Indices &indices_;
const Config &config_;
const std::vector<SchemaProperty> schema_;
const NameIdMapper &name_id_mapper_;
};
} // namespace memgraph::storage::v3

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -31,15 +31,6 @@ struct CommitInfo {
};
struct Transaction {
Transaction(coordinator::Hlc start_timestamp, CommitInfo new_commit_info, uint64_t command_id, bool must_abort,
bool is_aborted, IsolationLevel isolation_level)
: start_timestamp{start_timestamp},
commit_info{std::make_unique<CommitInfo>(new_commit_info)},
command_id(command_id),
must_abort(must_abort),
is_aborted(is_aborted),
isolation_level(isolation_level){};
Transaction(coordinator::Hlc start_timestamp, IsolationLevel isolation_level)
: start_timestamp(start_timestamp),
commit_info(std::make_unique<CommitInfo>(CommitInfo{false, {start_timestamp}})),
@@ -63,56 +54,6 @@ struct Transaction {
~Transaction() {}
std::list<Delta> CopyDeltas(CommitInfo *commit_info) const {
std::list<Delta> copied_deltas;
for (const auto &delta : deltas) {
switch (delta.action) {
case Delta::Action::DELETE_OBJECT:
copied_deltas.emplace_back(Delta::DeleteObjectTag{}, commit_info, delta.id, command_id);
break;
case Delta::Action::RECREATE_OBJECT:
copied_deltas.emplace_back(Delta::RecreateObjectTag{}, commit_info, delta.id, command_id);
break;
case Delta::Action::ADD_LABEL:
copied_deltas.emplace_back(Delta::AddLabelTag{}, delta.label, commit_info, delta.id, command_id);
break;
case Delta::Action::REMOVE_LABEL:
copied_deltas.emplace_back(Delta::RemoveLabelTag{}, delta.label, commit_info, delta.id, command_id);
break;
case Delta::Action::ADD_IN_EDGE:
copied_deltas.emplace_back(Delta::AddInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
delta.vertex_edge.edge, commit_info, delta.id, command_id);
break;
case Delta::Action::ADD_OUT_EDGE:
copied_deltas.emplace_back(Delta::AddOutEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
delta.vertex_edge.edge, commit_info, delta.id, command_id);
break;
case Delta::Action::REMOVE_IN_EDGE:
copied_deltas.emplace_back(Delta::RemoveInEdgeTag{}, delta.vertex_edge.edge_type, delta.vertex_edge.vertex_id,
delta.vertex_edge.edge, commit_info, delta.id, command_id);
break;
case Delta::Action::REMOVE_OUT_EDGE:
copied_deltas.emplace_back(Delta::RemoveOutEdgeTag{}, delta.vertex_edge.edge_type,
delta.vertex_edge.vertex_id, delta.vertex_edge.edge, commit_info, delta.id,
command_id);
break;
case Delta::Action::SET_PROPERTY:
copied_deltas.emplace_back(Delta::SetPropertyTag{}, delta.property.key, delta.property.value, commit_info,
delta.id, command_id);
break;
}
}
return copied_deltas;
}
// This does not solve the whole problem of copying deltas
std::unique_ptr<Transaction> Clone() const {
auto transaction_ptr = std::make_unique<Transaction>(start_timestamp, *commit_info, command_id, must_abort,
is_aborted, isolation_level);
transaction_ptr->deltas = CopyDeltas(transaction_ptr->commit_info.get());
return transaction_ptr;
}
coordinator::Hlc start_timestamp;
std::unique_ptr<CommitInfo> commit_info;
uint64_t command_id;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -126,17 +126,6 @@ inline std::vector<Value> ConvertValueVector(const std::vector<v3::PropertyValue
return ret;
}
inline std::vector<Value> ConvertValueVector(std::vector<v3::PropertyValue> &&vec) {
std::vector<Value> ret;
ret.reserve(vec.size());
for (auto &&elem : vec) {
ret.push_back(FromPropertyValueToValue(std::move(elem)));
}
return ret;
}
inline msgs::VertexId ToMsgsVertexId(const v3::VertexId &vertex_id) {
return {msgs::Label{vertex_id.primary_label}, ConvertValueVector(vertex_id.primary_key)};
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,7 +12,6 @@
#pragma once
#include <concepts>
#include <iterator>
#include <type_traits>
namespace memgraph::utils {
template <typename T, typename... Args>
@@ -35,7 +34,4 @@ template <typename T>
concept Dereferenceable = requires(T t) {
{ *t } -> CanReference;
};
template <typename T>
concept Object = std::is_object_v<T>;
} // namespace memgraph::utils

View File

@@ -79,12 +79,3 @@ target_link_libraries(${test_prefix}data_structures_contains mg-utils mg-storage
add_benchmark(data_structures_remove.cpp)
target_link_libraries(${test_prefix}data_structures_remove mg-utils mg-storage-v3)
add_benchmark(storage_v3_split.cpp)
target_link_libraries(${test_prefix}storage_v3_split mg-storage-v3 mg-query-v2)
add_benchmark(storage_v3_split_1.cpp)
target_link_libraries(${test_prefix}storage_v3_split_1 mg-storage-v3 mg-query-v2)
add_benchmark(storage_v3_split_2.cpp)
target_link_libraries(${test_prefix}storage_v3_split_2 mg-storage-v3 mg-query-v2)

View File

@@ -1,249 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include <optional>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
namespace memgraph::benchmark {
class ShardSplitBenchmark : public ::benchmark::Fixture {
protected:
using PrimaryKey = storage::v3::PrimaryKey;
using PropertyId = storage::v3::PropertyId;
using PropertyValue = storage::v3::PropertyValue;
using LabelId = storage::v3::LabelId;
using EdgeTypeId = storage::v3::EdgeTypeId;
using Shard = storage::v3::Shard;
using VertexId = storage::v3::VertexId;
using Gid = storage::v3::Gid;
void SetUp(const ::benchmark::State &state) override {
storage.emplace(primary_label, min_pk, std::nullopt, schema_property_vector);
storage->StoreMapping(
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
}
void TearDown(const ::benchmark::State &) override { storage = std::nullopt; }
const PropertyId primary_property{PropertyId::FromUint(2)};
const PropertyId secondary_property{PropertyId::FromUint(5)};
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
const LabelId primary_label{LabelId::FromUint(1)};
const LabelId secondary_label{LabelId::FromUint(4)};
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
std::optional<Shard> storage;
coordinator::Hlc last_hlc{0, io::Time{}};
coordinator::Hlc GetNextHlc() {
++last_hlc.logical_id;
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
return last_hlc;
}
};
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplit)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithGc)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)(::benchmark::State &state) {
const auto number_of_vertices = state.range(0);
const auto number_of_edges = state.range(1);
const auto number_of_transactions = state.range(2);
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices - number_of_transactions - 1);
// Create Vertices
int64_t vertex_count{0};
{
auto acc = storage->Access(GetNextHlc());
for (; vertex_count < number_of_vertices - number_of_transactions; ++vertex_count) {
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
}
// Create Edges
for (int64_t i{0}; i < number_of_edges; ++i) {
const auto vtx1 = uniform_dist(e1);
const auto vtx2 = uniform_dist(e1);
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
// Clean up transactional data
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
// Create rest of the objects and leave transactions
for (; vertex_count < number_of_vertices; ++vertex_count) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
// Don't create shard since shard deallocation can take some time as well
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
// Range:
// Number of vertices
// This run is pessimistic, number of vertices corresponds with number if transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplit)
// ->RangeMultiplier(10)
// ->Range(100'000, 100'000)
// ->Unit(::benchmark::kMillisecond);
// Range:
// Number of vertices
// This run is optimistic, in this run there are no transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithGc)
// ->RangeMultiplier(10)
// ->Range(100'000, 1'000'000)
// ->Unit(::benchmark::kMillisecond);
// Args:
// Number of vertices
// Number of edges
// Number of transaction
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
->Args({100'000, 100'000, 100})
->Args({200'000, 100'000, 100})
->Args({300'000, 100'000, 100})
->Args({400'000, 100'000, 100})
->Args({500'000, 100'000, 100})
->Args({600'000, 100'000, 100})
->Args({700'000, 100'000, 100})
->Args({800'000, 100'000, 100})
->Args({900'000, 100'000, 100})
->Args({1'000'000, 100'000, 100})
->Args({2'000'000, 100'000, 100})
->Args({3'000'000, 100'000, 100})
->Args({4'000'000, 100'000, 100})
->Args({5'000'000, 100'000, 100})
->Args({6'000'000, 100'000, 100})
->Args({7'000'000, 100'000, 100})
->Args({8'000'000, 100'000, 100})
->Args({9'000'000, 100'000, 100})
->Args({10'000'000, 100'000, 100})
->Unit(::benchmark::kMillisecond)
->Name("IncreaseVertices");
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
->Args({100'000, 100'000, 100})
->Args({100'000, 200'000, 100})
->Args({100'000, 300'000, 100})
->Args({100'000, 400'000, 100})
->Args({100'000, 500'000, 100})
->Args({100'000, 600'000, 100})
->Args({100'000, 700'000, 100})
->Args({100'000, 800'000, 100})
->Args({100'000, 900'000, 100})
->Args({100'000, 1'000'000, 100})
->Args({100'000, 2'000'000, 100})
->Args({100'000, 3'000'000, 100})
->Args({100'000, 4'000'000, 100})
->Args({100'000, 5'000'000, 100})
->Args({100'000, 6'000'000, 100})
->Args({100'000, 7'000'000, 100})
->Args({100'000, 8'000'000, 100})
->Args({100'000, 9'000'000, 100})
->Args({100'000, 10'000'000, 100})
->Unit(::benchmark::kMillisecond)
->Name("IncreaseEdges");
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
->Args({100'000, 100'000, 100})
->Args({100'000, 100'000, 1'000})
->Args({100'000, 100'000, 10'000})
->Args({100'000, 100'000, 100'000})
->Unit(::benchmark::kMillisecond)
->Name("IncreaseTransactions");
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

View File

@@ -1,270 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include <optional>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
namespace memgraph::benchmark {
class ShardSplitBenchmark : public ::benchmark::Fixture {
protected:
using PrimaryKey = storage::v3::PrimaryKey;
using PropertyId = storage::v3::PropertyId;
using PropertyValue = storage::v3::PropertyValue;
using LabelId = storage::v3::LabelId;
using EdgeTypeId = storage::v3::EdgeTypeId;
using Shard = storage::v3::Shard;
using VertexId = storage::v3::VertexId;
using Gid = storage::v3::Gid;
void SetUp(const ::benchmark::State &state) override {
storage.emplace(primary_label, min_pk, std::nullopt, schema_property_vector);
storage->StoreMapping(
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
}
void TearDown(const ::benchmark::State &) override { storage = std::nullopt; }
const PropertyId primary_property{PropertyId::FromUint(2)};
const PropertyId secondary_property{PropertyId::FromUint(5)};
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
const LabelId primary_label{LabelId::FromUint(1)};
const LabelId secondary_label{LabelId::FromUint(4)};
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
std::optional<Shard> storage;
coordinator::Hlc last_hlc{0, io::Time{}};
coordinator::Hlc GetNextHlc() {
++last_hlc.logical_id;
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
return last_hlc;
}
};
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplit)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithGc)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)(::benchmark::State &state) {
const auto number_of_vertices = state.range(0);
const auto number_of_edges = state.range(1);
const auto number_of_transactions = state.range(2);
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices - number_of_transactions - 1);
// Create Vertices
int64_t vertex_count{0};
{
auto acc = storage->Access(GetNextHlc());
for (; vertex_count < number_of_vertices - number_of_transactions; ++vertex_count) {
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
}
// Create Edges
for (int64_t i{0}; i < number_of_edges; ++i) {
const auto vtx1 = uniform_dist(e1);
const auto vtx2 = uniform_dist(e1);
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
// Clean up transactional data
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
// Create rest of the objects and leave transactions
for (; vertex_count < number_of_vertices; ++vertex_count) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
// Don't create shard since shard deallocation can take some time as well
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
// Range:
// Number of vertices
// This run is pessimistic, number of vertices corresponds with number if transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplit)
// ->RangeMultiplier(10)
// ->Range(100'000, 100'000)
// ->Unit(::benchmark::kMillisecond);
// Range:
// Number of vertices
// This run is optimistic, in this run there are no transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithGc)
// ->RangeMultiplier(10)
// ->Range(100'000, 1'000'000)
// ->Unit(::benchmark::kMillisecond);
// Args:
// Number of vertices
// Number of edges
// Number of transaction
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({200'000, 100'000, 100})
// ->Args({300'000, 100'000, 100})
// ->Args({400'000, 100'000, 100})
// ->Args({500'000, 100'000, 100})
// ->Args({600'000, 100'000, 100})
// ->Args({700'000, 100'000, 100})
// ->Args({800'000, 100'000, 100})
->Args({900'000, 100'000, 100})
// ->Args({1'000'000, 100'000, 100})
// ->Args({2'000'000, 100'000, 100})
// ->Args({3'000'000, 100'000, 100})
// ->Args({4'000'000, 100'000, 100})
// ->Args({6'000'000, 100'000, 100})
// ->Args({7'000'000, 100'000, 100})
// ->Args({8'000'000, 100'000, 100})
// ->Args({9'000'000, 100'000, 100})
// ->Args({10'000'000, 100'000, 100})
->Unit(::benchmark::kMillisecond)
->Name("IncreaseVertices");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({200'000, 100'000, 100})
// ->Args({300'000, 100'000, 100})
// ->Args({400'000, 100'000, 100})
// ->Args({500'000, 100'000, 100})
// ->Args({600'000, 100'000, 100})
// ->Args({700'000, 100'000, 100})
// ->Args({800'000, 100'000, 100})
// ->Args({900'000, 100'000, 100})
// ->Args({1'000'000, 100'000, 100})
// ->Args({2'000'000, 100'000, 100})
// ->Args({3'000'000, 100'000, 100})
// ->Args({4'000'000, 100'000, 100})
// ->Args({6'000'000, 100'000, 100})
// ->Args({7'000'000, 100'000, 100})
// ->Args({8'000'000, 100'000, 100})
// ->Args({9'000'000, 100'000, 100})
// ->Args({10'000'000, 100'000, 100})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseVertices");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({100'000, 200'000, 100})
// ->Args({100'000, 300'000, 100})
// ->Args({100'000, 400'000, 100})
// ->Args({100'000, 500'000, 100})
// ->Args({100'000, 600'000, 100})
// ->Args({100'000, 700'000, 100})
// ->Args({100'000, 800'000, 100})
// ->Args({100'000, 900'000, 100})
// ->Args({100'000, 1'000'000, 100})
// ->Args({100'000, 2'000'000, 100})
// ->Args({100'000, 3'000'000, 100})
// ->Args({100'000, 4'000'000, 100})
// ->Args({100'000, 5'000'000, 100})
// ->Args({100'000, 6'000'000, 100})
// ->Args({100'000, 7'000'000, 100})
// ->Args({100'000, 8'000'000, 100})
// ->Args({100'000, 9'000'000, 100})
// ->Args({100'000, 10'000'000, 100})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseEdges");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({100'000, 100'000, 1'000})
// ->Args({100'000, 100'000, 10'000})
// ->Args({100'000, 100'000, 100'000})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseTransactions");
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

View File

@@ -1,270 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include <optional>
#include <vector>
#include <benchmark/benchmark.h>
#include <gflags/gflags.h>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
namespace memgraph::benchmark {
class ShardSplitBenchmark : public ::benchmark::Fixture {
protected:
using PrimaryKey = storage::v3::PrimaryKey;
using PropertyId = storage::v3::PropertyId;
using PropertyValue = storage::v3::PropertyValue;
using LabelId = storage::v3::LabelId;
using EdgeTypeId = storage::v3::EdgeTypeId;
using Shard = storage::v3::Shard;
using VertexId = storage::v3::VertexId;
using Gid = storage::v3::Gid;
void SetUp(const ::benchmark::State &state) override {
storage.emplace(primary_label, min_pk, std::nullopt, schema_property_vector);
storage->StoreMapping(
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
}
void TearDown(const ::benchmark::State &) override { storage = std::nullopt; }
const PropertyId primary_property{PropertyId::FromUint(2)};
const PropertyId secondary_property{PropertyId::FromUint(5)};
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
const LabelId primary_label{LabelId::FromUint(1)};
const LabelId secondary_label{LabelId::FromUint(4)};
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
std::optional<Shard> storage;
coordinator::Hlc last_hlc{0, io::Time{}};
coordinator::Hlc GetNextHlc() {
++last_hlc.logical_id;
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
return last_hlc;
}
};
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplit)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithGc)(::benchmark::State &state) {
const auto number_of_vertices{state.range(0)};
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices);
for (int64_t i{0}; i < number_of_vertices; ++i) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(i)},
{{secondary_property, PropertyValue(i)}})
.HasValue(),
"Failed creating with pk {}", i);
if (i > 1) {
const auto vtx1 = uniform_dist(e1) % i;
const auto vtx2 = uniform_dist(e1) % i;
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
for (auto _ : state) {
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
BENCHMARK_DEFINE_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)(::benchmark::State &state) {
const auto number_of_vertices = state.range(0);
const auto number_of_edges = state.range(1);
const auto number_of_transactions = state.range(2);
std::random_device r;
std::default_random_engine e1(r());
std::uniform_int_distribution<int> uniform_dist(0, number_of_vertices - number_of_transactions - 1);
// Create Vertices
int64_t vertex_count{0};
{
auto acc = storage->Access(GetNextHlc());
for (; vertex_count < number_of_vertices - number_of_transactions; ++vertex_count) {
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
}
// Create Edges
for (int64_t i{0}; i < number_of_edges; ++i) {
const auto vtx1 = uniform_dist(e1);
const auto vtx2 = uniform_dist(e1);
MG_ASSERT(acc.CreateEdge(VertexId{primary_label, {PropertyValue(vtx1)}},
VertexId{primary_label, {PropertyValue(vtx2)}}, edge_type_id, Gid::FromUint(i))
.HasValue(),
"Failed on {} and {}", vtx1, vtx2);
}
acc.Commit(GetNextHlc());
}
// Clean up transactional data
storage->CollectGarbage(GetNextHlc().coordinator_wall_clock);
// Create rest of the objects and leave transactions
for (; vertex_count < number_of_vertices; ++vertex_count) {
auto acc = storage->Access(GetNextHlc());
MG_ASSERT(acc.CreateVertexAndValidate({secondary_label}, PrimaryKey{PropertyValue(vertex_count)},
{{secondary_property, PropertyValue(vertex_count)}})
.HasValue(),
"Failed creating with pk {}", vertex_count);
acc.Commit(GetNextHlc());
}
for (auto _ : state) {
// Don't create shard since shard deallocation can take some time as well
auto data = storage->PerformSplit(PrimaryKey{PropertyValue{number_of_vertices / 2}}, 2);
}
}
// Range:
// Number of vertices
// This run is pessimistic, number of vertices corresponds with number if transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplit)
// ->RangeMultiplier(10)
// ->Range(100'000, 100'000)
// ->Unit(::benchmark::kMillisecond);
// Range:
// Number of vertices
// This run is optimistic, in this run there are no transactions
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithGc)
// ->RangeMultiplier(10)
// ->Range(100'000, 1'000'000)
// ->Unit(::benchmark::kMillisecond);
// Args:
// Number of vertices
// Number of edges
// Number of transaction
BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({200'000, 100'000, 100})
// ->Args({300'000, 100'000, 100})
// ->Args({400'000, 100'000, 100})
// ->Args({500'000, 100'000, 100})
// ->Args({600'000, 100'000, 100})
// ->Args({700'000, 100'000, 100})
// ->Args({800'000, 100'000, 100})
// ->Args({900'000, 100'000, 100})
->Args({1'000'000, 100'000, 100})
// ->Args({2'000'000, 100'000, 100})
// ->Args({3'000'000, 100'000, 100})
// ->Args({4'000'000, 100'000, 100})
// ->Args({6'000'000, 100'000, 100})
// ->Args({7'000'000, 100'000, 100})
// ->Args({8'000'000, 100'000, 100})
// ->Args({9'000'000, 100'000, 100})
// ->Args({10'000'000, 100'000, 100})
->Unit(::benchmark::kMillisecond)
->Name("IncreaseVertices");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({200'000, 100'000, 100})
// ->Args({300'000, 100'000, 100})
// ->Args({400'000, 100'000, 100})
// ->Args({500'000, 100'000, 100})
// ->Args({600'000, 100'000, 100})
// ->Args({700'000, 100'000, 100})
// ->Args({800'000, 100'000, 100})
// ->Args({900'000, 100'000, 100})
// ->Args({1'000'000, 100'000, 100})
// ->Args({2'000'000, 100'000, 100})
// ->Args({3'000'000, 100'000, 100})
// ->Args({4'000'000, 100'000, 100})
// ->Args({6'000'000, 100'000, 100})
// ->Args({7'000'000, 100'000, 100})
// ->Args({8'000'000, 100'000, 100})
// ->Args({9'000'000, 100'000, 100})
// ->Args({10'000'000, 100'000, 100})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseVertices");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({100'000, 200'000, 100})
// ->Args({100'000, 300'000, 100})
// ->Args({100'000, 400'000, 100})
// ->Args({100'000, 500'000, 100})
// ->Args({100'000, 600'000, 100})
// ->Args({100'000, 700'000, 100})
// ->Args({100'000, 800'000, 100})
// ->Args({100'000, 900'000, 100})
// ->Args({100'000, 1'000'000, 100})
// ->Args({100'000, 2'000'000, 100})
// ->Args({100'000, 3'000'000, 100})
// ->Args({100'000, 4'000'000, 100})
// ->Args({100'000, 5'000'000, 100})
// ->Args({100'000, 6'000'000, 100})
// ->Args({100'000, 7'000'000, 100})
// ->Args({100'000, 8'000'000, 100})
// ->Args({100'000, 9'000'000, 100})
// ->Args({100'000, 10'000'000, 100})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseEdges");
// BENCHMARK_REGISTER_F(ShardSplitBenchmark, BigDataSplitWithFewTransactionsOnVertices)
// ->Args({100'000, 100'000, 100})
// ->Args({100'000, 100'000, 1'000})
// ->Args({100'000, 100'000, 10'000})
// ->Args({100'000, 100'000, 100'000})
// ->Unit(::benchmark::kMillisecond)
// ->Name("IncreaseTransactions");
} // namespace memgraph::benchmark
BENCHMARK_MAIN();

View File

@@ -36,9 +36,7 @@ def test_awesome_memgraph_functions(connection):
assert len(results) == 1
assert results[0][0] == 5
results = execute_and_fetch_all(
cursor, "UNWIND [2, 1, 3] AS value WITH COLLECT(value) as nn RETURN ALL(i IN nn WHERE i > 0)"
)
results = execute_and_fetch_all(cursor, "MATCH (n) WITH COLLECT(n.property) as nn RETURN ALL(i IN nn WHERE i > 0)")
assert len(results) == 1
assert results[0][0] == True

View File

@@ -9,13 +9,11 @@
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import time
import typing
import mgclient
import sys
import pytest
import time
from common import *
@@ -32,7 +30,8 @@ def test_distinct(connection):
assert len(results) == 2
for i, n in enumerate(results):
n_props = n[0].properties
assert len(n_props) == 0
assert len(n_props) == 1
assert n_props["property"] == i
if __name__ == "__main__":

View File

@@ -13,12 +13,7 @@ import sys
import pytest
from common import (
connection,
execute_and_fetch_all,
has_n_result_row,
wait_for_shard_manager_to_initialize,
)
from common import connection, execute_and_fetch_all, has_n_result_row, wait_for_shard_manager_to_initialize
def test_sequenced_expand_one(connection):
@@ -27,21 +22,15 @@ def test_sequenced_expand_one(connection):
for i in range(1, 4):
assert has_n_result_row(cursor, f"CREATE (:label {{property:{i}}})", 0), f"Failed creating node"
assert has_n_result_row(cursor, "MATCH (n:label {property:1}), (m:label {property:2}) CREATE (n)-[:TO]->(m)", 0)
assert has_n_result_row(cursor, "MATCH (n:label {property:2}), (m:label {property:3}) CREATE (n)-[:TO]->(m)", 0)
assert has_n_result_row(cursor, "MATCH (n {property:1}), (m {property:2}) CREATE (n)-[:TO]->(m)", 0)
assert has_n_result_row(cursor, "MATCH (n {property:2}), (m {property:3}) CREATE (n)-[:TO]->(m)", 0)
results = execute_and_fetch_all(cursor, "MATCH (n)-[:TO]->(m)-[:TO]->(l) RETURN n,m,l")
assert len(results) == 1
n, m, l = results[0]
assert (
len(n.properties) == 0
), "we don't return any properties of the node received from expansion and the bolt layer doesn't serialize the primary key of vertices"
assert (
len(m.properties) == 0
), "we don't return any properties of the node received from expansion and the bolt layer doesn't serialize the primary key of vertices"
assert (
len(l.properties) == 0
), "we don't return any properties of the node received from expansion and the bolt layer doesn't serialize the primary key of vertices"
assert n.properties["property"] == 1
assert m.properties["property"] == 2
assert l.properties["property"] == 3
if __name__ == "__main__":

View File

@@ -9,13 +9,11 @@
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import time
import typing
import mgclient
import sys
import pytest
import time
from common import *
@@ -37,13 +35,13 @@ def test_vertex_creation_and_scanall(connection):
assert len(results) == 9
for (n, r, m) in results:
n_props = n.properties
assert len(n_props) == 0, "n is not expected to have properties, update the test!"
assert len(n_props) == 1, "n is not expected to have properties, update the test!"
assert len(n.labels) == 0, "n is not expected to have labels, update the test!"
assert r.type == "TO"
m_props = m.properties
assert len(m_props) == 0, "n is not expected to have properties, update the test!"
assert m_props["property"] <= 3 and m_props["property"] >= 0, "Wrong key"
assert len(m.labels) == 0, "m is not expected to have labels, update the test!"

View File

@@ -9,13 +9,11 @@
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import time
import typing
import mgclient
import sys
import pytest
import time
from common import *
@@ -23,23 +21,23 @@ def test_order_by_and_limit(connection):
wait_for_shard_manager_to_initialize()
cursor = connection.cursor()
results = execute_and_fetch_all(
cursor,
"UNWIND [{property:1}, {property:3}, {property:2}] AS map RETURN map ORDER BY map.property DESC",
)
assert len(results) == 3
i = 3
for map in results:
assert len(map) == 1
assert map[0]["property"] == i
assert has_n_result_row(cursor, "CREATE (n :label {property:1})", 0)
assert has_n_result_row(cursor, "CREATE (n :label {property:2})", 0)
assert has_n_result_row(cursor, "CREATE (n :label {property:3})", 0)
assert has_n_result_row(cursor, "CREATE (n :label {property:4})", 0)
results = execute_and_fetch_all(cursor, "MATCH (n) RETURN n ORDER BY n.property DESC")
assert len(results) == 4
i = 4
for n in results:
n_props = n[0].properties
assert len(n_props) == 1
assert n_props["property"] == i
i = i - 1
result = execute_and_fetch_all(
cursor,
"UNWIND [{property:1}, {property:3}, {property:2}] AS map RETURN map ORDER BY map.property LIMIT 1",
)
result = execute_and_fetch_all(cursor, "MATCH (n) RETURN n ORDER BY n.property LIMIT 1")
assert len(result) == 1
assert result[0][0]["property"] == 1
assert result[0][0].properties["property"] == 1
if __name__ == "__main__":

View File

@@ -9,7 +9,6 @@ function(add_manual_test test_cpp)
get_filename_component(exec_name ${test_cpp} NAME_WE)
set(target_name ${test_prefix}${exec_name})
add_executable(${target_name} ${test_cpp} ${ARGN})
# OUTPUT_NAME sets the real name of a target when it is built and can be
# used to help create two targets of the same name even though CMake
# requires unique logical target names
@@ -22,7 +21,7 @@ target_link_libraries(${test_prefix}antlr_parser antlr_opencypher_parser_lib)
add_manual_test(antlr_sigsegv.cpp)
target_link_libraries(${test_prefix}antlr_sigsegv gtest gtest_main
antlr_opencypher_parser_lib mg-utils)
antlr_opencypher_parser_lib mg-utils)
add_manual_test(antlr_tree_pretty_print.cpp)
target_link_libraries(${test_prefix}antlr_tree_pretty_print antlr_opencypher_parser_lib)
@@ -38,15 +37,13 @@ target_link_libraries(${test_prefix}query_hash mg-query)
add_manual_test(query_planner.cpp interactive/planning.cpp)
target_link_libraries(${test_prefix}query_planner mg-query)
if(READLINE_FOUND)
if (READLINE_FOUND)
target_link_libraries(${test_prefix}query_planner readline)
endif()
add_manual_test(query_execution_dummy.cpp)
target_link_libraries(${test_prefix}query_execution_dummy mg-query)
if(READLINE_FOUND)
if (READLINE_FOUND)
target_link_libraries(${test_prefix}query_execution_dummy readline)
endif()
@@ -64,6 +61,3 @@ target_link_libraries(${test_prefix}ssl_client mg-communication)
add_manual_test(ssl_server.cpp)
target_link_libraries(${test_prefix}ssl_server mg-communication)
add_manual_test(query_performance.cpp)
target_link_libraries(${test_prefix}query_performance mg-communication mg-utils mg-io mg-io-simulator mg-coordinator mg-query-v2 mg-storage-v3 mg-query mg-storage-v2)

View File

@@ -1,352 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// This binary is meant to easily compare the performance of:
// - Memgraph v2
// - Memgraph v3
// - Memgraph v3 with MultiFrame
// This binary measures three things which provides a high level and easily understandable metric about the performance
// difference between the different versions:
// 1. Read time: how much time does it take to read the files:
// 2. Init time: how much time does it take to run the init queries, including the index creation. For details please
// check RunV2.
// 3. Benchmark time: how much time does it take to run the benchmark queries.
// To quickly compare performance of the different versions just change the query or queries in the benchmark queries
// file you can see the different by running this executable. This way we don't have keep multiple binaries of Memgraph
// v2 and Memgraph v3 with/without MultiFrame, start Memgraph and connect to it with mgconsole and other hassles. As
// everything is run in this binary, it makes easier to generate perf reports/flamegraphs from the query execution of
// different Memgraph versions compared to using the full blown version of Memgraph.
//
// A few important notes:
// - All the input files are mandated to have an empty line at the end of the file as the reading logic expect that.
// - tests/mgbench/dataset_creator_unwind.py is recommended to generate the dataset because it generates queries with
// UNWIND that makes the import faster in Memgraph v3, thus we can compare the performance on non trivial datasets
// also. To make it possible to use the generated dataset, you have to move the generated index queries into a
// separate file that can be supplied as index queries file for this binary when using Memgraph v2. The reason for
// this is Memgraph v3 cannot handle indices yet, thus it crashes.
// - Check the command line flags and their description defined in this file.
// - Also check out the --default-multi-frame-size command line flag if you want to play with that.
// - The log level is manually set to warning in the main function to avoid the overwhelming log messages from Memgraph
// v3. Apart from ease of use, the huge amount of looging can degrade the actual performance.
//
// Example usage with Memgraph v2:
// ./query_performance
// --index-queries-file indices.cypher
// --init-queries-file dataset.cypher
// --benchmark-queries-files expand.cypher,match.cypyher
// --use-v3=false
//
// Example usage with Memgraph v3 without MultiFrame:
// ./query_performance
// --split-file split_file
// --init-queries-file dataset.cypher
// --benchmark-queries-files expand.cypher,match.cypyher
// --use-v3=true
// --use-multi-frame=false
//
// Example usage with Memgraph v3 with MultiFrame:
// ./query_performance
// --split-file split_file
// --init-queries-file dataset.cypher
// --benchmark-queries-files expand.cypher,match.cypyher
// --use-v3=true
// --use-multi-frame=true
//
// The examples are using only the necessary flags, however specifying all of them is not a problem, so if you specify
// --index-queries-file for Memgraph v3, then it will be safely ignored just as --split-file for Memgraph v2.
//
// To generate flamegraph you can use the following command:
// flamegraph --cmd "record -F 997 --call-graph fp -g" --root -o flamegraph.svg -- ./query_performance <flags>
// Using the default option (dwarf) for --call-graph when calling perf might result in too long runtine of flamegraph
// because of address resolution. See https://github.com/flamegraph-rs/flamegraph/issues/74.
#include <chrono>
#include <filesystem>
#include <fstream>
#include <istream>
#include <thread>
#include <fmt/core.h>
#include <gflags/gflags.h>
#include <spdlog/cfg/env.h>
#include <spdlog/spdlog.h>
#include <json/json.hpp>
// v3 includes
#include "io/address.hpp"
#include "io/local_transport/local_system.hpp"
#include "io/message_histogram_collector.hpp"
#include "machine_manager/machine_manager.hpp"
#include "query/discard_value_stream.hpp"
#include "query/v2/discard_value_stream.hpp"
#include "query/v2/interpreter.hpp"
#include "query/v2/request_router.hpp"
// v2 includes
#include "query/interpreter.hpp"
#include "storage/v2/storage.hpp"
// common includes
#include "utils/string.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(index_queries_file, "",
"Path to the file which contains the queries to create indices. Used only for v2. Must contain an empty "
"line at the end of the file after the queries.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(split_file, "",
"Path to the split file which contains the predefined labels, properties, edge types and shard-ranges. "
"Used only for v3. Must contain an empty line at the end of the file.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(init_queries_file, "",
"Path to the file that is used to insert the initial dataset, one query per line. Must contain an empty "
"line at the end of the file after the queries.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(benchmark_queries_files, "",
"Comma separated paths to the files that contain the queries that we want to compare, one query per "
"line. Must contain an empty line at the end of each file after the queries.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(use_v3, true, "If set to true, then Memgraph v3 will be used, otherwise Memgraph v2 will be used.");
DEFINE_string(export_json_results, "", "If not empty, then the results will be exported as a json file.");
DEFINE_string(data_directory, "mg_data", "Path to directory to use as storage directory for Memgraph v2.");
namespace memgraph::tests::manual {
template <typename TInterpreterContext>
struct DependantTypes {};
template <>
struct DependantTypes<query::InterpreterContext> {
using Interpreter = query::Interpreter;
using DiscardValueResultStream = query::DiscardValueResultStream;
};
template <>
struct DependantTypes<query::v2::InterpreterContext> {
using Interpreter = query::v2::Interpreter;
using DiscardValueResultStream = query::v2::DiscardValueResultStream;
};
template <typename TRep, typename TPeriod>
void PutResult(nlohmann::json &json, const std::string_view name, std::chrono::duration<TRep, TPeriod> duration) {
json[name] = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
}
template <typename TInterpreterContext>
using Interpreter = typename DependantTypes<TInterpreterContext>::Interpreter;
template <typename TInterpreterContext>
using DiscardValueResultStream = typename DependantTypes<TInterpreterContext>::DiscardValueResultStream;
template <typename TInterpreterContext>
void RunQueries(TInterpreterContext &interpreter_context, const std::vector<std::string> &queries) {
Interpreter<TInterpreterContext> interpreter{&interpreter_context};
DiscardValueResultStream<TInterpreterContext> stream;
for (const auto &query : queries) {
auto result = interpreter.Prepare(query, {}, nullptr);
interpreter.Pull(&stream, std::nullopt, result.qid);
}
}
template <typename TInterpreterContext>
void RunInitQueries(TInterpreterContext &interpreter_context, const std::vector<std::string> &init_queries) {
RunQueries(interpreter_context, init_queries);
}
template <typename TInterpreterContext>
void RunBenchmarkQueries(TInterpreterContext &interpreter_context, const std::vector<std::string> &benchmark_queries) {
RunQueries(interpreter_context, benchmark_queries);
}
std::vector<std::string> ReadQueries(const std::string &file_name) {
std::vector<std::string> queries{};
std::string buffer;
std::ifstream file{file_name, std::ios::in};
MG_ASSERT(file.good(), "Cannot open queries file to read: {}", file_name);
while (file.good()) {
std::getline(file, buffer);
if (buffer.empty()) {
continue;
}
// Trim the trailing `;`
queries.push_back(buffer.substr(0, buffer.size() - 1));
}
return queries;
}
std::map<std::string, std::vector<std::string>> ReadBenchmarkQueries(const std::string benchmark_queries_files) {
auto benchmark_files = utils::Split(benchmark_queries_files, ",");
std::map<std::string, std::vector<std::string>> result;
for (const auto &benchmark_file : benchmark_files) {
const auto path = std::filesystem::path(benchmark_file);
result.emplace(path.stem().string(), ReadQueries(benchmark_file));
}
return result;
}
void RunV2() {
spdlog::critical("Running V2");
const auto run_start = std::chrono::high_resolution_clock::now();
const auto index_queries = ReadQueries(FLAGS_index_queries_file);
const auto init_queries = ReadQueries(FLAGS_init_queries_file);
const auto benchmarks = ReadBenchmarkQueries(FLAGS_benchmark_queries_files);
storage::Storage storage{
storage::Config{.durability{.storage_directory = FLAGS_data_directory,
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::DISABLED}}};
memgraph::query::InterpreterContext interpreter_context{
&storage,
{.query = {.allow_load_csv = false},
.execution_timeout_sec = 0,
.replication_replica_check_frequency = std::chrono::seconds(0),
.default_kafka_bootstrap_servers = "",
.default_pulsar_service_url = "",
.stream_transaction_conflict_retries = 0,
.stream_transaction_retry_interval = std::chrono::milliseconds(0)},
FLAGS_data_directory};
const auto init_start = std::chrono::high_resolution_clock::now();
RunInitQueries(interpreter_context, index_queries);
RunInitQueries(interpreter_context, init_queries);
const auto benchmark_start = std::chrono::high_resolution_clock::now();
spdlog::critical("Read: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(init_start - run_start).count());
spdlog::critical("Init: {}ms",
std::chrono::duration_cast<std::chrono::milliseconds>(benchmark_start - init_start).count());
std::map<std::string, std::chrono::nanoseconds> benchmark_results;
for (const auto &[name, queries] : benchmarks) {
const auto current_start = std::chrono::high_resolution_clock::now();
RunBenchmarkQueries(interpreter_context, queries);
const auto current_stop = std::chrono::high_resolution_clock::now();
const auto elapsed = current_stop - current_start;
spdlog::critical("Benchmark {}: {}ms", name,
std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count());
benchmark_results.emplace(name, elapsed);
}
const auto benchmark_end = std::chrono::high_resolution_clock::now();
spdlog::critical("Benchmark: {}ms",
std::chrono::duration_cast<std::chrono::milliseconds>(benchmark_end - benchmark_start).count());
if (!FLAGS_export_json_results.empty()) {
nlohmann::json results;
PutResult(results, "init", benchmark_start - init_start);
nlohmann::json benchmark_results_json;
for (const auto &[name, duration] : benchmark_results) {
PutResult(benchmark_results_json, name, duration);
}
results["benchmarks"] = std::move(benchmark_results_json);
std::ofstream results_file{FLAGS_export_json_results};
results_file << results.dump();
}
}
void RunV3() {
spdlog::critical("Running V3");
const auto run_start = std::chrono::high_resolution_clock::now();
std::ifstream sm_file{FLAGS_split_file, std::ios::in};
MG_ASSERT(sm_file.good(), "Cannot open split file to read: {}", FLAGS_split_file);
auto sm = memgraph::coordinator::ShardMap::Parse(sm_file);
const auto init_queries = ReadQueries(FLAGS_init_queries_file);
const auto benchmarks = ReadBenchmarkQueries(FLAGS_benchmark_queries_files);
io::local_transport::LocalSystem ls;
auto unique_local_addr_query = io::Address::UniqueLocalAddress();
auto io = ls.Register(unique_local_addr_query);
memgraph::machine_manager::MachineConfig config{
.coordinator_addresses = std::vector<memgraph::io::Address>{unique_local_addr_query},
.is_storage = true,
.is_coordinator = true,
.listen_ip = unique_local_addr_query.last_known_ip,
.listen_port = unique_local_addr_query.last_known_port,
.shard_worker_threads = 2,
};
memgraph::coordinator::Coordinator coordinator{sm};
memgraph::machine_manager::MachineManager<memgraph::io::local_transport::LocalTransport> mm{io, config, coordinator};
std::jthread mm_thread([&mm] { mm.Run(); });
auto rr_factory = std::make_unique<memgraph::query::v2::LocalRequestRouterFactory>(io);
query::v2::InterpreterContext interpreter_context{(memgraph::storage::v3::Shard *)(nullptr),
{.execution_timeout_sec = 0},
"data",
std::move(rr_factory),
mm.CoordinatorAddress()};
// without this it fails sometimes because the CreateVertices request might reach the shard worker faster than the
// ShardToInitialize
std::this_thread::sleep_for(std::chrono::milliseconds(150));
const auto init_start = std::chrono::high_resolution_clock::now();
RunInitQueries(interpreter_context, init_queries);
const auto benchmark_start = std::chrono::high_resolution_clock::now();
spdlog::critical("Read: {}ms", std::chrono::duration_cast<std::chrono::milliseconds>(init_start - run_start).count());
spdlog::critical("Init: {}ms",
std::chrono::duration_cast<std::chrono::milliseconds>(benchmark_start - init_start).count());
std::map<std::string, std::chrono::nanoseconds> benchmark_results;
for (const auto &[name, queries] : benchmarks) {
const auto current_start = std::chrono::high_resolution_clock::now();
RunBenchmarkQueries(interpreter_context, queries);
const auto current_stop = std::chrono::high_resolution_clock::now();
const auto elapsed = current_stop - current_start;
spdlog::critical("Benchmark {}: {}ms", name,
std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count());
benchmark_results.emplace(name, elapsed);
}
const auto benchmark_end = std::chrono::high_resolution_clock::now();
spdlog::critical("Benchmark: {}ms",
std::chrono::duration_cast<std::chrono::milliseconds>(benchmark_end - benchmark_start).count());
ls.ShutDown();
auto latency_histograms = nlohmann::json::parse(fmt::format("{}", io.ResponseLatencies()));
spdlog::warn(latency_histograms.dump(4));
if (!FLAGS_export_json_results.empty()) {
nlohmann::json results;
PutResult(results, "init", benchmark_start - init_start);
nlohmann::json benchmark_results_json;
for (const auto &[name, duration] : benchmark_results) {
PutResult(benchmark_results_json, name, duration);
}
results["benchmarks"] = std::move(benchmark_results_json);
results["latencies"] = std::move(latency_histograms);
std::ofstream results_file{FLAGS_export_json_results};
results_file << results.dump();
}
}
} // namespace memgraph::tests::manual
int main(int argc, char **argv) {
spdlog::set_level(spdlog::level::warn);
spdlog::cfg::load_env_levels();
gflags::ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_use_v3) {
memgraph::tests::manual::RunV3();
} else {
memgraph::tests::manual::RunV2();
}
return 0;
}

View File

@@ -1,116 +0,0 @@
#!/usr/bin/env python3
# Copyright 2023 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import argparse
import io
import json
import os
import subprocess
import tarfile
import tempfile
import requests
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
BUILD_DIR = os.path.join(PROJECT_DIR, "build")
BINARY_DIR = os.path.join(BUILD_DIR, "tests/manual")
DEFAULT_BENCHMARK_DIR = os.path.join(BINARY_DIR, "query_performance_benchmark")
DATA_URL = (
"https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/dataset/query_performance/query_performance_benchmark.tar.gz"
)
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
"--binary",
type=str,
default=os.path.join(BINARY_DIR, "query_performance"),
help="Path to the binary to use for the benchmark.",
)
parser.add_argument(
"--data-dir",
type=str,
default=tempfile.TemporaryDirectory().name,
help="Path to directory that can be used as a data directory for ",
)
parser.add_argument(
"--summary-path",
type=str,
default=os.path.join(DEFAULT_BENCHMARK_DIR, "summary.json"),
help="Path to which file write the summary.",
)
parser.add_argument("--init-queries-file", type=str, default=os.path.join(DEFAULT_BENCHMARK_DIR, "dataset.cypher"))
parser.add_argument("--index-queries-file", type=str, default=os.path.join(DEFAULT_BENCHMARK_DIR, "indices.cypher"))
parser.add_argument("--split-file", type=str, default=os.path.join(DEFAULT_BENCHMARK_DIR, "split_file"))
parser.add_argument(
"--benchmark-queries-files",
type=str,
default=",".join(
[os.path.join(DEFAULT_BENCHMARK_DIR, file_name) for file_name in ["expand.cypher", "match_files.cypher"]]
),
)
args = parser.parse_args()
v2_results_path = os.path.join(DEFAULT_BENCHMARK_DIR, "v2_results.json")
v3_results_path = os.path.join(DEFAULT_BENCHMARK_DIR, "v3_results.json")
if os.path.exists(DEFAULT_BENCHMARK_DIR):
print(f"Using cachced data from {DEFAULT_BENCHMARK_DIR}")
else:
print(f"Downloading benchmark data to {DEFAULT_BENCHMARK_DIR}")
r = requests.get(DATA_URL)
assert r.ok, "Cannot download data"
file_like_object = io.BytesIO(r.content)
tar = tarfile.open(fileobj=file_like_object)
tar.extractall(os.path.dirname(DEFAULT_BENCHMARK_DIR))
subprocess.run(
[
args.binary,
f"--split-file={args.split_file}",
f"--index-queries-file={args.index_queries_file}",
f"--init-queries-file={args.init_queries_file}",
f"--benchmark-queries-files={args.benchmark_queries_files}",
"--use-v3=false",
"--use-multi-frame=true",
f"--export-json-results={v2_results_path}",
f"--data-directory={args.data_dir}",
]
)
subprocess.run(
[
args.binary,
f"--split-file={args.split_file}",
f"--index-queries-file={args.index_queries_file}",
f"--init-queries-file={args.init_queries_file}",
f"--benchmark-queries-files={args.benchmark_queries_files}",
"--use-v3=true",
"--use-multi-frame=true",
f"--export-json-results={v3_results_path}",
f"--data-directory={args.data_dir}",
]
)
v2_results_file = open(v2_results_path)
v2_results = json.load(v2_results_file)
v3_results_file = open(v3_results_path)
v3_results = json.load(v3_results_file)
with open(args.summary_path, "w") as summary:
json.dump({"v2": v2_results, "v3": v3_results}, summary)

View File

@@ -51,22 +51,10 @@ import helpers
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--number_of_identities",
type=int,
default=10,
help="Determines how many :Identity nodes will the dataset contain.",
)
parser.add_argument(
"--number_of_files", type=int, default=10, help="Determines how many :File nodes will the dataset contain."
)
parser.add_argument(
"--percentage_of_permissions",
type=float,
default=1.0,
help="Determines approximately what percentage of the all possible identity-permission-file connections will be created.",
)
parser.add_argument("--filename", default="dataset.cypher", help="The name of the output file.")
parser.add_argument("--number_of_identities", type=int, default=10)
parser.add_argument("--number_of_files", type=int, default=10)
parser.add_argument("--percentage_of_permissions", type=float, default=1.0)
parser.add_argument("--filename", default="dataset.cypher")
args = parser.parse_args()

View File

@@ -51,22 +51,10 @@ import helpers
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--number_of_identities",
type=int,
default=10,
help="Determines how many :Identity nodes will the dataset contain.",
)
parser.add_argument(
"--number_of_files", type=int, default=10, help="Determines how many :File nodes will the dataset contain."
)
parser.add_argument(
"--percentage_of_permissions",
type=float,
default=1.0,
help="Determines approximately what percentage of the all possible identity-permission-file connections will be created.",
)
parser.add_argument("--filename", default="dataset.cypher", help="The name of the output file.")
parser.add_argument("--number_of_identities", type=int, default=10)
parser.add_argument("--number_of_files", type=int, default=10)
parser.add_argument("--percentage_of_permissions", type=float, default=1.0)
parser.add_argument("--filename", default="dataset.cypher")
args = parser.parse_args()

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -55,7 +55,7 @@ void run_server(Io<SimulatorTransport> io) {
highest_seen = std::max(highest_seen, req.proposal);
auto srv_res = CounterResponse{highest_seen};
io.Send(request_envelope.from_address, request_envelope.request_id, std::move(srv_res));
io.Send(request_envelope.from_address, request_envelope.request_id, srv_res);
}
}
@@ -76,7 +76,7 @@ std::pair<SimulatorStats, LatencyHistogramSummaries> RunWorkload(SimulatorConfig
CounterRequest cli_req;
cli_req.proposal = i;
spdlog::info("[CLIENT] calling Request");
auto res_f = cli_io.Request<CounterResponse, CounterRequest>(srv_addr, std::move(cli_req));
auto res_f = cli_io.Request<CounterRequest, CounterResponse>(srv_addr, cli_req);
spdlog::info("[CLIENT] calling Wait");
auto res_rez = std::move(res_f).Wait();
spdlog::info("[CLIENT] Wait returned");

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -44,7 +44,7 @@ void run_server(Io<SimulatorTransport> io) {
for (auto index = start_index; index < start_index + req.count; ++index) {
response.vertices.push_back({std::string("Vertex_") + std::to_string(index)});
}
io.Send(request_envelope.from_address, request_envelope.request_id, std::move(response));
io.Send(request_envelope.from_address, request_envelope.request_id, response);
}
}
@@ -78,7 +78,7 @@ int main() {
auto req = ScanVerticesRequest{2, std::nullopt};
auto res_f = cli_io.Request<VerticesResponse, ScanVerticesRequest>(srv_addr, std::move(req));
auto res_f = cli_io.Request<ScanVerticesRequest, VerticesResponse>(srv_addr, req);
auto res_rez = std::move(res_f).Wait();
simulator.ShutDown();
return 0;

View File

@@ -294,9 +294,6 @@ target_link_libraries(${test_prefix}storage_v3_expr mg-storage-v3 mg-expr)
add_unit_test(storage_v3_schema.cpp)
target_link_libraries(${test_prefix}storage_v3_schema mg-storage-v3)
add_unit_test(storage_v3_shard_split.cpp)
target_link_libraries(${test_prefix}storage_v3_shard_split mg-storage-v3 mg-query-v2)
# Test mg-query-v2
# These are commented out because of the new TypedValue in the query engine
# add_unit_test(query_v2_interpreter.cpp ${CMAKE_SOURCE_DIR}/src/glue/v2/communication.cpp)

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -48,7 +48,7 @@ void RunServer(Io<LocalTransport> io) {
highest_seen = std::max(highest_seen, req.proposal);
auto srv_res = CounterResponse{highest_seen};
io.Send(request_envelope.from_address, request_envelope.request_id, std::move(srv_res));
io.Send(request_envelope.from_address, request_envelope.request_id, srv_res);
}
}
@@ -70,7 +70,7 @@ TEST(LocalTransport, BasicRequest) {
auto value = 1; // i;
cli_req.proposal = value;
spdlog::info("[CLIENT] sending request");
auto res_f = cli_io.Request<CounterResponse, CounterRequest>(srv_addr, std::move(cli_req));
auto res_f = cli_io.Request<CounterRequest, CounterResponse>(srv_addr, cli_req);
spdlog::info("[CLIENT] waiting on future");
auto res_rez = std::move(res_f).Wait();

View File

@@ -45,6 +45,10 @@ class MockedRequestRouter : public RequestRouterInterface {
MOCK_METHOD((std::optional<std::pair<uint64_t, uint64_t>>), AllocateInitialEdgeIds, (io::Address));
MOCK_METHOD(void, InstallSimulatorTicker, (std::function<bool()>));
MOCK_METHOD(const std::vector<coordinator::SchemaProperty> &, GetSchemaForLabel, (storage::v3::LabelId), (const));
MOCK_METHOD(int64_t, GetApproximateVertexCount, (), (const));
MOCK_METHOD(int64_t, GetApproximateVertexCount, (storage::v3::LabelId label), (const));
MOCK_METHOD(int64_t, GetApproximateVertexCount, (storage::v3::LabelId label, storage::v3::PropertyId property),
(const));
};
class MockedLogicalOperator : public plan::LogicalOperator {

View File

@@ -31,8 +31,8 @@ MultiFrame CreateMultiFrame(const size_t max_pos, const Symbol &src, const Symbo
auto frames_populator = multi_frame.GetInvalidFramesPopulator();
size_t i = 0;
for (auto &frame : frames_populator) {
auto &src_acc = frame.At(src);
auto &dst_acc = frame.At(dst);
auto &src_acc = frame.at(src);
auto &dst_acc = frame.at(dst);
auto v1 = msgs::Vertex{.id = {{msgs::LabelId::FromUint(1)}, {msgs::Value(static_cast<int64_t>(i++))}}};
auto v2 = msgs::Vertex{.id = {{msgs::LabelId::FromUint(1)}, {msgs::Value(static_cast<int64_t>(i++))}}};
std::map<msgs::PropertyId, msgs::Value> mp;

View File

@@ -135,6 +135,13 @@ class MockedRequestRouter : public RequestRouterInterface {
return schema;
};
// TODO(gvolfing) once the real implementation is done make sure these are solved as well.
int64_t GetApproximateVertexCount() const override { return 1; }
int64_t GetApproximateVertexCount(storage::v3::LabelId label) const override { return 1; }
int64_t GetApproximateVertexCount(storage::v3::LabelId label, storage::v3::PropertyId property) const override {
return 1;
}
private:
void SetUpNameIdMappers() {
std::unordered_map<uint64_t, std::string> id_to_name;

View File

@@ -1,501 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include <memory>
#include <gmock/gmock-matchers.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "coordinator/hybrid_logical_clock.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/delta.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
using testing::Pair;
using testing::UnorderedElementsAre;
namespace memgraph::storage::v3::tests {
class ShardSplitTest : public testing::Test {
protected:
void SetUp() override {
storage.StoreMapping(
{{1, "label"}, {2, "property"}, {3, "edge_property"}, {4, "secondary_label"}, {5, "secondary_prop"}});
}
const PropertyId primary_property{PropertyId::FromUint(2)};
const PropertyId secondary_property{PropertyId::FromUint(5)};
std::vector<storage::v3::SchemaProperty> schema_property_vector = {
storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}};
const std::vector<PropertyValue> min_pk{PropertyValue{0}};
const LabelId primary_label{LabelId::FromUint(1)};
const LabelId secondary_label{LabelId::FromUint(4)};
const EdgeTypeId edge_type_id{EdgeTypeId::FromUint(3)};
Shard storage{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
coordinator::Hlc last_hlc{0, io::Time{}};
coordinator::Hlc GetNextHlc() {
++last_hlc.logical_id;
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
return last_hlc;
}
void AssertShardState(auto &shard, const int split_min, const int split_max) {
auto acc = shard.Access(GetNextHlc());
for (int i{0}; i < split_min; ++i) {
EXPECT_FALSE(acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD).has_value());
}
for (int i{split_min}; i < split_max; ++i) {
const auto vtx = acc.FindVertex(PrimaryKey{{PropertyValue(i)}}, View::OLD);
ASSERT_TRUE(vtx.has_value());
EXPECT_TRUE(vtx->InEdges(View::OLD)->size() == 1 || vtx->OutEdges(View::OLD)->size() == 1);
}
}
};
void AssertEqVertexContainer(const VertexContainer &actual, const VertexContainer &expected) {
ASSERT_EQ(actual.size(), expected.size());
auto expected_it = expected.begin();
auto actual_it = actual.begin();
while (expected_it != expected.end()) {
EXPECT_EQ(actual_it->first, expected_it->first);
EXPECT_EQ(actual_it->second.deleted, expected_it->second.deleted);
EXPECT_EQ(actual_it->second.labels, expected_it->second.labels);
auto *expected_delta = expected_it->second.delta;
auto *actual_delta = actual_it->second.delta;
// This asserts delta chain
while (expected_delta != nullptr) {
EXPECT_EQ(actual_delta->action, expected_delta->action);
EXPECT_EQ(actual_delta->id, expected_delta->id);
switch (expected_delta->action) {
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL: {
EXPECT_EQ(actual_delta->label, expected_delta->label);
break;
}
case Delta::Action::SET_PROPERTY: {
EXPECT_EQ(actual_delta->property.key, expected_delta->property.key);
EXPECT_EQ(actual_delta->property.value, expected_delta->property.value);
break;
}
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::RECREATE_OBJECT:
case Delta::Action::DELETE_OBJECT:
case Delta::Action::REMOVE_OUT_EDGE: {
break;
}
}
const auto expected_prev = expected_delta->prev.Get();
const auto actual_prev = actual_delta->prev.Get();
switch (expected_prev.type) {
case PreviousPtr::Type::NULLPTR: {
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::NULLPTR) << "Expected type is nullptr!";
break;
}
case PreviousPtr::Type::DELTA: {
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::DELTA) << "Expected type is delta!";
EXPECT_EQ(actual_prev.delta->action, expected_prev.delta->action);
EXPECT_EQ(actual_prev.delta->id, expected_prev.delta->id);
break;
}
case v3::PreviousPtr::Type::EDGE: {
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::EDGE) << "Expected type is edge!";
EXPECT_EQ(actual_prev.edge->gid, expected_prev.edge->gid);
break;
}
case v3::PreviousPtr::Type::VERTEX: {
ASSERT_EQ(actual_prev.type, PreviousPtr::Type::VERTEX) << "Expected type is vertex!";
EXPECT_EQ(actual_prev.vertex->first, expected_prev.vertex->first);
break;
}
}
expected_delta = expected_delta->next;
actual_delta = actual_delta->next;
}
EXPECT_EQ(expected_delta, nullptr);
EXPECT_EQ(actual_delta, nullptr);
++expected_it;
++actual_it;
}
}
void AssertEqDeltaLists(const std::list<Delta> &actual, const std::list<Delta> &expected) {
EXPECT_EQ(actual.size(), expected.size());
auto actual_it = actual.begin();
auto expected_it = expected.begin();
while (actual_it != actual.end()) {
EXPECT_EQ(actual_it->id, expected_it->id);
EXPECT_EQ(actual_it->action, expected_it->action);
++actual_it;
++expected_it;
}
}
void AddDeltaToDeltaChain(Vertex *object, Delta *new_delta) {
auto *delta_holder = GetDeltaHolder(object);
new_delta->next = delta_holder->delta;
new_delta->prev.Set(object);
if (delta_holder->delta) {
delta_holder->delta->prev.Set(new_delta);
}
delta_holder->delta = new_delta;
}
TEST_F(ShardSplitTest, TestBasicSplitWithVertices) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(121)}})
.HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
auto current_hlc = GetNextHlc();
acc.Commit(current_hlc);
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 0);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 4, 1};
Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 5, 2};
Delta delta_remove_label{Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4};
Delta delta_set_property{Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4};
Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 8, 3};
VertexContainer expected_vertices;
auto [it_4, inserted1] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
delta_delete1.prev.Set(&*it_4);
auto [it_5, inserted2] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
delta_delete2.prev.Set(&*it_5);
auto [it_6, inserted3] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
delta_delete3.prev.Set(&*it_6);
it_5->second.labels.push_back(secondary_label);
AddDeltaToDeltaChain(&*it_5, &delta_set_property);
AddDeltaToDeltaChain(&*it_5, &delta_remove_label);
AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
// This is to ensure that the transaction that we have don't point to invalid
// object on the other shard
std::list<Delta> expected_deltas;
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 4, 1);
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 5, 2);
expected_deltas.emplace_back(Delta::SetPropertyTag{}, secondary_property, PropertyValue(), &commit_info, 6, 4);
expected_deltas.emplace_back(Delta::RemoveLabelTag{}, secondary_label, &commit_info, 7, 4);
expected_deltas.emplace_back(Delta::DeleteObjectTag{}, &commit_info, 8, 3);
AssertEqDeltaLists(splitted_data.transactions.begin()->second->deltas, expected_deltas);
}
TEST_F(ShardSplitTest, TestBasicSplitVerticesAndEdges) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
.HasError());
auto current_hlc = GetNextHlc();
acc.Commit(current_hlc);
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 2);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
CommitInfo commit_info{.start_or_commit_timestamp = current_hlc};
Delta delta_delete1{Delta::DeleteObjectTag{}, &commit_info, 12, 1};
Delta delta_delete2{Delta::DeleteObjectTag{}, &commit_info, 13, 1};
Delta delta_delete3{Delta::DeleteObjectTag{}, &commit_info, 14, 1};
Delta delta_add_in_edge1{Delta::RemoveInEdgeTag{},
edge_type_id,
VertexId{primary_label, {PropertyValue(1)}},
EdgeRef{Gid::FromUint(1)},
&commit_info,
17,
1};
Delta delta_add_out_edge2{Delta::RemoveOutEdgeTag{},
edge_type_id,
VertexId{primary_label, {PropertyValue(6)}},
EdgeRef{Gid::FromUint(2)},
&commit_info,
19,
1};
Delta delta_add_in_edge2{Delta::RemoveInEdgeTag{},
edge_type_id,
VertexId{primary_label, {PropertyValue(4)}},
EdgeRef{Gid::FromUint(2)},
&commit_info,
20,
1};
VertexContainer expected_vertices;
auto [vtx4, inserted4] = expected_vertices.emplace(PrimaryKey{PropertyValue{4}}, VertexData(&delta_delete1));
auto [vtx5, inserted5] = expected_vertices.emplace(PrimaryKey{PropertyValue{5}}, VertexData(&delta_delete2));
auto [vtx6, inserted6] = expected_vertices.emplace(PrimaryKey{PropertyValue{6}}, VertexData(&delta_delete3));
AddDeltaToDeltaChain(&*vtx4, &delta_add_out_edge2);
AddDeltaToDeltaChain(&*vtx5, &delta_add_in_edge1);
AddDeltaToDeltaChain(&*vtx6, &delta_add_in_edge2);
AssertEqVertexContainer(splitted_data.vertices, expected_vertices);
}
TEST_F(ShardSplitTest, TestBasicSplitBeforeCommit) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
.HasError());
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 2);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
}
TEST_F(ShardSplitTest, TestBasicSplitWithCommitedAndOngoingTransactions) {
{
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
acc.Commit(GetNextHlc());
}
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
.HasError());
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 2);
EXPECT_EQ(splitted_data.transactions.size(), 2);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
}
TEST_F(ShardSplitTest, TestBasicSplitWithLabelIndex) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {}).HasError());
acc.Commit(GetNextHlc());
storage.CreateIndex(secondary_label);
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 0);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 1);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
}
TEST_F(ShardSplitTest, TestBasicSplitWithLabelPropertyIndex) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(1)}, {{secondary_property, PropertyValue(1)}})
.HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(5)}, {{secondary_property, PropertyValue(21)}})
.HasError());
EXPECT_FALSE(
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(6)}, {{secondary_property, PropertyValue(22)}})
.HasError());
acc.Commit(GetNextHlc());
storage.CreateIndex(secondary_label, secondary_property);
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 0);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
}
TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroyOriginalShard) {
const auto split_value{4};
PrimaryKey splitted_value{{PropertyValue(4)}};
std::unique_ptr<Shard> splitted_shard;
{
Shard storage2{primary_label, min_pk, std::nullopt /*max_primary_key*/, schema_property_vector};
auto acc = storage2.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
.HasError());
acc.Commit(GetNextHlc());
auto splitted_data = storage2.PerformSplit({PropertyValue(split_value)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 2);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
// Create a new shard
splitted_shard = Shard::FromSplitData(std::move(splitted_data));
// Call gc on old shard
storage2.CollectGarbage(GetNextHlc().coordinator_wall_clock);
// Destroy original
}
splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
AssertShardState(*splitted_shard, 4, 6);
}
TEST_F(ShardSplitTest, TestSplittingShardsWithGcDestroySplittedShard) {
PrimaryKey splitted_value{{PropertyValue(4)}};
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(1)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(2)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(3)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(4)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(5)}, {}).HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(6)}, {}).HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(1)}},
VertexId{primary_label, PrimaryKey{PropertyValue(2)}}, edge_type_id, Gid::FromUint(0))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(3)}},
VertexId{primary_label, PrimaryKey{PropertyValue(5)}}, edge_type_id, Gid::FromUint(1))
.HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(4)}},
VertexId{primary_label, PrimaryKey{PropertyValue(6)}}, edge_type_id, Gid::FromUint(2))
.HasError());
acc.Commit(GetNextHlc());
auto splitted_data = storage.PerformSplit({PropertyValue(4)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 3);
EXPECT_EQ(splitted_data.edges->size(), 2);
EXPECT_EQ(splitted_data.transactions.size(), 1);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 0);
{
// Create a new shard
auto splitted_shard = Shard::FromSplitData(std::move(splitted_data));
// Call gc on new shard
splitted_shard->CollectGarbage(GetNextHlc().coordinator_wall_clock);
// Destroy splitted shard
}
storage.CollectGarbage(GetNextHlc().coordinator_wall_clock);
AssertShardState(storage, 1, 3);
}
TEST_F(ShardSplitTest, TestBigSplit) {
int pk{0};
for (int64_t i{0}; i < 10'000; ++i) {
auto acc = storage.Access(GetNextHlc());
EXPECT_FALSE(
acc.CreateVertexAndValidate({secondary_label}, {PropertyValue(pk++)}, {{secondary_property, PropertyValue(i)}})
.HasError());
EXPECT_FALSE(acc.CreateVertexAndValidate({}, {PropertyValue(pk++)}, {}).HasError());
EXPECT_FALSE(acc.CreateEdge(VertexId{primary_label, PrimaryKey{PropertyValue(pk - 2)}},
VertexId{primary_label, PrimaryKey{PropertyValue(pk - 1)}}, edge_type_id,
Gid::FromUint(pk))
.HasError());
acc.Commit(GetNextHlc());
}
storage.CreateIndex(secondary_label, secondary_property);
const auto split_value = pk / 2;
auto splitted_data = storage.PerformSplit({PropertyValue(split_value)}, 2);
EXPECT_EQ(splitted_data.vertices.size(), 10000);
EXPECT_EQ(splitted_data.edges->size(), 5000);
EXPECT_EQ(splitted_data.transactions.size(), 5000);
EXPECT_EQ(splitted_data.label_indices.size(), 0);
EXPECT_EQ(splitted_data.label_property_indices.size(), 1);
auto shard = Shard::FromSplitData(std::move(splitted_data));
AssertShardState(*shard, split_value, split_value * 2);
}
} // namespace memgraph::storage::v3::tests