Compare commits
15 Commits
accessors-
...
disk-stora
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c785bed20c | ||
|
|
b34044459e | ||
|
|
a22b011736 | ||
|
|
876359f4b6 | ||
|
|
c441b7de0b | ||
|
|
b6beffa9e2 | ||
|
|
1bed62da4a | ||
|
|
2019f5cc72 | ||
|
|
f070329e77 | ||
|
|
b8d956294b | ||
|
|
72371cc8ee | ||
|
|
e193d4036e | ||
|
|
ceb3e67f8e | ||
|
|
f7c1923ab2 | ||
|
|
7503b304f2 |
@@ -98,6 +98,8 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
#include "audit/log.hpp"
|
||||
#endif
|
||||
// Disk storage includes
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
|
||||
constexpr const char *kMgUser = "MEMGRAPH_USER";
|
||||
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
|
||||
@@ -773,6 +775,7 @@ int main(int argc, char **argv) {
|
||||
auto gil = memgraph::py::EnsureGIL();
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
auto *flag = PyLong_FromLong(RTLD_NOW | RTLD_DEEPBIND);
|
||||
// auto *flag = PyLong_FromLong(RTLD_NOW);
|
||||
auto *setdl = PySys_GetObject("setdlopenflags");
|
||||
MG_ASSERT(setdl);
|
||||
auto *arg = PyTuple_New(1);
|
||||
@@ -907,10 +910,15 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
|
||||
}
|
||||
// here in the future, a specific instantiation of storage type will be created
|
||||
memgraph::storage::Storage db(db_config);
|
||||
// for experiments, I will first bind together both storages to make sure we solve serialization correctly
|
||||
// if we decide to use dynamic polymorphism, the concrete storage object should care about RocksDB details
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db;
|
||||
|
||||
memgraph::query::InterpreterContext interpreter_context{
|
||||
&db,
|
||||
&disk_db,
|
||||
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
|
||||
.execution_timeout_sec = FLAGS_query_execution_timeout_sec,
|
||||
.replication_replica_check_frequency = std::chrono::seconds(FLAGS_replication_replica_check_frequency_sec),
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include "query/parameters.hpp"
|
||||
#include "query/plan/profile.hpp"
|
||||
#include "query/trigger.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "utils/async_timer.hpp"
|
||||
|
||||
namespace memgraph::query {
|
||||
@@ -86,6 +87,7 @@ struct ExecutionContext {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::unique_ptr<FineGrainedAuthChecker> auth_checker{nullptr};
|
||||
#endif
|
||||
storage::rocks::RocksDBStorage *disk_db;
|
||||
};
|
||||
|
||||
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -112,16 +112,16 @@ query::Graph *SubgraphDbAccessor::getGraph() { return graph_; }
|
||||
VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; }
|
||||
|
||||
auto SubgraphVertexAccessor::OutEdges(storage::View view) const -> decltype(impl_.OutEdges(view)) {
|
||||
auto maybe_edges = impl_.impl_->OutEdges(view, {});
|
||||
auto maybe_edges = impl_.impl_.OutEdges(view, {});
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
auto edges = std::move(*maybe_edges);
|
||||
const auto &graph_edges = graph_->edges();
|
||||
|
||||
std::vector<std::unique_ptr<storage::EdgeAccessor>> filteredOutEdges;
|
||||
std::vector<storage::EdgeAccessor> filteredOutEdges;
|
||||
for (auto &edge : edges) {
|
||||
auto edge_q = EdgeAccessor(edge);
|
||||
if (graph_edges.contains(edge_q)) {
|
||||
filteredOutEdges.push_back(std::move(edge));
|
||||
filteredOutEdges.push_back(edge);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,16 +129,16 @@ auto SubgraphVertexAccessor::OutEdges(storage::View view) const -> decltype(impl
|
||||
}
|
||||
|
||||
auto SubgraphVertexAccessor::InEdges(storage::View view) const -> decltype(impl_.InEdges(view)) {
|
||||
auto maybe_edges = impl_.impl_->InEdges(view, {});
|
||||
auto maybe_edges = impl_.impl_.InEdges(view, {});
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
auto edges = std::move(*maybe_edges);
|
||||
const auto &graph_edges = graph_->edges();
|
||||
|
||||
std::vector<std::unique_ptr<storage::EdgeAccessor>> filteredOutEdges;
|
||||
std::vector<storage::EdgeAccessor> filteredOutEdges;
|
||||
for (auto &edge : edges) {
|
||||
auto edge_q = EdgeAccessor(edge);
|
||||
if (graph_edges.contains(edge_q)) {
|
||||
filteredOutEdges.push_back(std::move(edge));
|
||||
filteredOutEdges.push_back(edge);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
|
||||
#include <cppitertools/filter.hpp>
|
||||
#include <cppitertools/imap.hpp>
|
||||
@@ -36,7 +35,6 @@
|
||||
// This cannot be avoided by simple include orderings so we
|
||||
// simply undefine those macros as we're sure that libkrb5
|
||||
// won't and can't be used anywhere in the query engine.
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
|
||||
#undef FALSE
|
||||
@@ -54,28 +52,31 @@ class VertexAccessor;
|
||||
|
||||
class EdgeAccessor final {
|
||||
public:
|
||||
storage::EdgeAccessor *impl_;
|
||||
storage::EdgeAccessor impl_;
|
||||
|
||||
public:
|
||||
explicit EdgeAccessor(std::unique_ptr<storage::EdgeAccessor> &impl) : impl_(impl.get()) {}
|
||||
explicit EdgeAccessor(storage::EdgeAccessor *impl) : impl_(impl) {}
|
||||
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
|
||||
|
||||
bool IsVisible(storage::View view) const { return impl_->IsVisible(view); }
|
||||
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
|
||||
|
||||
storage::EdgeTypeId EdgeType() const { return impl_->EdgeType(); }
|
||||
storage::EdgeTypeId EdgeType() const { return impl_.EdgeType(); }
|
||||
|
||||
auto Properties(storage::View view) const { return impl_->Properties(view); }
|
||||
auto Properties(storage::View view) const { return impl_.Properties(view); }
|
||||
|
||||
auto PropertyStore() const { return impl_.PropertyStore(); }
|
||||
|
||||
void SetPropertyStore(const std::string_view buffer) const { impl_.SetPropertyStore(buffer); }
|
||||
|
||||
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
|
||||
return impl_->GetProperty(key, view);
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::Result<storage::PropertyValue> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
|
||||
return impl_->SetProperty(key, value);
|
||||
return impl_.SetProperty(key, value);
|
||||
}
|
||||
|
||||
storage::Result<bool> InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
return impl_->InitProperties(properties);
|
||||
return impl_.InitProperties(properties);
|
||||
}
|
||||
|
||||
storage::Result<storage::PropertyValue> RemoveProperty(storage::PropertyId key) {
|
||||
@@ -83,7 +84,7 @@ class EdgeAccessor final {
|
||||
}
|
||||
|
||||
storage::Result<std::map<storage::PropertyId, storage::PropertyValue>> ClearProperties() {
|
||||
return impl_->ClearProperties();
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
VertexAccessor To() const;
|
||||
@@ -92,9 +93,11 @@ class EdgeAccessor final {
|
||||
|
||||
bool IsCycle() const;
|
||||
|
||||
int64_t CypherId() const { return impl_->Gid().AsInt(); }
|
||||
int64_t CypherId() const { return impl_.Gid().AsInt(); }
|
||||
|
||||
storage::Gid Gid() const noexcept { return impl_->Gid(); }
|
||||
storage::Gid Gid() const noexcept { return impl_.Gid(); }
|
||||
|
||||
void SetGid(storage::Gid gid_) { impl_.SetGid(gid_); }
|
||||
|
||||
bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; }
|
||||
|
||||
@@ -103,37 +106,41 @@ class EdgeAccessor final {
|
||||
|
||||
class VertexAccessor final {
|
||||
public:
|
||||
storage::VertexAccessor *impl_;
|
||||
storage::VertexAccessor impl_;
|
||||
|
||||
static EdgeAccessor MakeEdgeAccessor(std::unique_ptr<storage::EdgeAccessor> &impl) { return EdgeAccessor(impl); }
|
||||
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
|
||||
|
||||
public:
|
||||
explicit VertexAccessor(storage::VertexAccessor *impl) : impl_(impl) {}
|
||||
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {}
|
||||
|
||||
bool IsVisible(storage::View view) const { return impl_->IsVisible(view); }
|
||||
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
|
||||
|
||||
auto Labels(storage::View view) const { return impl_->Labels(view); }
|
||||
auto Labels(storage::View view) const { return impl_.Labels(view); }
|
||||
|
||||
storage::Result<bool> AddLabel(storage::LabelId label) { return impl_->AddLabel(label); }
|
||||
storage::Result<bool> AddLabel(storage::LabelId label) { return impl_.AddLabel(label); }
|
||||
|
||||
storage::Result<bool> RemoveLabel(storage::LabelId label) { return impl_->RemoveLabel(label); }
|
||||
storage::Result<bool> RemoveLabel(storage::LabelId label) { return impl_.RemoveLabel(label); }
|
||||
|
||||
storage::Result<bool> HasLabel(storage::View view, storage::LabelId label) const {
|
||||
return impl_->HasLabel(label, view);
|
||||
return impl_.HasLabel(label, view);
|
||||
}
|
||||
|
||||
auto Properties(storage::View view) const { return impl_->Properties(view); }
|
||||
auto Properties(storage::View view) const { return impl_.Properties(view); }
|
||||
|
||||
auto PropertyStore() const { return impl_.PropertyStore(); }
|
||||
|
||||
void SetPropertyStore(const std::string_view buffer) const { impl_.SetPropertyStore(buffer); }
|
||||
|
||||
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
|
||||
return impl_->GetProperty(key, view);
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::Result<storage::PropertyValue> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
|
||||
return impl_->SetProperty(key, value);
|
||||
return impl_.SetProperty(key, value);
|
||||
}
|
||||
|
||||
storage::Result<bool> InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
return impl_->InitProperties(properties);
|
||||
return impl_.InitProperties(properties);
|
||||
}
|
||||
|
||||
storage::Result<storage::PropertyValue> RemoveProperty(storage::PropertyId key) {
|
||||
@@ -141,12 +148,12 @@ class VertexAccessor final {
|
||||
}
|
||||
|
||||
storage::Result<std::map<storage::PropertyId, storage::PropertyValue>> ClearProperties() {
|
||||
return impl_->ClearProperties();
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *(impl_->InEdges(view))))> {
|
||||
auto maybe_edges = impl_->InEdges(view, edge_types);
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
@@ -154,15 +161,15 @@ class VertexAccessor final {
|
||||
auto InEdges(storage::View view) const { return InEdges(view, {}); }
|
||||
|
||||
auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest) const
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *(impl_->InEdges(view))))> {
|
||||
auto maybe_edges = impl_->InEdges(view, edge_types, dest.impl_);
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
|
||||
auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *(impl_->OutEdges(view))))> {
|
||||
auto maybe_edges = impl_->OutEdges(view, edge_types);
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
@@ -171,19 +178,21 @@ class VertexAccessor final {
|
||||
|
||||
auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
|
||||
const VertexAccessor &dest) const
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *(impl_->OutEdges(view))))> {
|
||||
auto maybe_edges = impl_->OutEdges(view, edge_types, dest.impl_);
|
||||
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
|
||||
storage::Result<size_t> InDegree(storage::View view) const { return impl_->InDegree(view); }
|
||||
storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
|
||||
|
||||
storage::Result<size_t> OutDegree(storage::View view) const { return impl_->OutDegree(view); }
|
||||
storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
|
||||
|
||||
int64_t CypherId() const { return impl_->Gid().AsInt(); }
|
||||
int64_t CypherId() const { return impl_.Gid().AsInt(); }
|
||||
|
||||
storage::Gid Gid() const noexcept { return impl_->Gid(); }
|
||||
storage::Gid Gid() const noexcept { return impl_.Gid(); }
|
||||
|
||||
void SetGid(storage::Gid gid) { impl_.SetGid(gid); }
|
||||
|
||||
bool operator==(const VertexAccessor &v) const noexcept {
|
||||
static_assert(noexcept(impl_ == v.impl_));
|
||||
@@ -193,9 +202,9 @@ class VertexAccessor final {
|
||||
bool operator!=(const VertexAccessor &v) const noexcept { return !(*this == v); }
|
||||
};
|
||||
|
||||
inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_->ToVertex().get()); }
|
||||
inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
|
||||
|
||||
inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_->FromVertex().get()); }
|
||||
inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
|
||||
|
||||
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
|
||||
|
||||
@@ -244,16 +253,12 @@ namespace std {
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::query::VertexAccessor> {
|
||||
size_t operator()(const memgraph::query::VertexAccessor &v) const {
|
||||
return std::hash<std::remove_pointer<decltype(v.impl_)>::type>{}(*v.impl_);
|
||||
}
|
||||
size_t operator()(const memgraph::query::VertexAccessor &v) const { return std::hash<decltype(v.impl_)>{}(v.impl_); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::query::EdgeAccessor> {
|
||||
size_t operator()(const memgraph::query::EdgeAccessor &e) const {
|
||||
return std::hash<std::remove_pointer<decltype(e.impl_)>::type>{}(*e.impl_);
|
||||
}
|
||||
size_t operator()(const memgraph::query::EdgeAccessor &e) const { return std::hash<decltype(e.impl_)>{}(e.impl_); }
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
@@ -325,7 +330,7 @@ class DbAccessor final {
|
||||
|
||||
std::optional<VertexAccessor> FindVertex(storage::Gid gid, storage::View view) {
|
||||
auto maybe_vertex = accessor_->FindVertex(gid, view);
|
||||
if (maybe_vertex) return VertexAccessor(maybe_vertex.get());
|
||||
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -352,17 +357,17 @@ class DbAccessor final {
|
||||
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
|
||||
}
|
||||
|
||||
VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex().get()); }
|
||||
VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); }
|
||||
|
||||
storage::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
const storage::EdgeTypeId &edge_type) {
|
||||
auto maybe_edge = accessor_->CreateEdge(from->impl_, to->impl_, edge_type);
|
||||
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
|
||||
if (maybe_edge.HasError()) return storage::Result<EdgeAccessor>(maybe_edge.GetError());
|
||||
return EdgeAccessor((*maybe_edge).get());
|
||||
return EdgeAccessor(*maybe_edge);
|
||||
}
|
||||
|
||||
storage::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
|
||||
auto res = accessor_->DeleteEdge(edge->impl_);
|
||||
auto res = accessor_->DeleteEdge(&edge->impl_);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
@@ -372,14 +377,14 @@ class DbAccessor final {
|
||||
return std::optional<EdgeAccessor>{};
|
||||
}
|
||||
|
||||
return std::make_optional<EdgeAccessor>(value.get());
|
||||
return std::make_optional<EdgeAccessor>(*value);
|
||||
}
|
||||
|
||||
storage::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
|
||||
VertexAccessor *vertex_accessor) {
|
||||
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
|
||||
|
||||
auto res = accessor_->DetachDeleteVertex(vertex_accessor->impl_);
|
||||
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
@@ -394,13 +399,13 @@ class DbAccessor final {
|
||||
std::vector<EdgeAccessor> deleted_edges;
|
||||
deleted_edges.reserve(edges.size());
|
||||
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
|
||||
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge.get()}; });
|
||||
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
|
||||
|
||||
return std::make_optional<ReturnType>(vertex.get(), std::move(deleted_edges));
|
||||
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
|
||||
}
|
||||
|
||||
storage::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
|
||||
auto res = accessor_->DeleteVertex(vertex_accessor->impl_);
|
||||
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
@@ -410,7 +415,7 @@ class DbAccessor final {
|
||||
return std::optional<VertexAccessor>{};
|
||||
}
|
||||
|
||||
return std::make_optional<VertexAccessor>(value.get());
|
||||
return std::make_optional<VertexAccessor>(*value);
|
||||
}
|
||||
|
||||
storage::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
#include "query/stream/common.hpp"
|
||||
#include "query/trigger.hpp"
|
||||
#include "query/typed_value.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
@@ -1015,6 +1016,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
ctx_.disk_db = interpreter_context->disk_db;
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp = QueryTimestamp();
|
||||
@@ -1127,9 +1129,13 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
|
||||
using RWType = plan::ReadWriteTypeChecker::RWType;
|
||||
} // namespace
|
||||
|
||||
InterpreterContext::InterpreterContext(storage::Storage *db, const InterpreterConfig config,
|
||||
const std::filesystem::path &data_directory)
|
||||
: db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory / "streams"} {}
|
||||
InterpreterContext::InterpreterContext(storage::Storage *db, storage::rocks::RocksDBStorage *disk_db,
|
||||
const InterpreterConfig config, const std::filesystem::path &data_directory)
|
||||
: db(db),
|
||||
disk_db(disk_db),
|
||||
trigger_store(data_directory / "triggers"),
|
||||
config(config),
|
||||
streams{this, data_directory / "streams"} {}
|
||||
|
||||
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
|
||||
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
|
||||
@@ -1147,7 +1153,8 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
in_explicit_transaction_ = true;
|
||||
expect_rollback_ = false;
|
||||
|
||||
db_accessor_ = interpreter_context_->db->Access(GetIsolationLevelOverride());
|
||||
db_accessor_ =
|
||||
std::make_unique<storage::Storage::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
|
||||
execution_db_accessor_.emplace(db_accessor_.get());
|
||||
transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release);
|
||||
|
||||
@@ -2638,7 +2645,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
|
||||
utils::Downcast<TriggerQuery>(parsed_query.query) || utils::Downcast<AnalyzeGraphQuery>(parsed_query.query) ||
|
||||
utils::Downcast<TransactionQueueQuery>(parsed_query.query))) {
|
||||
db_accessor_ = interpreter_context_->db->Access(GetIsolationLevelOverride());
|
||||
db_accessor_ =
|
||||
std::make_unique<storage::Storage::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
|
||||
execution_db_accessor_.emplace(db_accessor_.get());
|
||||
transaction_status_.store(TransactionStatus::ACTIVE, std::memory_order_release);
|
||||
|
||||
@@ -2782,7 +2790,7 @@ void RunTriggersIndividually(const utils::SkipList<Trigger> &triggers, Interpret
|
||||
|
||||
// create a new transaction for each trigger
|
||||
auto storage_acc = interpreter_context->db->Access();
|
||||
DbAccessor db_accessor{storage_acc.get()};
|
||||
DbAccessor db_accessor{&storage_acc};
|
||||
|
||||
trigger_context.AdaptForAccessor(&db_accessor);
|
||||
try {
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
#include "query/stream/streams.hpp"
|
||||
#include "query/trigger.hpp"
|
||||
#include "query/typed_value.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "storage/v2/isolation_level.hpp"
|
||||
#include "utils/event_counter.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -208,10 +209,11 @@ class Interpreter;
|
||||
*
|
||||
*/
|
||||
struct InterpreterContext {
|
||||
explicit InterpreterContext(storage::Storage *db, InterpreterConfig config,
|
||||
explicit InterpreterContext(storage::Storage *db, storage::rocks::RocksDBStorage *disk_db, InterpreterConfig config,
|
||||
const std::filesystem::path &data_directory);
|
||||
|
||||
storage::Storage *db;
|
||||
storage::rocks::RocksDBStorage *disk_db;
|
||||
|
||||
// ANTLR has singleton instance that is shared between threads. It is
|
||||
// protected by locks inside of ANTLR. Unfortunately, they are not protected
|
||||
|
||||
@@ -53,7 +53,6 @@
|
||||
#include "utils/likely.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/pmr/list.hpp"
|
||||
#include "utils/pmr/unordered_map.hpp"
|
||||
#include "utils/pmr/unordered_set.hpp"
|
||||
@@ -493,7 +492,8 @@ UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
|
||||
|
||||
auto vertices = [this](Frame &, ExecutionContext &context) {
|
||||
auto *db = context.db_accessor;
|
||||
return std::make_optional(db->Vertices(view_));
|
||||
auto vertices = std::make_optional(db->Vertices(view_));
|
||||
return vertices;
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem), view_,
|
||||
std::move(vertices), "ScanAll");
|
||||
|
||||
@@ -28,7 +28,6 @@
|
||||
#include "utils/fnv.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
#include "utils/visitor.hpp"
|
||||
|
||||
namespace memgraph {
|
||||
|
||||
@@ -845,7 +845,8 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
|
||||
file_path_ = file_path;
|
||||
dlerror(); // Clear any existing error.
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND);
|
||||
// handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND);
|
||||
handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL);
|
||||
if (!handle_) {
|
||||
spdlog::error(
|
||||
utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
|
||||
|
||||
@@ -496,7 +496,7 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std
|
||||
[interpreter_context, interpreter]() { interpreter_context->interpreters->erase(interpreter.get()); }};
|
||||
|
||||
EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
|
||||
CallCustomTransformation(transformation_name, messages, result, *accessor, *memory_resource, stream_name);
|
||||
CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
|
||||
|
||||
DiscardValueResultStream stream;
|
||||
|
||||
@@ -743,7 +743,7 @@ TransformationResult Streams::Check(const std::string &stream_name, std::optiona
|
||||
&transformation_name = transformation_name, &result,
|
||||
&test_result]<typename T>(const std::vector<T> &messages) mutable {
|
||||
auto accessor = interpreter_context->db->Access();
|
||||
CallCustomTransformation(transformation_name, messages, result, *accessor, *memory_resource, stream_name);
|
||||
CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
|
||||
|
||||
auto result_row = std::vector<TypedValue>();
|
||||
result_row.reserve(kCheckStreamResultSize);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -269,7 +269,7 @@ class TypedValue {
|
||||
|
||||
explicit TypedValue(const EdgeAccessor &edge, utils::MemoryResource *memory = utils::NewDeleteResource())
|
||||
: memory_(memory), type_(Type::Edge) {
|
||||
new (&edge_v) EdgeAccessor(std::move(edge));
|
||||
new (&edge_v) EdgeAccessor(edge);
|
||||
}
|
||||
|
||||
explicit TypedValue(const Path &path, utils::MemoryResource *memory = utils::NewDeleteResource())
|
||||
|
||||
@@ -12,6 +12,14 @@ set(storage_v2_src_files
|
||||
inmemory/storage.cpp
|
||||
inmemory/vertex_accessor.cpp
|
||||
property_store.cpp
|
||||
disk/compaction_filter.cpp
|
||||
disk/old_storage.hpp
|
||||
disk/old_storage.cpp
|
||||
disk/new_storage.hpp
|
||||
disk/vertex_accessor.cpp
|
||||
disk/vertex_accessor.hpp
|
||||
disk/edge_accessor.cpp
|
||||
disk/edge_accessor.hpp
|
||||
storage.cpp
|
||||
vertex_accessor.cpp)
|
||||
|
||||
|
||||
27
src/storage/v2/disk/compaction_filter.cpp
Normal file
27
src/storage/v2/disk/compaction_filter.cpp
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <rocksdb/cache.h>
|
||||
#include <rocksdb/compaction_filter.h>
|
||||
#include <rocksdb/options.h>
|
||||
#include <rocksdb/slice_transform.h>
|
||||
|
||||
class TimestampCompactionFilter : public rocksdb::CompactionFilter {
|
||||
public:
|
||||
const char *Name() const override { return "TimestampCompactionFilter"; }
|
||||
|
||||
/// Return true if the key-value pair should be removed from the database during compaction.
|
||||
/// Filters KV entries that are older than the specified timestamp.
|
||||
bool Filter(int level, const rocksdb::Slice &key, const rocksdb::Slice &existing_value, std::string *new_value,
|
||||
bool *value_changed) const override {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
273
src/storage/v2/disk/edge_accessor.cpp
Normal file
273
src/storage/v2/disk/edge_accessor.cpp
Normal file
@@ -0,0 +1,273 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "storage/v2/disk/edge_accessor.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <tuple>
|
||||
|
||||
#include "storage/v2/disk/vertex_accessor.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
bool DiskEdgeAccessor::IsVisible(const View view) const {
|
||||
/*
|
||||
bool exists = true;
|
||||
bool deleted = true;
|
||||
// When edges don't have properties, their isolation level is still dictated by MVCC ->
|
||||
// iterate over the deltas of the from_vertex_ and see which deltas can be applied on edges.
|
||||
if (!config_.properties_on_edges) {
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(from_vertex_->lock);
|
||||
// Initialize deleted by checking if out edges contain edge_
|
||||
deleted = std::find_if(from_vertex_->out_edges.begin(), from_vertex_->out_edges.end(), [&](const auto &out_edge) {
|
||||
return std::get<2>(out_edge) == edge_;
|
||||
}) == from_vertex_->out_edges.end();
|
||||
delta = from_vertex_->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
break;
|
||||
case Delta::Action::ADD_OUT_EDGE: { // relevant for the from_vertex_ -> we just deleted the edge
|
||||
if (delta.vertex_edge.edge == edge_) {
|
||||
deleted = false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_OUT_EDGE: { // also relevant for the from_vertex_ -> we just added the edge
|
||||
if (delta.vertex_edge.edge == edge_) {
|
||||
exists = false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
return exists && (for_deleted_ || !deleted);
|
||||
}
|
||||
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
deleted = edge_.ptr->deleted;
|
||||
delta = edge_.ptr->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
return exists && (for_deleted_ || !deleted);
|
||||
*/
|
||||
throw utils::NotYetImplemented("IsVisible for edges without properties is not implemented yet.");
|
||||
}
|
||||
|
||||
std::unique_ptr<VertexAccessor> DiskEdgeAccessor::FromVertex() const {
|
||||
// return std::make_unique<DiskVertexAccessor>(from_vertex_, transaction_, indices_, constraints_, config_);
|
||||
throw utils::NotYetImplemented("FromVertex is not implemented yet.");
|
||||
}
|
||||
|
||||
std::unique_ptr<VertexAccessor> DiskEdgeAccessor::ToVertex() const {
|
||||
// return std::make_unique<DiskVertexAccessor>(to_vertex_, transaction_, indices_, constraints_, config_);
|
||||
throw utils::NotYetImplemented("ToVertex is not implemented yet.");
|
||||
}
|
||||
|
||||
Result<storage::PropertyValue> DiskEdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
|
||||
/*utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
|
||||
|
||||
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
|
||||
|
||||
auto current_value = edge_.ptr->properties.GetProperty(property);
|
||||
// We could skip setting the value if the previous one is the same to the new
|
||||
// one. This would save some memory as a delta would not be created as well as
|
||||
// avoid copying the value. The reason we are not doing that is because the
|
||||
// current code always follows the logical pattern of "create a delta" and
|
||||
// "modify in-place". Additionally, the created delta will make other
|
||||
// transactions get a SERIALIZATION_ERROR.
|
||||
CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, current_value);
|
||||
edge_.ptr->properties.SetProperty(property, value);
|
||||
|
||||
return std::move(current_value);
|
||||
*/
|
||||
throw utils::NotYetImplemented("SetProperty is not implemented yet.");
|
||||
}
|
||||
|
||||
Result<bool> DiskEdgeAccessor::InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
/*utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
|
||||
|
||||
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
|
||||
|
||||
if (!edge_.ptr->properties.InitProperties(properties)) return false;
|
||||
for (const auto &[property, _] : properties) {
|
||||
CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, PropertyValue());
|
||||
}
|
||||
|
||||
return true;
|
||||
*/
|
||||
throw utils::NotYetImplemented("InitProperties is not implemented yet.");
|
||||
}
|
||||
|
||||
Result<std::map<PropertyId, PropertyValue>> DiskEdgeAccessor::ClearProperties() {
|
||||
// if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;//
|
||||
//
|
||||
// std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// auto properties = edge_.ptr->properties.Properties();
|
||||
// for (const auto &property : properties) {
|
||||
// CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property.first, property.second);
|
||||
// }
|
||||
//
|
||||
// edge_.ptr->properties.ClearProperties();
|
||||
//
|
||||
// return std::move(properties);
|
||||
throw utils::NotYetImplemented("ClearProperties is not implemented yet.");
|
||||
}
|
||||
|
||||
Result<PropertyValue> DiskEdgeAccessor::GetProperty(PropertyId property, View view) const {
|
||||
/*
|
||||
if (!config_.properties_on_edges) return PropertyValue();
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
PropertyValue value;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
deleted = edge_.ptr->deleted;
|
||||
value = edge_.ptr->properties.GetProperty(property);
|
||||
delta = edge_.ptr->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &value, property](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
if (delta.property.key == property) {
|
||||
value = delta.property.value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return std::move(value);
|
||||
*/
|
||||
throw utils::NotYetImplemented("GetProperty is not implemented yet.");
|
||||
}
|
||||
|
||||
Result<std::map<PropertyId, PropertyValue>> DiskEdgeAccessor::Properties(View view) const {
|
||||
/*if (!config_.properties_on_edges) return std::map<PropertyId, PropertyValue>{};
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
std::map<PropertyId, PropertyValue> properties;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
|
||||
deleted = edge_.ptr->deleted;
|
||||
properties = edge_.ptr->properties.Properties();
|
||||
delta = edge_.ptr->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
auto it = properties.find(delta.property.key);
|
||||
if (it != properties.end()) {
|
||||
if (delta.property.value.IsNull()) {
|
||||
// remove the property
|
||||
properties.erase(it);
|
||||
} else {
|
||||
// set the value
|
||||
it->second = delta.property.value;
|
||||
}
|
||||
} else if (!delta.property.value.IsNull()) {
|
||||
properties.emplace(delta.property.key, delta.property.value);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return std::move(properties);
|
||||
*/
|
||||
throw utils::NotYetImplemented("Properties is not implemented yet.");
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage
|
||||
100
src/storage/v2/disk/edge_accessor.hpp
Normal file
100
src/storage/v2/disk/edge_accessor.hpp
Normal file
@@ -0,0 +1,100 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/edge_ref.hpp"
|
||||
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
struct Vertex;
|
||||
class VertexAccessor;
|
||||
struct Indices;
|
||||
struct Constraints;
|
||||
|
||||
class DiskEdgeAccessor final : public EdgeAccessor {
|
||||
private:
|
||||
friend class DiskStorage;
|
||||
|
||||
public:
|
||||
DiskEdgeAccessor(EdgeRef edge, EdgeTypeId edge_type, Vertex *from_vertex, Vertex *to_vertex, Transaction *transaction,
|
||||
Indices *indices, Constraints *constraints, Config::Items config, bool for_deleted = false)
|
||||
: EdgeAccessor(edge_type, transaction, config, for_deleted),
|
||||
edge_(edge),
|
||||
from_vertex_(from_vertex),
|
||||
to_vertex_(to_vertex),
|
||||
indices_(indices),
|
||||
constraints_(constraints) {}
|
||||
|
||||
/// @return true if the object is visible from the current transaction
|
||||
bool IsVisible(View view) const override;
|
||||
|
||||
std::unique_ptr<VertexAccessor> FromVertex() const override;
|
||||
|
||||
std::unique_ptr<VertexAccessor> ToVertex() const override;
|
||||
|
||||
EdgeTypeId EdgeType() const { return edge_type_; }
|
||||
|
||||
/// Set a property value and return the old value.
|
||||
/// @throw std::bad_alloc
|
||||
Result<storage::PropertyValue> SetProperty(PropertyId property, const PropertyValue &value) override;
|
||||
|
||||
/// Set property values only if property store is empty. Returns `true` if successully set all values,
|
||||
/// `false` otherwise.
|
||||
/// @throw std::bad_alloc
|
||||
Result<bool> InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) override;
|
||||
|
||||
/// Remove all properties and return old values for each removed property.
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::map<PropertyId, PropertyValue>> ClearProperties() override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<PropertyValue> GetProperty(PropertyId property, View view) const override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const override;
|
||||
|
||||
storage::Gid Gid() const noexcept override {
|
||||
if (config_.properties_on_edges) {
|
||||
return edge_.ptr->gid;
|
||||
} else {
|
||||
return edge_.gid;
|
||||
}
|
||||
}
|
||||
|
||||
bool IsCycle() const override { return from_vertex_ == to_vertex_; }
|
||||
|
||||
bool operator==(const EdgeAccessor &other) const noexcept override {
|
||||
const auto *otherEdge = dynamic_cast<const DiskEdgeAccessor *>(&other);
|
||||
if (otherEdge == nullptr) return false;
|
||||
return edge_ == otherEdge->edge_ && transaction_ == otherEdge->transaction_;
|
||||
}
|
||||
bool operator!=(const EdgeAccessor &other) const noexcept { return !(*this == other); }
|
||||
|
||||
private:
|
||||
EdgeRef edge_;
|
||||
Vertex *from_vertex_;
|
||||
Vertex *to_vertex_;
|
||||
Indices *indices_;
|
||||
Constraints *constraints_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
358
src/storage/v2/disk/old_storage.cpp
Normal file
358
src/storage/v2/disk/old_storage.cpp
Normal file
@@ -0,0 +1,358 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <variant>
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "storage/v2/disk/edge_accessor.hpp"
|
||||
#include "storage/v2/disk/old_storage.hpp"
|
||||
#include "storage/v2/disk/vertex_accessor.hpp"
|
||||
#include "storage/v2/durability/durability.hpp"
|
||||
#include "storage/v2/durability/metadata.hpp"
|
||||
#include "storage/v2/durability/paths.hpp"
|
||||
#include "storage/v2/durability/snapshot.hpp"
|
||||
#include "storage/v2/durability/wal.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/indices.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/replication/config.hpp"
|
||||
#include "storage/v2/replication/enums.hpp"
|
||||
#include "storage/v2/replication/replication_persistence_helper.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/stat.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
/// REPLICATION ///
|
||||
#include "storage/v2/replication/replication_client.hpp"
|
||||
#include "storage/v2/replication/replication_server.hpp"
|
||||
#include "storage/v2/replication/rpc.hpp"
|
||||
#include "storage/v2/storage_error.hpp"
|
||||
|
||||
/// RocksDB
|
||||
#include <rocksdb/db.h>
|
||||
#include <rocksdb/iterator.h>
|
||||
#include <rocksdb/options.h>
|
||||
#include <rocksdb/status.h>
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler;
|
||||
|
||||
namespace {
|
||||
inline constexpr uint16_t kEpochHistoryRetention = 1000;
|
||||
|
||||
// /// Use it for operations that must successfully finish.
|
||||
inline void AssertRocksDBStatus(const rocksdb::Status &status) {
|
||||
MG_ASSERT(status.ok(), "rocksdb: {}", status.ToString());
|
||||
}
|
||||
|
||||
inline bool CheckRocksDBStatus(const rocksdb::Status &status) {
|
||||
if (!status.ok()) [[unlikely]] {
|
||||
spdlog::error("rocksdb: {}", status.ToString());
|
||||
}
|
||||
return status.ok();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DiskStorage::DiskAccessor::DiskAccessor(DiskStorage *storage, IsolationLevel isolation_level)
|
||||
: storage_(storage),
|
||||
// The lock must be acquired before creating the transaction object to
|
||||
// prevent freshly created transactions from dangling in an active state
|
||||
// during exclusive operations.
|
||||
storage_guard_(storage_->main_lock_),
|
||||
transaction_(storage->CreateTransaction(isolation_level)),
|
||||
is_transaction_active_(true),
|
||||
config_(storage->config_.items) {}
|
||||
|
||||
DiskStorage::DiskAccessor::DiskAccessor(DiskAccessor &&other) noexcept
|
||||
: storage_(other.storage_),
|
||||
storage_guard_(std::move(other.storage_guard_)),
|
||||
transaction_(std::move(other.transaction_)),
|
||||
commit_timestamp_(other.commit_timestamp_),
|
||||
is_transaction_active_(other.is_transaction_active_),
|
||||
config_(other.config_) {
|
||||
// Don't allow the other accessor to abort our transaction in destructor.
|
||||
other.is_transaction_active_ = false;
|
||||
other.commit_timestamp_.reset();
|
||||
}
|
||||
|
||||
DiskStorage::DiskAccessor::~DiskAccessor() {
|
||||
if (is_transaction_active_) {
|
||||
Abort();
|
||||
}
|
||||
|
||||
FinalizeTransaction();
|
||||
}
|
||||
|
||||
/*
|
||||
This function will be the same for both storage modes. We can move it on a different place later.
|
||||
*/
|
||||
std::unique_ptr<VertexAccessor> DiskStorage::DiskAccessor::CreateVertex() {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
auto gid = storage_->vertex_id_.fetch_add(1, std::memory_order_acq_rel);
|
||||
auto acc = storage_->vertices_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Vertex{storage::Gid::FromUint(gid), delta});
|
||||
MG_ASSERT(inserted, "The vertex must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
|
||||
delta->prev.Set(&*it);
|
||||
return std::make_unique<DiskVertexAccessor>(&*it, &transaction_, &storage_->indices_, &storage_->constraints_,
|
||||
config_);
|
||||
}
|
||||
|
||||
/*
|
||||
Same as function above, but with a given GID.
|
||||
*/
|
||||
std::unique_ptr<VertexAccessor> DiskStorage::DiskAccessor::CreateVertex(storage::Gid gid) {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
// NOTE: When we update the next `vertex_id_` here we perform a RMW
|
||||
// (read-modify-write) operation that ISN'T atomic! But, that isn't an issue
|
||||
// because this function is only called from the replication delta applier
|
||||
// that runs single-threadedly and while this instance is set-up to apply
|
||||
// threads (it is the replica), it is guaranteed that no other writes are
|
||||
// possible.
|
||||
storage_->vertex_id_.store(std::max(storage_->vertex_id_.load(std::memory_order_acquire), gid.AsUint() + 1),
|
||||
std::memory_order_release);
|
||||
auto acc = storage_->vertices_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Vertex{gid, delta});
|
||||
MG_ASSERT(inserted, "The vertex must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
|
||||
delta->prev.Set(&*it);
|
||||
return std::make_unique<DiskVertexAccessor>(&*it, &transaction_, &storage_->indices_, &storage_->constraints_,
|
||||
config_);
|
||||
}
|
||||
|
||||
std::unique_ptr<VertexAccessor> DiskStorage::DiskAccessor::DeserializeVertex(const std::string_view key,
|
||||
const std::string_view value) {
|
||||
/// Create vertex
|
||||
spdlog::info("Key to deserialize: {}", key);
|
||||
const auto vertex_parts = utils::Split(key, "|");
|
||||
auto impl = CreateVertex(storage::Gid::FromUint(std::stoull(vertex_parts[1])));
|
||||
// Deserialize labels
|
||||
if (!vertex_parts[0].empty()) {
|
||||
const auto labels = utils::Split(vertex_parts[0], ",");
|
||||
for (const auto &label : labels) {
|
||||
const storage::LabelId label_id = storage::LabelId::FromUint(std::stoull(label));
|
||||
// TODO: introduce error handling
|
||||
impl->AddLabel(label_id);
|
||||
}
|
||||
}
|
||||
impl->SetPropertyStore(value);
|
||||
return impl;
|
||||
}
|
||||
|
||||
std::unique_ptr<VertexAccessor> DiskStorage::DiskAccessor::FindVertex(storage::Gid gid, View /*view*/) {
|
||||
auto it =
|
||||
std::unique_ptr<rocksdb::Iterator>(storage_->db_->NewIterator(rocksdb::ReadOptions(), storage_->vertex_chandle));
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
const auto &key = it->key().ToString();
|
||||
if (const auto vertex_parts = utils::Split(key, "|"); vertex_parts[1] == SerializeIdType(gid)) {
|
||||
return DeserializeVertex(key, it->value().ToStringView());
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Result<std::unique_ptr<VertexAccessor>> DiskStorage::DiskAccessor::DeleteVertex(VertexAccessor *vertex) {
|
||||
if (!CheckRocksDBStatus(
|
||||
storage_->db_->Delete(rocksdb::WriteOptions(), storage_->vertex_chandle, SerializeVertex(vertex)))) {
|
||||
return Error::SERIALIZATION_ERROR;
|
||||
}
|
||||
return Result<std::unique_ptr<VertexAccessor>>{std::unique_ptr<VertexAccessor>(vertex)};
|
||||
}
|
||||
|
||||
Result<std::optional<std::pair<std::unique_ptr<VertexAccessor>, std::vector<std::unique_ptr<EdgeAccessor>>>>>
|
||||
DiskStorage::DiskAccessor::DetachDeleteVertex(VertexAccessor *vertex) {
|
||||
using ReturnType = std::pair<std::unique_ptr<VertexAccessor>, std::vector<std::unique_ptr<EdgeAccessor>>>;
|
||||
auto del_vertex = DeleteVertex(vertex);
|
||||
if (del_vertex.HasError()) {
|
||||
return del_vertex.GetError();
|
||||
}
|
||||
|
||||
auto out_edges = vertex->OutEdges(storage::View::OLD);
|
||||
auto in_edges = vertex->InEdges(storage::View::OLD);
|
||||
if (out_edges.HasError() || in_edges.HasError()) {
|
||||
return out_edges.GetError();
|
||||
}
|
||||
// if (auto del_edges = DeleteEdges(*out_edges), del_in_edges = DeleteEdges(*in_edges);
|
||||
// del_edges.has_value() && del_in_edges.has_value()) {
|
||||
// del_edges->insert(del_in_edges->end(), std::make_move_iterator(del_in_edges->begin()),
|
||||
// std::make_move_iterator(del_in_edges->end()));
|
||||
// return std::make_pair(*del_vertex, *del_edges);
|
||||
// }
|
||||
return Error::SERIALIZATION_ERROR;
|
||||
}
|
||||
|
||||
Result<std::unique_ptr<EdgeAccessor>> DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
EdgeTypeId edge_type) {
|
||||
throw utils::NotYetImplemented("CreateEdge");
|
||||
}
|
||||
|
||||
Result<std::unique_ptr<EdgeAccessor>> DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
EdgeTypeId edge_type, storage::Gid gid) {
|
||||
throw utils::NotYetImplemented("CreateEdge");
|
||||
}
|
||||
|
||||
Result<std::unique_ptr<EdgeAccessor>> DiskStorage::DiskAccessor::DeleteEdge(EdgeAccessor *edge) {
|
||||
throw utils::NotYetImplemented("DeleteEdge");
|
||||
}
|
||||
|
||||
const std::string &DiskStorage::DiskAccessor::LabelToName(LabelId label) const { return storage_->LabelToName(label); }
|
||||
|
||||
const std::string &DiskStorage::DiskAccessor::PropertyToName(PropertyId property) const {
|
||||
return storage_->PropertyToName(property);
|
||||
}
|
||||
|
||||
const std::string &DiskStorage::DiskAccessor::EdgeTypeToName(EdgeTypeId edge_type) const {
|
||||
return storage_->EdgeTypeToName(edge_type);
|
||||
}
|
||||
|
||||
LabelId DiskStorage::DiskAccessor::NameToLabel(const std::string_view name) { return storage_->NameToLabel(name); }
|
||||
|
||||
PropertyId DiskStorage::DiskAccessor::NameToProperty(const std::string_view name) {
|
||||
return storage_->NameToProperty(name);
|
||||
}
|
||||
|
||||
EdgeTypeId DiskStorage::DiskAccessor::NameToEdgeType(const std::string_view name) {
|
||||
return storage_->NameToEdgeType(name);
|
||||
}
|
||||
|
||||
void DiskStorage::DiskAccessor::AdvanceCommand() { ++transaction_.command_id; }
|
||||
|
||||
utils::BasicResult<StorageDataManipulationError, void> DiskStorage::DiskAccessor::Commit(
|
||||
const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("Commit");
|
||||
}
|
||||
|
||||
void DiskStorage::DiskAccessor::Abort() { throw utils::NotYetImplemented("Abort"); }
|
||||
|
||||
void DiskStorage::DiskAccessor::FinalizeTransaction() { throw utils::NotYetImplemented("FinalizeTransaction"); }
|
||||
|
||||
std::optional<uint64_t> DiskStorage::DiskAccessor::GetTransactionId() const {
|
||||
throw utils::NotYetImplemented("GetTransactionId");
|
||||
}
|
||||
|
||||
const std::string &DiskStorage::LabelToName(LabelId label) const { return name_id_mapper_.IdToName(label.AsUint()); }
|
||||
|
||||
const std::string &DiskStorage::PropertyToName(PropertyId property) const {
|
||||
return name_id_mapper_.IdToName(property.AsUint());
|
||||
}
|
||||
|
||||
const std::string &DiskStorage::EdgeTypeToName(EdgeTypeId edge_type) const {
|
||||
return name_id_mapper_.IdToName(edge_type.AsUint());
|
||||
}
|
||||
|
||||
LabelId DiskStorage::NameToLabel(const std::string_view name) {
|
||||
return LabelId::FromUint(name_id_mapper_.NameToId(name));
|
||||
}
|
||||
|
||||
PropertyId DiskStorage::NameToProperty(const std::string_view name) {
|
||||
return PropertyId::FromUint(name_id_mapper_.NameToId(name));
|
||||
}
|
||||
|
||||
EdgeTypeId DiskStorage::NameToEdgeType(const std::string_view name) {
|
||||
return EdgeTypeId::FromUint(name_id_mapper_.NameToId(name));
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::CreateIndex(
|
||||
LabelId label, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("CreateIndex");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::CreateIndex(
|
||||
LabelId label, PropertyId property, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("CreateIndex");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::DropIndex(
|
||||
LabelId label, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("DropIndex");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DiskStorage::DropIndex(
|
||||
LabelId label, PropertyId property, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("DropIndex");
|
||||
}
|
||||
|
||||
IndicesInfo DiskStorage::ListAllIndices() const { throw utils::NotYetImplemented("ListAllIndices"); }
|
||||
|
||||
utils::BasicResult<StorageExistenceConstraintDefinitionError, void> DiskStorage::CreateExistenceConstraint(
|
||||
LabelId label, PropertyId property, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("CreateExistenceConstraint");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageExistenceConstraintDroppingError, void> DiskStorage::DropExistenceConstraint(
|
||||
LabelId label, PropertyId property, const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("DropExistenceConstraint");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageUniqueConstraintDefinitionError, UniqueConstraints::CreationStatus>
|
||||
DiskStorage::CreateUniqueConstraint(LabelId label, const std::set<PropertyId> &properties,
|
||||
const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("CreateUniqueConstraint");
|
||||
}
|
||||
|
||||
utils::BasicResult<StorageUniqueConstraintDroppingError, UniqueConstraints::DeletionStatus>
|
||||
DiskStorage::DropUniqueConstraint(LabelId label, const std::set<PropertyId> &properties,
|
||||
const std::optional<uint64_t> desired_commit_timestamp) {
|
||||
throw utils::NotYetImplemented("DropUniqueConstraint");
|
||||
}
|
||||
|
||||
ConstraintsInfo DiskStorage::ListAllConstraints() const { throw utils::NotYetImplemented("ListAllConstraints"); }
|
||||
|
||||
StorageInfo DiskStorage::GetInfo() const { throw utils::NotYetImplemented("GetInfo"); }
|
||||
|
||||
VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, View view) {
|
||||
throw utils::NotYetImplemented("Vertices");
|
||||
}
|
||||
|
||||
VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId property, View view) {
|
||||
throw utils::NotYetImplemented("Vertices");
|
||||
}
|
||||
|
||||
VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId property, const PropertyValue &value,
|
||||
View view) {
|
||||
throw utils::NotYetImplemented("Vertices");
|
||||
}
|
||||
|
||||
VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound,
|
||||
View view) {
|
||||
throw utils::NotYetImplemented("Vertices");
|
||||
}
|
||||
|
||||
Transaction DiskStorage::CreateTransaction(IsolationLevel isolation_level) {
|
||||
throw utils::NotYetImplemented("CreateTransaction");
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage
|
||||
555
src/storage/v2/disk/old_storage.hpp
Normal file
555
src/storage/v2/disk/old_storage.hpp
Normal file
@@ -0,0 +1,555 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <optional>
|
||||
#include <shared_mutex>
|
||||
#include <span>
|
||||
#include <variant>
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "kvstore/kvstore.hpp"
|
||||
#include "storage/v2/commit_log.hpp"
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/constraints.hpp"
|
||||
#include "storage/v2/disk/vertex_accessor.hpp"
|
||||
#include "storage/v2/durability/metadata.hpp"
|
||||
#include "storage/v2/durability/wal.hpp"
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/indices.hpp"
|
||||
#include "storage/v2/isolation_level.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/name_id_mapper.hpp"
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/vertex.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/scheduler.hpp"
|
||||
#include "utils/skip_list.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
/// REPLICATION ///
|
||||
#include "rpc/server.hpp"
|
||||
#include "storage/v2/replication/config.hpp"
|
||||
#include "storage/v2/replication/enums.hpp"
|
||||
#include "storage/v2/replication/rpc.hpp"
|
||||
#include "storage/v2/replication/serialization.hpp"
|
||||
#include "storage/v2/storage_error.hpp"
|
||||
|
||||
/// ROCKSDB
|
||||
#include <rocksdb/db.h>
|
||||
#include <rocksdb/iterator.h>
|
||||
#include <rocksdb/options.h>
|
||||
#include <rocksdb/status.h>
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
class DiskStorage final : public Storage {
|
||||
public:
|
||||
/// @throw std::system_error
|
||||
/// @throw std::bad_alloc
|
||||
explicit DiskStorage();
|
||||
|
||||
~DiskStorage();
|
||||
|
||||
class DiskAccessor final : public Storage::Accessor {
|
||||
private:
|
||||
friend class DiskStorage;
|
||||
|
||||
explicit DiskAccessor(DiskStorage *storage, IsolationLevel isolation_level);
|
||||
|
||||
public:
|
||||
DiskAccessor(const DiskAccessor &) = delete;
|
||||
DiskAccessor &operator=(const DiskAccessor &) = delete;
|
||||
DiskAccessor &operator=(DiskAccessor &&other) = delete;
|
||||
|
||||
// NOTE: After the accessor is moved, all objects derived from it (accessors
|
||||
// and iterators) are *invalid*. You have to get all derived objects again.
|
||||
DiskAccessor(DiskAccessor &&other) noexcept;
|
||||
|
||||
~DiskAccessor() override;
|
||||
|
||||
/// @throw std::alloc
|
||||
std::unique_ptr<VertexAccessor> CreateVertex() override;
|
||||
|
||||
std::unique_ptr<VertexAccessor> FindVertex(Gid gid, View view) override;
|
||||
|
||||
VerticesIterable Vertices(View view) override {
|
||||
return VerticesIterable(AllVerticesIterable(storage_->vertices_.access(), &transaction_, view,
|
||||
&storage_->indices_, &storage_->constraints_,
|
||||
storage_->config_.items));
|
||||
}
|
||||
|
||||
VerticesIterable Vertices(LabelId label, View view) override;
|
||||
|
||||
VerticesIterable Vertices(LabelId label, PropertyId property, View view) override;
|
||||
|
||||
VerticesIterable Vertices(LabelId label, PropertyId property, const PropertyValue &value, View view) override;
|
||||
|
||||
VerticesIterable Vertices(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view) override;
|
||||
|
||||
/// Return approximate number of all vertices in the database.
|
||||
/// Note that this is always an over-estimate and never an under-estimate.
|
||||
int64_t ApproximateVertexCount() const override { return storage_->vertices_.size(); }
|
||||
|
||||
/// Return approximate number of vertices with the given label.
|
||||
/// Note that this is always an over-estimate and never an under-estimate.
|
||||
int64_t ApproximateVertexCount(LabelId label) const override {
|
||||
return storage_->indices_.label_index.ApproximateVertexCount(label);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and property.
|
||||
/// Note that this is always an over-estimate and never an under-estimate.
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const override {
|
||||
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and the given
|
||||
/// value for the given property. Note that this is always an over-estimate
|
||||
/// and never an under-estimate.
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const override {
|
||||
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and value for
|
||||
/// the given property in the range defined by provided upper and lower
|
||||
/// bounds.
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const override {
|
||||
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
|
||||
}
|
||||
|
||||
std::optional<storage::IndexStats> GetIndexStats(const storage::LabelId &label,
|
||||
const storage::PropertyId &property) const override {
|
||||
return storage_->indices_.label_property_index.GetIndexStats(label, property);
|
||||
}
|
||||
|
||||
std::vector<std::pair<LabelId, PropertyId>> ClearIndexStats() override {
|
||||
return storage_->indices_.label_property_index.ClearIndexStats();
|
||||
}
|
||||
|
||||
std::vector<std::pair<LabelId, PropertyId>> DeleteIndexStatsForLabels(
|
||||
const std::span<std::string> labels) override {
|
||||
std::vector<std::pair<LabelId, PropertyId>> deleted_indexes;
|
||||
std::for_each(labels.begin(), labels.end(), [this, &deleted_indexes](const auto &label_str) {
|
||||
std::vector<std::pair<LabelId, PropertyId>> loc_results =
|
||||
storage_->indices_.label_property_index.DeleteIndexStatsForLabel(NameToLabel(label_str));
|
||||
deleted_indexes.insert(deleted_indexes.end(), std::make_move_iterator(loc_results.begin()),
|
||||
std::make_move_iterator(loc_results.end()));
|
||||
});
|
||||
return deleted_indexes;
|
||||
}
|
||||
|
||||
void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property,
|
||||
const IndexStats &stats) override {
|
||||
storage_->indices_.label_property_index.SetIndexStats(label, property, stats);
|
||||
}
|
||||
|
||||
/// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::unique_ptr<VertexAccessor>> DeleteVertex(VertexAccessor *vertex) override;
|
||||
|
||||
/// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::optional<std::pair<std::unique_ptr<VertexAccessor>, std::vector<std::unique_ptr<EdgeAccessor>>>>>
|
||||
DetachDeleteVertex(VertexAccessor *vertex) override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::unique_ptr<EdgeAccessor>> CreateEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
EdgeTypeId edge_type) override;
|
||||
|
||||
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::unique_ptr<EdgeAccessor>> DeleteEdge(EdgeAccessor *edge) override;
|
||||
|
||||
const std::string &LabelToName(LabelId label) const override;
|
||||
const std::string &PropertyToName(PropertyId property) const override;
|
||||
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
LabelId NameToLabel(std::string_view name) override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
PropertyId NameToProperty(std::string_view name) override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
EdgeTypeId NameToEdgeType(std::string_view name) override;
|
||||
|
||||
bool LabelIndexExists(LabelId label) const override { return storage_->indices_.label_index.IndexExists(label); }
|
||||
|
||||
bool LabelPropertyIndexExists(LabelId label, PropertyId property) const override {
|
||||
return storage_->indices_.label_property_index.IndexExists(label, property);
|
||||
}
|
||||
|
||||
IndicesInfo ListAllIndices() const override {
|
||||
return {storage_->indices_.label_index.ListIndices(), storage_->indices_.label_property_index.ListIndices()};
|
||||
}
|
||||
|
||||
ConstraintsInfo ListAllConstraints() const override {
|
||||
return {ListExistenceConstraints(storage_->constraints_),
|
||||
storage_->constraints_.unique_constraints.ListConstraints()};
|
||||
}
|
||||
|
||||
void AdvanceCommand() override;
|
||||
|
||||
/// Returns void if the transaction has been committed.
|
||||
/// Returns `StorageDataManipulationError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `ConstraintViolation`: the changes made by this transaction violate an existence or unique constraint. In this
|
||||
/// case the transaction is automatically aborted.
|
||||
/// @throw std::bad_alloc
|
||||
utils::BasicResult<StorageDataManipulationError, void> Commit(
|
||||
std::optional<uint64_t> desired_commit_timestamp = {}) override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
void Abort() override;
|
||||
|
||||
void FinalizeTransaction() override;
|
||||
|
||||
std::optional<uint64_t> GetTransactionId() const override;
|
||||
|
||||
private:
|
||||
/// @throw std::bad_alloc
|
||||
std::unique_ptr<VertexAccessor> CreateVertex(storage::Gid gid);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::unique_ptr<EdgeAccessor>> CreateEdge(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type,
|
||||
storage::Gid gid);
|
||||
|
||||
inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); }
|
||||
|
||||
// (De)serialization utility methods
|
||||
|
||||
std::string SerializeLabels(const auto &&labels) {
|
||||
if (labels.HasError() || (*labels).empty()) {
|
||||
return "";
|
||||
}
|
||||
std::string result = std::to_string((*labels)[0].AsUint());
|
||||
std::string ser_labels = std::accumulate(
|
||||
std::next((*labels).begin()), (*labels).end(), result,
|
||||
[](const std::string &join, const auto &label_id) { return join + "," + std::to_string(label_id.AsUint()); });
|
||||
return ser_labels;
|
||||
}
|
||||
|
||||
std::string SerializeVertex(VertexAccessor *vertex_acc) {
|
||||
std::string result = SerializeLabels(vertex_acc->Labels(storage::View::OLD)) + "|";
|
||||
result += SerializeIdType(vertex_acc->Gid());
|
||||
return result;
|
||||
}
|
||||
|
||||
std::unique_ptr<VertexAccessor> DeserializeVertex(std::string_view key, std::string_view value);
|
||||
|
||||
DiskStorage *storage_;
|
||||
std::shared_lock<utils::RWLock> storage_guard_;
|
||||
Transaction transaction_;
|
||||
std::optional<uint64_t> commit_timestamp_;
|
||||
bool is_transaction_active_;
|
||||
Config::Items config_;
|
||||
};
|
||||
|
||||
std::unique_ptr<Storage::Accessor> Access(std::optional<IsolationLevel> override_isolation_level) override {
|
||||
return std::unique_ptr<DiskAccessor>(new DiskAccessor{this, override_isolation_level.value_or(isolation_level_)});
|
||||
}
|
||||
|
||||
const std::string &LabelToName(LabelId label) const override;
|
||||
const std::string &PropertyToName(PropertyId property) const override;
|
||||
const std::string &EdgeTypeToName(EdgeTypeId edge_type) const override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
LabelId NameToLabel(std::string_view name) override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
PropertyId NameToProperty(std::string_view name) override;
|
||||
|
||||
/// @throw std::bad_alloc if unable to insert a new mapping
|
||||
EdgeTypeId NameToEdgeType(std::string_view name) override;
|
||||
|
||||
/// Create an index.
|
||||
/// Returns void if the index has been created.
|
||||
/// Returns `StorageIndexDefinitionError` if an error occures. Error can be:
|
||||
/// * `IndexDefinitionError`: the index already exists.
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// @throw std::bad_alloc
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> CreateIndex(
|
||||
LabelId label, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Create an index.
|
||||
/// Returns void if the index has been created.
|
||||
/// Returns `StorageIndexDefinitionError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `IndexDefinitionError`: the index already exists.
|
||||
/// @throw std::bad_alloc
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> CreateIndex(
|
||||
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Drop an existing index.
|
||||
/// Returns void if the index has been dropped.
|
||||
/// Returns `StorageIndexDefinitionError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `IndexDefinitionError`: the index does not exist.
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DropIndex(
|
||||
LabelId label, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Drop an existing index.
|
||||
/// Returns void if the index has been dropped.
|
||||
/// Returns `StorageIndexDefinitionError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `IndexDefinitionError`: the index does not exist.
|
||||
utils::BasicResult<StorageIndexDefinitionError, void> DropIndex(
|
||||
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
IndicesInfo ListAllIndices() const override;
|
||||
|
||||
/// Returns void if the existence constraint has been created.
|
||||
/// Returns `StorageExistenceConstraintDefinitionError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `ConstraintViolation`: there is already a vertex existing that would break this new constraint.
|
||||
/// * `ConstraintDefinitionError`: the constraint already exists.
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error
|
||||
utils::BasicResult<StorageExistenceConstraintDefinitionError, void> CreateExistenceConstraint(
|
||||
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Drop an existing existence constraint.
|
||||
/// Returns void if the existence constraint has been dropped.
|
||||
/// Returns `StorageExistenceConstraintDroppingError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `ConstraintDefinitionError`: the constraint did not exists.
|
||||
utils::BasicResult<StorageExistenceConstraintDroppingError, void> DropExistenceConstraint(
|
||||
LabelId label, PropertyId property, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Create an unique constraint.
|
||||
/// Returns `StorageUniqueConstraintDefinitionError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// * `ConstraintViolation`: there are already vertices violating the constraint.
|
||||
/// Returns `UniqueConstraints::CreationStatus` otherwise. Value can be:
|
||||
/// * `SUCCESS` if the constraint was successfully created,
|
||||
/// * `ALREADY_EXISTS` if the constraint already existed,
|
||||
/// * `EMPTY_PROPERTIES` if the property set is empty, or
|
||||
/// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties.
|
||||
/// @throw std::bad_alloc
|
||||
utils::BasicResult<StorageUniqueConstraintDefinitionError, UniqueConstraints::CreationStatus> CreateUniqueConstraint(
|
||||
LabelId label, const std::set<PropertyId> &properties, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
/// Removes an existing unique constraint.
|
||||
/// Returns `StorageUniqueConstraintDroppingError` if an error occures. Error can be:
|
||||
/// * `ReplicationError`: there is at least one SYNC replica that has not confirmed receiving the transaction.
|
||||
/// Returns `UniqueConstraints::DeletionStatus` otherwise. Value can be:
|
||||
/// * `SUCCESS` if constraint was successfully removed,
|
||||
/// * `NOT_FOUND` if the specified constraint was not found,
|
||||
/// * `EMPTY_PROPERTIES` if the property set is empty, or
|
||||
/// * `PROPERTIES_SIZE_LIMIT_EXCEEDED` if the property set exceeds the limit of maximum number of properties.
|
||||
utils::BasicResult<StorageUniqueConstraintDroppingError, UniqueConstraints::DeletionStatus> DropUniqueConstraint(
|
||||
LabelId label, const std::set<PropertyId> &properties, std::optional<uint64_t> desired_commit_timestamp) override;
|
||||
|
||||
ConstraintsInfo ListAllConstraints() const override;
|
||||
|
||||
StorageInfo GetInfo() const override;
|
||||
|
||||
bool LockPath() override;
|
||||
bool UnlockPath() override;
|
||||
|
||||
bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) override;
|
||||
|
||||
bool SetMainReplicationRole() override;
|
||||
|
||||
/// @pre The instance should have a MAIN role
|
||||
/// @pre Timeout can only be set for SYNC replication
|
||||
utils::BasicResult<RegisterReplicaError, void> RegisterReplica(
|
||||
std::string name, io::network::Endpoint endpoint, replication::ReplicationMode replication_mode,
|
||||
replication::RegistrationMode registration_mode, const replication::ReplicationClientConfig &config) override;
|
||||
/// @pre The instance should have a MAIN role
|
||||
bool UnregisterReplica(const std::string &name) override;
|
||||
|
||||
std::optional<replication::ReplicaState> GetReplicaState(std::string_view name) override;
|
||||
|
||||
ReplicationRole GetReplicationRole() const override;
|
||||
|
||||
std::vector<ReplicaInfo> ReplicasInfo() override;
|
||||
|
||||
void FreeMemory() override;
|
||||
|
||||
void SetIsolationLevel(IsolationLevel isolation_level) override;
|
||||
|
||||
utils::BasicResult<CreateSnapshotError> CreateSnapshot() override;
|
||||
|
||||
private:
|
||||
Transaction CreateTransaction(IsolationLevel isolation_level);
|
||||
|
||||
/// The force parameter determines the behaviour of the garbage collector.
|
||||
/// If it's set to true, it will behave as a global operation, i.e. it can't
|
||||
/// be part of a transaction, and no other transaction can be active at the same time.
|
||||
/// This allows it to delete immediately vertices without worrying that some other
|
||||
/// transaction is possibly using it. If there are active transactions when this method
|
||||
/// is called with force set to true, it will fallback to the same method with the force
|
||||
/// set to false.
|
||||
/// If it's set to false, it will execute in parallel with other transactions, ensuring
|
||||
/// that no object in use can be deleted.
|
||||
/// @throw std::system_error
|
||||
/// @throw std::bad_alloc
|
||||
template <bool force>
|
||||
void CollectGarbage();
|
||||
|
||||
bool InitializeWalFile();
|
||||
void FinalizeWalFile();
|
||||
|
||||
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
|
||||
[[nodiscard]] bool AppendToWalDataManipulation(const Transaction &transaction, uint64_t final_commit_timestamp);
|
||||
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
|
||||
[[nodiscard]] bool AppendToWalDataDefinition(durability::StorageGlobalOperation operation, LabelId label,
|
||||
const std::set<PropertyId> &properties, uint64_t final_commit_timestamp);
|
||||
|
||||
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});
|
||||
|
||||
void RestoreReplicas();
|
||||
|
||||
bool ShouldStoreAndRestoreReplicas() const;
|
||||
|
||||
// Main storage lock.
|
||||
//
|
||||
// Accessors take a shared lock when starting, so it is possible to block
|
||||
// creation of new accessors by taking a unique lock. This is used when doing
|
||||
// operations on storage that affect the global state, for example index
|
||||
// creation.
|
||||
mutable utils::RWLock main_lock_{utils::RWLock::Priority::WRITE};
|
||||
|
||||
// Main object storage
|
||||
utils::SkipList<storage::Vertex> vertices_;
|
||||
utils::SkipList<storage::Edge> edges_;
|
||||
std::atomic<uint64_t> vertex_id_{0};
|
||||
std::atomic<uint64_t> edge_id_{0};
|
||||
// Even though the edge count is already kept in the `edges_` SkipList, the
|
||||
// list is used only when properties are enabled for edges. Because of that we
|
||||
// keep a separate count of edges that is always updated.
|
||||
std::atomic<uint64_t> edge_count_{0};
|
||||
|
||||
NameIdMapper name_id_mapper_;
|
||||
|
||||
Constraints constraints_;
|
||||
Indices indices_;
|
||||
|
||||
// Transaction engine
|
||||
utils::SpinLock engine_lock_;
|
||||
uint64_t timestamp_{kTimestampInitialId};
|
||||
uint64_t transaction_id_{kTransactionInitialId};
|
||||
// TODO: This isn't really a commit log, it doesn't even care if a
|
||||
// transaction commited or aborted. We could probably combine this with
|
||||
// `timestamp_` in a sensible unit, something like TransactionClock or
|
||||
// whatever.
|
||||
std::optional<CommitLog> commit_log_;
|
||||
|
||||
utils::Synchronized<std::list<Transaction>, utils::SpinLock> committed_transactions_;
|
||||
IsolationLevel isolation_level_;
|
||||
|
||||
Config config_;
|
||||
utils::Scheduler gc_runner_;
|
||||
std::mutex gc_lock_;
|
||||
|
||||
// Undo buffers that were unlinked and now are waiting to be freed.
|
||||
utils::Synchronized<std::list<std::pair<uint64_t, std::list<Delta>>>, utils::SpinLock> garbage_undo_buffers_;
|
||||
|
||||
// Vertices that are logically deleted but still have to be removed from
|
||||
// indices before removing them from the main storage.
|
||||
utils::Synchronized<std::list<Gid>, utils::SpinLock> deleted_vertices_;
|
||||
|
||||
// Vertices that are logically deleted and removed from indices and now wait
|
||||
// to be removed from the main storage.
|
||||
std::list<std::pair<uint64_t, Gid>> garbage_vertices_;
|
||||
|
||||
// Edges that are logically deleted and wait to be removed from the main
|
||||
// storage.
|
||||
utils::Synchronized<std::list<Gid>, utils::SpinLock> deleted_edges_;
|
||||
|
||||
// Durability
|
||||
std::filesystem::path snapshot_directory_;
|
||||
std::filesystem::path wal_directory_;
|
||||
std::filesystem::path lock_file_path_;
|
||||
utils::OutputFile lock_file_handle_;
|
||||
std::unique_ptr<kvstore::KVStore> storage_;
|
||||
|
||||
utils::Scheduler snapshot_runner_;
|
||||
utils::SpinLock snapshot_lock_;
|
||||
|
||||
// UUID used to distinguish snapshots and to link snapshots to WALs
|
||||
std::string uuid_;
|
||||
// Sequence number used to keep track of the chain of WALs.
|
||||
uint64_t wal_seq_num_{0};
|
||||
|
||||
// UUID to distinguish different main instance runs for replication process
|
||||
// on SAME storage.
|
||||
// Multiple instances can have same storage UUID and be MAIN at the same time.
|
||||
// We cannot compare commit timestamps of those instances if one of them
|
||||
// becomes the replica of the other so we use epoch_id_ as additional
|
||||
// discriminating property.
|
||||
// Example of this:
|
||||
// We have 2 instances of the same storage, S1 and S2.
|
||||
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
|
||||
// At the moment when S1 commited a transaction with timestamp 20, and S2
|
||||
// a different transaction with timestamp 15, we change S2's role to REPLICA
|
||||
// and register it on S1.
|
||||
// Without using the epoch_id, we don't know that S1 and S2 have completely
|
||||
// different transactions, we think that the S2 is behind only by 5 commits.
|
||||
std::string epoch_id_;
|
||||
// History of the previous epoch ids.
|
||||
// Each value consists of the epoch id along the last commit belonging to that
|
||||
// epoch.
|
||||
std::deque<std::pair<std::string, uint64_t>> epoch_history_;
|
||||
|
||||
std::optional<durability::WalFile> wal_file_;
|
||||
uint64_t wal_unsynced_transactions_{0};
|
||||
|
||||
utils::FileRetainer file_retainer_;
|
||||
|
||||
// Global locker that is used for clients file locking
|
||||
utils::FileRetainer::FileLocker global_locker_;
|
||||
|
||||
// Last commited timestamp
|
||||
std::atomic<uint64_t> last_commit_timestamp_{kTimestampInitialId};
|
||||
|
||||
class ReplicationServer;
|
||||
std::unique_ptr<ReplicationServer> replication_server_{nullptr};
|
||||
|
||||
class ReplicationClient;
|
||||
// We create ReplicationClient using unique_ptr so we can move
|
||||
// newly created client into the vector.
|
||||
// We cannot move the client directly because it contains ThreadPool
|
||||
// which cannot be moved. Also, the move is necessary because
|
||||
// we don't want to create the client directly inside the vector
|
||||
// because that would require the lock on the list putting all
|
||||
// commits (they iterate list of clients) to halt.
|
||||
// This way we can initialize client in main thread which means
|
||||
// that we can immediately notify the user if the initialization
|
||||
// failed.
|
||||
using ReplicationClientList = utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>, utils::SpinLock>;
|
||||
ReplicationClientList replication_clients_;
|
||||
|
||||
std::atomic<ReplicationRole> replication_role_{ReplicationRole::MAIN};
|
||||
|
||||
rocksdb::Options options_;
|
||||
rocksdb::DB *db_;
|
||||
rocksdb::ColumnFamilyHandle *vertex_chandle = nullptr;
|
||||
rocksdb::ColumnFamilyHandle *edge_chandle = nullptr;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
389
src/storage/v2/disk/storage.hpp
Normal file
389
src/storage/v2/disk/storage.hpp
Normal file
@@ -0,0 +1,389 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <rocksdb/db.h>
|
||||
#include <rocksdb/iterator.h>
|
||||
#include <rocksdb/options.h>
|
||||
#include <rocksdb/status.h>
|
||||
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::storage::rocks {
|
||||
|
||||
constexpr const char *vertexHandle = "vertex";
|
||||
constexpr const char *edgeHandle = "edge";
|
||||
constexpr const char *outEdgeDirection = "0";
|
||||
constexpr const char *inEdgeDirection = "1";
|
||||
|
||||
// /// Use it for operations that must successfully finish.
|
||||
inline void AssertRocksDBStatus(const rocksdb::Status &status) {
|
||||
MG_ASSERT(status.ok(), "rocksdb: {}", status.ToString());
|
||||
}
|
||||
|
||||
// inline bool CheckRocksDBStatus(const rocksdb::Status &status) {
|
||||
// if (!status.ok()) [[unlikely]] {
|
||||
// spdlog::error("rocksdb: {}", status.ToString());
|
||||
// }
|
||||
// return status.ok();
|
||||
// }
|
||||
|
||||
class RocksDBStorage {
|
||||
public:
|
||||
explicit RocksDBStorage() {
|
||||
options_.create_if_missing = true;
|
||||
// options_.OptimizeLevelStyleCompaction();
|
||||
std::filesystem::path rocksdb_path = "./rocks_experiment_unit";
|
||||
MG_ASSERT(utils::EnsureDir(rocksdb_path), "Unable to create storage folder on the disk.");
|
||||
AssertRocksDBStatus(rocksdb::DB::Open(options_, rocksdb_path, &db_));
|
||||
AssertRocksDBStatus(db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), vertexHandle, &vertex_chandle));
|
||||
AssertRocksDBStatus(db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), edgeHandle, &edge_chandle));
|
||||
}
|
||||
|
||||
RocksDBStorage(const RocksDBStorage &) = delete;
|
||||
RocksDBStorage &operator=(const RocksDBStorage &) = delete;
|
||||
RocksDBStorage &operator=(RocksDBStorage &&) = delete;
|
||||
RocksDBStorage(RocksDBStorage &&) = delete;
|
||||
|
||||
~RocksDBStorage() {
|
||||
AssertRocksDBStatus(db_->DropColumnFamily(vertex_chandle));
|
||||
AssertRocksDBStatus(db_->DropColumnFamily(edge_chandle));
|
||||
AssertRocksDBStatus(db_->DestroyColumnFamilyHandle(vertex_chandle));
|
||||
AssertRocksDBStatus(db_->DestroyColumnFamilyHandle(edge_chandle));
|
||||
AssertRocksDBStatus(db_->Close());
|
||||
delete db_;
|
||||
}
|
||||
|
||||
// // EDGE ACCESSOR FUNCTIONALITIES
|
||||
// // -----------------------------------------------------------
|
||||
|
||||
// /// fetch the edge's source vertex by its GID
|
||||
// std::optional<query::VertexAccessor> FromVertex(const query::EdgeAccessor &edge_acc, query::DbAccessor &dba) {
|
||||
// return FindVertex(SerializeIdType(edge_acc.From().Gid()), dba);
|
||||
// }
|
||||
|
||||
// /// fetch the edge's destination vertex by its GID
|
||||
// std::optional<query::VertexAccessor> ToVertex(const query::EdgeAccessor &edge_acc, query::DbAccessor &dba) {
|
||||
// return FindVertex(SerializeIdType(edge_acc.To().Gid()), dba);
|
||||
// }
|
||||
|
||||
// /// VERTEX ACCESSOR FUNCTIONALITIES
|
||||
// /// ------------------------------------------------------------
|
||||
|
||||
// /// The VertexAccessor's out edge with gid src_gid has the following format in the RocksDB:
|
||||
// /// src_gid | other_vertex_gid | 0 | ...
|
||||
// /// other_vertex_gid | src_gid | 1 | ...
|
||||
// /// We use the firt way since this should be possible to optimize using Bloom filters and prefix search
|
||||
// std::vector<query::EdgeAccessor> OutEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) {
|
||||
// const auto vertex_acc_gid = SerializeIdType(vertex_acc.Gid());
|
||||
// std::vector<query::EdgeAccessor> out_edges;
|
||||
// auto it = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions(), edge_chandle));
|
||||
// for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
// const std::string_view key = it->key().ToStringView();
|
||||
// const auto vertex_parts = utils::Split(key, "|");
|
||||
// if (vertex_parts[0] == vertex_acc_gid && vertex_parts[2] == outEdgeDirection) {
|
||||
// out_edges.push_back(DeserializeEdge(key, it->value().ToStringView(), dba));
|
||||
// }
|
||||
// }
|
||||
// return out_edges;
|
||||
// }
|
||||
|
||||
// /// The VertexAccessor's out edge with gid src_gid has the following format in the RocksDB:
|
||||
// /// other_vertex_gid | dest_gid | 0 | ...
|
||||
// /// dest_gid | other_verte_gid | 1 | ...
|
||||
// /// we use the second way since this should be possible to optimize using Bloom filters and prefix search.
|
||||
// std::vector<query::EdgeAccessor> InEdges(const query::VertexAccessor &vertex_acc, query::DbAccessor &dba) {
|
||||
// const auto vertex_acc_gid = SerializeIdType(vertex_acc.Gid());
|
||||
// std::vector<query::EdgeAccessor> in_edges;
|
||||
// auto it = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions(), edge_chandle));
|
||||
// for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
// const std::string_view key = it->key().ToStringView();
|
||||
// const auto vertex_parts = utils::Split(key, "|");
|
||||
// if (vertex_parts[0] == vertex_acc_gid && vertex_parts[2] == inEdgeDirection) {
|
||||
// in_edges.push_back(DeserializeEdge(key, it->value().ToStringView(), dba));
|
||||
// }
|
||||
// }
|
||||
// return in_edges;
|
||||
// }
|
||||
|
||||
// /// TODO: how will we handle new vertex creation
|
||||
|
||||
// /// STORAGE ACCESSOR FUNCTIONALITIES
|
||||
// /// -----------------------------------------------------------
|
||||
|
||||
// /// TODO: how will we handle new edge creation
|
||||
|
||||
// /// @return Accessor to the deleted edge if a deletion took place, std::nullopt otherwise.
|
||||
// /// Delete two edge entries since on edge is represented on a two-fold level.
|
||||
// /// Edges are deleted from logical partition containing edges.
|
||||
// std::optional<query::EdgeAccessor> DeleteEdge(const query::EdgeAccessor &edge_acc) {
|
||||
// auto [src_dest_key, dest_src_key] = SerializeEdge(edge_acc);
|
||||
// if (!CheckRocksDBStatus(db_->Delete(rocksdb::WriteOptions(), edge_chandle, src_dest_key)) ||
|
||||
// !CheckRocksDBStatus(db_->Delete(rocksdb::WriteOptions(), edge_chandle, dest_src_key))) {
|
||||
// return std::nullopt;
|
||||
// }
|
||||
// return edge_acc;
|
||||
// }
|
||||
|
||||
// /// Helper function, not used in the real accessor.
|
||||
// std::optional<std::vector<query::EdgeAccessor>> DeleteEdges(const auto &edge_accessors) {
|
||||
// std::vector<query::EdgeAccessor> edge_accs;
|
||||
// for (auto &&it : edge_accessors) {
|
||||
// if (const auto deleted_edge_res = DeleteEdge(it); !deleted_edge_res.has_value()) {
|
||||
// return std::nullopt;
|
||||
// }
|
||||
// edge_accs.push_back(it);
|
||||
// }
|
||||
// return edge_accs;
|
||||
// }
|
||||
|
||||
// /// @return A reference to the deleted vertex accessor if deleted, otherwise std::nullopt.
|
||||
// /// Delete vertex from logical partition containing vertices.
|
||||
// std::optional<query::VertexAccessor> DeleteVertex(const query::VertexAccessor &vertex_acc) {
|
||||
// if (!CheckRocksDBStatus(db_->Delete(rocksdb::WriteOptions(), vertex_chandle, SerializeVertex(vertex_acc)))) {
|
||||
// return std::nullopt;
|
||||
// }
|
||||
// return vertex_acc;
|
||||
// }
|
||||
|
||||
// /// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise.
|
||||
// /// Delete vertex from logical partition containing vertices.
|
||||
// /// For each edge delete two key-value entries from logical partition containing edges.
|
||||
// std::optional<std::pair<query::VertexAccessor, std::vector<query::EdgeAccessor>>> DetachDeleteVertex(
|
||||
// const query::VertexAccessor &vertex_acc) {
|
||||
// auto del_vertex = DeleteVertex(vertex_acc);
|
||||
// if (!del_vertex.has_value()) {
|
||||
// return std::nullopt;
|
||||
// }
|
||||
// auto out_edges = vertex_acc.OutEdges(storage::View::OLD);
|
||||
// auto in_edges = vertex_acc.InEdges(storage::View::OLD);
|
||||
// if (out_edges.HasError() || in_edges.HasError()) {
|
||||
// return std::nullopt;
|
||||
// }
|
||||
// if (auto del_edges = DeleteEdges(*out_edges), del_in_edges = DeleteEdges(*in_edges);
|
||||
// del_edges.has_value() && del_in_edges.has_value()) {
|
||||
// del_edges->insert(del_in_edges->end(), std::make_move_iterator(del_in_edges->begin()),
|
||||
// std::make_move_iterator(del_in_edges->end()));
|
||||
// return std::make_pair(*del_vertex, *del_edges);
|
||||
// }
|
||||
// return std::nullopt;
|
||||
// }
|
||||
|
||||
// /// STORING
|
||||
// /// -----------------------------------------------------------
|
||||
|
||||
// /// Serialize and store in-memory vertex to the disk.
|
||||
// /// Properties are serialized as the value
|
||||
// void StoreVertex(const query::VertexAccessor &vertex_acc) {
|
||||
// AssertRocksDBStatus(db_->Put(rocksdb::WriteOptions(), vertex_chandle, SerializeVertex(vertex_acc),
|
||||
// SerializeProperties(vertex_acc.PropertyStore())));
|
||||
// }
|
||||
|
||||
// /// Store edge as two key-value entries in the RocksDB.
|
||||
// void StoreEdge(const query::EdgeAccessor &edge_acc) {
|
||||
// auto [src_dest_key, dest_src_key] = SerializeEdge(edge_acc);
|
||||
// const std::string value = SerializeProperties(edge_acc.PropertyStore());
|
||||
// AssertRocksDBStatus(db_->Put(rocksdb::WriteOptions(), edge_chandle, src_dest_key, value));
|
||||
// AssertRocksDBStatus(db_->Put(rocksdb::WriteOptions(), edge_chandle, dest_src_key, value));
|
||||
// }
|
||||
|
||||
// /// UPDATE PART
|
||||
// /// -----------------------------------------------------------
|
||||
|
||||
// /// Clear all entries from the database.
|
||||
// /// TODO: check if this deletes all entries, or you also need to specify handle here
|
||||
// /// TODO: This will not be needed in the production code and can possibly removed in testing
|
||||
// void Clear() {
|
||||
// auto it = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions()));
|
||||
// for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
// db_->Delete(rocksdb::WriteOptions(), it->key().ToString());
|
||||
// }
|
||||
// }
|
||||
|
||||
// /// READ PART
|
||||
// /// -----------------------------------------------------------
|
||||
|
||||
// /// TODO: if the need comes for using also a GID object, use std::variant
|
||||
// /// This should again be changed when we have mulitple same vertices
|
||||
// std::optional<query::VertexAccessor> FindVertex(const std::string_view gid, query::DbAccessor &dba) {
|
||||
// auto it = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle));
|
||||
// std::optional<query::VertexAccessor> result = {};
|
||||
// for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
// const auto &key = it->key().ToString();
|
||||
// if (const auto vertex_parts = utils::Split(key, "|"); vertex_parts[1] == gid) {
|
||||
// result = DeserializeVertex(key, it->value().ToStringView(), dba);
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// return result;
|
||||
// }
|
||||
|
||||
// /// Get all vertices by a label.
|
||||
// std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::LabelId &label_id) {
|
||||
// return Vertices(dba, [label_id](const auto &vertex) {
|
||||
// const auto res = vertex.HasLabel(storage::View::OLD, label_id);
|
||||
// return !res.HasError() && *res;
|
||||
// });
|
||||
// }
|
||||
|
||||
// /// Read all vertices stored in the database by a property
|
||||
// std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const storage::PropertyId &property_id,
|
||||
// const storage::PropertyValue &property_value) {
|
||||
// return Vertices(dba, [property_id, property_value](const auto &vertex) {
|
||||
// const auto res = vertex.GetProperty(storage::View::OLD, property_id);
|
||||
// return !res.HasError() && *res == property_value;
|
||||
// });
|
||||
// }
|
||||
|
||||
// /// Get all vertices.
|
||||
// std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba) {
|
||||
// return Vertices(dba, [](const auto & /*vertex*/) { return true; });
|
||||
// }
|
||||
|
||||
// /// Read all vertices stored in the database and filter them by a lambda function.
|
||||
// std::vector<query::VertexAccessor> Vertices(query::DbAccessor &dba, const auto &vertex_filter) {
|
||||
// std::vector<query::VertexAccessor> vertices;
|
||||
// auto it = std::unique_ptr<rocksdb::Iterator>(db_->NewIterator(rocksdb::ReadOptions(), vertex_chandle));
|
||||
// for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
// auto vertex = DeserializeVertex(it->key().ToStringView(), it->value().ToStringView(), dba);
|
||||
// if (vertex_filter(vertex)) {
|
||||
// vertices.push_back(vertex);
|
||||
// }
|
||||
// }
|
||||
// return vertices;
|
||||
// }
|
||||
|
||||
// private:
|
||||
// /// Serialization of properties is done by saving the property store buffer
|
||||
// /// If the data is stored in the local buffer of the property store, data from the buffer is copied to the string
|
||||
// /// If the data is stored in some external buffer, the data is read from that location and copied to the string
|
||||
// inline std::string SerializeProperties(const auto &&properties) { return properties; }
|
||||
|
||||
// /// Serialize labels delimitied by | to string
|
||||
// std::string SerializeLabels(const auto &&labels) {
|
||||
// if (labels.HasError() || (*labels).empty()) {
|
||||
// return "";
|
||||
// }
|
||||
// std::string result = std::to_string((*labels)[0].AsUint());
|
||||
// std::string ser_labels = std::accumulate(
|
||||
// std::next((*labels).begin()), (*labels).end(), result,
|
||||
// [](const std::string &join, const auto &label_id) { return join + "," + std::to_string(label_id.AsUint());
|
||||
// });
|
||||
// return ser_labels;
|
||||
// }
|
||||
|
||||
// /// Serializes id type to string
|
||||
// inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); }
|
||||
|
||||
// /// Serialize vertex to string
|
||||
// /// The format: | label1,label2,label3 | gid
|
||||
// std::string SerializeVertex(const query::VertexAccessor &vertex_acc) {
|
||||
// std::string result = SerializeLabels(vertex_acc.Labels(storage::View::OLD)) + "|";
|
||||
// result += SerializeIdType(vertex_acc.Gid());
|
||||
// return result;
|
||||
// }
|
||||
|
||||
// /// Deserialize vertex from string
|
||||
// /// Properties are read from value and set to the vertex later
|
||||
// query::VertexAccessor DeserializeVertex(const std::string_view key, const std::string_view value,
|
||||
// query::DbAccessor &dba) {
|
||||
// /// Create vertex
|
||||
// auto impl = dba.InsertVertex();
|
||||
// spdlog::info("Key to deserialize: {}", key);
|
||||
// const auto vertex_parts = utils::Split(key, "|");
|
||||
// // Deserialize labels
|
||||
// if (!vertex_parts[0].empty()) {
|
||||
// const auto labels = utils::Split(vertex_parts[0], ",");
|
||||
// for (const auto &label : labels) {
|
||||
// const storage::LabelId label_id = storage::LabelId::FromUint(std::stoull(label));
|
||||
// auto maybe_error = impl.AddLabel(label_id);
|
||||
// if (maybe_error.HasError()) {
|
||||
// switch (maybe_error.GetError()) {
|
||||
// case storage::Error::SERIALIZATION_ERROR:
|
||||
// throw utils::BasicException("Serialization");
|
||||
// case storage::Error::DELETED_OBJECT:
|
||||
// throw utils::BasicException("Trying to set a label on a deleted node.");
|
||||
// case storage::Error::VERTEX_HAS_EDGES:
|
||||
// case storage::Error::PROPERTIES_DISABLED:
|
||||
// case storage::Error::NONEXISTENT_OBJECT:
|
||||
// throw utils::BasicException("Unexpected error when setting a label.");
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// impl.SetGid(storage::Gid::FromUint(std::stoull(vertex_parts[1])));
|
||||
// impl.SetPropertyStore(value);
|
||||
// return impl;
|
||||
// }
|
||||
|
||||
// /// Serializes edge accessor to obtain a key for the key-value store.
|
||||
// /// @return two strings because there will be two keys since edge is stored in both directions.
|
||||
// // | from_gid | to_gid | direction | edge_type | edge_gid
|
||||
// std::pair<std::string, std::string> SerializeEdge(const query::EdgeAccessor &edge_acc) {
|
||||
// // Serialized objects
|
||||
// auto from_gid = SerializeIdType(edge_acc.From().Gid());
|
||||
// auto to_gid = SerializeIdType(edge_acc.To().Gid());
|
||||
// auto edge_type = SerializeIdType(edge_acc.EdgeType());
|
||||
// auto edge_gid = SerializeIdType(edge_acc.Gid());
|
||||
// // source->destination key
|
||||
// std::string src_dest_key = from_gid + "|";
|
||||
// src_dest_key += to_gid + "|";
|
||||
// src_dest_key += outEdgeDirection;
|
||||
// src_dest_key += "|" + edge_type + "|";
|
||||
// src_dest_key += edge_gid;
|
||||
// // destination->source key
|
||||
// std::string dest_src_key = to_gid + "|";
|
||||
// dest_src_key += from_gid + "|";
|
||||
// dest_src_key += inEdgeDirection;
|
||||
// dest_src_key += "|" + edge_type + "|";
|
||||
// dest_src_key += edge_gid;
|
||||
// return {src_dest_key, dest_src_key};
|
||||
// }
|
||||
|
||||
// /// Deserialize edge from the given key-value.
|
||||
// /// Properties are read from value and set to the edge later.
|
||||
// query::EdgeAccessor DeserializeEdge(const std::string_view key, const std::string_view value,
|
||||
// query::DbAccessor &dba) {
|
||||
// const auto edge_parts = utils::Split(key, "|");
|
||||
// auto [from_gid, to_gid] = std::invoke(
|
||||
// [&](const auto &edge_parts) {
|
||||
// if (edge_parts[2] == "0") { // out edge
|
||||
// return std::make_pair(edge_parts[0], edge_parts[1]);
|
||||
// }
|
||||
// // in edge
|
||||
// return std::make_pair(edge_parts[1], edge_parts[0]);
|
||||
// },
|
||||
// edge_parts);
|
||||
// // load vertex accessors
|
||||
// auto from_acc = FindVertex(from_gid, dba);
|
||||
// auto to_acc = FindVertex(to_gid, dba);
|
||||
// if (!from_acc.has_value() || !to_acc.has_value()) {
|
||||
// throw utils::BasicException("Non-existing vertices during edge deserialization");
|
||||
// }
|
||||
// const auto edge_type_id = storage::EdgeTypeId::FromUint(std::stoull(edge_parts[3]));
|
||||
// auto maybe_edge = dba.InsertEdge(&*from_acc, &*to_acc, edge_type_id);
|
||||
// MG_ASSERT(maybe_edge.HasValue());
|
||||
// // in the new storage API, setting gid must be done atomically
|
||||
// maybe_edge->SetGid(storage::Gid::FromUint(std::stoull(edge_parts[4])));
|
||||
// maybe_edge->SetPropertyStore(value);
|
||||
// return *maybe_edge;
|
||||
// }
|
||||
|
||||
rocksdb::Options options_;
|
||||
rocksdb::DB *db_;
|
||||
rocksdb::ColumnFamilyHandle *vertex_chandle = nullptr;
|
||||
rocksdb::ColumnFamilyHandle *edge_chandle = nullptr;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage::rocks
|
||||
621
src/storage/v2/disk/vertex_accessor.cpp
Normal file
621
src/storage/v2/disk/vertex_accessor.cpp
Normal file
@@ -0,0 +1,621 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "storage/v2/disk/vertex_accessor.hpp"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "storage/v2/disk/edge_accessor.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/indices.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
namespace detail {
|
||||
namespace {
|
||||
std::pair<bool, bool> IsVisible(Vertex *vertex, Transaction *transaction, View view) {
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(vertex->lock);
|
||||
deleted = vertex->deleted;
|
||||
delta = vertex->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction, delta, view, [&](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return {exists, deleted};
|
||||
}
|
||||
} // namespace
|
||||
} // namespace detail
|
||||
|
||||
std::unique_ptr<DiskVertexAccessor> DiskVertexAccessor::Create(Vertex *vertex, Transaction *transaction,
|
||||
Indices *indices, Constraints *constraints,
|
||||
Config::Items config, View view) {
|
||||
// if (const auto [exists, deleted] = detail::IsVisible(vertex, transaction, view); !exists || deleted) {
|
||||
// return {};
|
||||
// }
|
||||
//
|
||||
// return std::make_unique<DiskVertexAccessor>(vertex, transaction, indices, constraints, config);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::Create");
|
||||
}
|
||||
|
||||
bool DiskVertexAccessor::IsVisible(View view) const {
|
||||
// const auto [exists, deleted] = detail::IsVisible(vertex_, transaction_, view);
|
||||
// return exists && (for_deleted_ || !deleted);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::IsVisible");
|
||||
}
|
||||
|
||||
Result<bool> DiskVertexAccessor::AddLabel(LabelId label) {
|
||||
// utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (vertex_->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// if (std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end()) return false;
|
||||
//
|
||||
// CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label);
|
||||
//
|
||||
// vertex_->labels.push_back(label);
|
||||
//
|
||||
// UpdateOnAddLabel(indices_, label, vertex_, *transaction_);
|
||||
//
|
||||
// return true;
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::AddLabel");
|
||||
}
|
||||
|
||||
Result<bool> DiskVertexAccessor::RemoveLabel(LabelId label) {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (vertex_->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// auto it = std::find(vertex_->labels.begin(), vertex_->labels.end(), label);
|
||||
// if (it == vertex_->labels.end()) return false;
|
||||
//
|
||||
// CreateAndLinkDelta(transaction_, vertex_, Delta::AddLabelTag(), label);
|
||||
//
|
||||
// std::swap(*it, *vertex_->labels.rbegin());
|
||||
// vertex_->labels.pop_back();
|
||||
// return true;
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::RemoveLabel");
|
||||
}
|
||||
|
||||
Result<bool> DiskVertexAccessor::HasLabel(LabelId label, View view) const {
|
||||
// bool exists = true;
|
||||
// bool deleted = false;
|
||||
// bool has_label = false;
|
||||
// Delta *delta = nullptr;
|
||||
// {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
// deleted = vertex_->deleted;
|
||||
// has_label = std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end();
|
||||
// delta = vertex_->delta;
|
||||
// }
|
||||
// ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &has_label, label](const Delta &delta) {
|
||||
// switch (delta.action) {
|
||||
// case Delta::Action::REMOVE_LABEL: {
|
||||
// if (delta.label == label) {
|
||||
// MG_ASSERT(has_label, "Invalid database state!");
|
||||
// has_label = false;
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::ADD_LABEL: {
|
||||
// if (delta.label == label) {
|
||||
// MG_ASSERT(!has_label, "Invalid database state!");
|
||||
// has_label = true;
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::DELETE_OBJECT: {
|
||||
// exists = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::RECREATE_OBJECT: {
|
||||
// deleted = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::SET_PROPERTY:
|
||||
// case Delta::Action::ADD_IN_EDGE:
|
||||
// case Delta::Action::ADD_OUT_EDGE:
|
||||
// case Delta::Action::REMOVE_IN_EDGE:
|
||||
// case Delta::Action::REMOVE_OUT_EDGE:
|
||||
// break;
|
||||
// }
|
||||
// });
|
||||
// if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
// if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
// return has_label;
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::HasLabel");
|
||||
}
|
||||
|
||||
Result<std::vector<LabelId>> DiskVertexAccessor::Labels(View view) const {
|
||||
// bool exists = true;
|
||||
// bool deleted = false;
|
||||
// std::vector<LabelId> labels;
|
||||
// Delta *delta = nullptr;
|
||||
// {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
// deleted = vertex_->deleted;
|
||||
// labels = vertex_->labels;
|
||||
// delta = vertex_->delta;
|
||||
// }
|
||||
// ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &labels](const Delta &delta) {
|
||||
// switch (delta.action) {
|
||||
// case Delta::Action::REMOVE_LABEL: {
|
||||
// // Remove the label because we don't see the addition.
|
||||
// auto it = std::find(labels.begin(), labels.end(), delta.label);
|
||||
// MG_ASSERT(it != labels.end(), "Invalid database state!");
|
||||
// std::swap(*it, *labels.rbegin());
|
||||
// labels.pop_back();
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::ADD_LABEL: {
|
||||
// // Add the label because we don't see the removal.
|
||||
// auto it = std::find(labels.begin(), labels.end(), delta.label);
|
||||
// MG_ASSERT(it == labels.end(), "Invalid database state!");
|
||||
// labels.push_back(delta.label);
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::DELETE_OBJECT: {
|
||||
// exists = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::RECREATE_OBJECT: {
|
||||
// deleted = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::SET_PROPERTY:
|
||||
// case Delta::Action::ADD_IN_EDGE:
|
||||
// case Delta::Action::ADD_OUT_EDGE:
|
||||
// case Delta::Action::REMOVE_IN_EDGE:
|
||||
// case Delta::Action::REMOVE_OUT_EDGE:
|
||||
// break;
|
||||
// }
|
||||
// });
|
||||
// if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
// if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
// return std::move(labels);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::Labels");
|
||||
}
|
||||
|
||||
Result<PropertyValue> DiskVertexAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
|
||||
// utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (vertex_->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// auto current_value = vertex_->properties.GetProperty(property);
|
||||
// We could skip setting the value if the previous one is the same to the new
|
||||
// one. This would save some memory as a delta would not be created as well as
|
||||
// avoid copying the value. The reason we are not doing that is because the
|
||||
// current code always follows the logical pattern of "create a delta" and
|
||||
// "modify in-place". Additionally, the created delta will make other
|
||||
// transactions get a SERIALIZATION_ERROR.
|
||||
// CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, current_value);
|
||||
// vertex_->properties.SetProperty(property, value);
|
||||
//
|
||||
// UpdateOnSetProperty(indices_, property, value, vertex_, *transaction_);
|
||||
//
|
||||
// return std::move(current_value);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::SetProperty");
|
||||
}
|
||||
|
||||
Result<bool> DiskVertexAccessor::InitProperties(
|
||||
const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
// utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (vertex_->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// if (!vertex_->properties.InitProperties(properties)) return false;
|
||||
// for (const auto &[property, value] : properties) {
|
||||
// CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, PropertyValue());
|
||||
// UpdateOnSetProperty(indices_, property, value, vertex_, *transaction_);
|
||||
// }
|
||||
//
|
||||
// return true;
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::InitProperties");
|
||||
}
|
||||
|
||||
Result<std::map<PropertyId, PropertyValue>> DiskVertexAccessor::ClearProperties() {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
//
|
||||
// if (!PrepareForWrite(transaction_, vertex_)) return Error::SERIALIZATION_ERROR;
|
||||
//
|
||||
// if (vertex_->deleted) return Error::DELETED_OBJECT;
|
||||
//
|
||||
// auto properties = vertex_->properties.Properties();
|
||||
// for (const auto &property : properties) {
|
||||
// CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property.first, property.second);
|
||||
// UpdateOnSetProperty(indices_, property.first, PropertyValue(), vertex_, *transaction_);
|
||||
// }
|
||||
//
|
||||
// vertex_->properties.ClearProperties();
|
||||
//
|
||||
// return std::move(properties);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::ClearProperties");
|
||||
}
|
||||
|
||||
Result<PropertyValue> DiskVertexAccessor::GetProperty(PropertyId property, View view) const {
|
||||
// bool exists = true;
|
||||
// bool deleted = false;
|
||||
// PropertyValue value;
|
||||
// Delta *delta = nullptr;
|
||||
// {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
// deleted = vertex_->deleted;
|
||||
// value = vertex_->properties.GetProperty(property);
|
||||
// delta = vertex_->delta;
|
||||
// }
|
||||
// ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &value, property](const Delta &delta) {
|
||||
// switch (delta.action) {
|
||||
// case Delta::Action::SET_PROPERTY: {
|
||||
// if (delta.property.key == property) {
|
||||
// value = delta.property.value;
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::DELETE_OBJECT: {
|
||||
// exists = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::RECREATE_OBJECT: {
|
||||
// deleted = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::ADD_LABEL:
|
||||
// case Delta::Action::REMOVE_LABEL:
|
||||
// case Delta::Action::ADD_IN_EDGE:
|
||||
// case Delta::Action::ADD_OUT_EDGE:
|
||||
// case Delta::Action::REMOVE_IN_EDGE:
|
||||
// case Delta::Action::REMOVE_OUT_EDGE:
|
||||
// break;
|
||||
// }
|
||||
// });
|
||||
// if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
// if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
// return std::move(value);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::GetProperty");
|
||||
}
|
||||
|
||||
Result<std::map<PropertyId, PropertyValue>> DiskVertexAccessor::Properties(View view) const {
|
||||
// bool exists = true;
|
||||
// bool deleted = false;
|
||||
// std::map<PropertyId, PropertyValue> properties;
|
||||
// Delta *delta = nullptr;
|
||||
// {
|
||||
// std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
// deleted = vertex_->deleted;
|
||||
// properties = vertex_->properties.Properties();
|
||||
// delta = vertex_->delta;
|
||||
// }
|
||||
// ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, &properties](const Delta &delta) {
|
||||
// switch (delta.action) {
|
||||
// case Delta::Action::SET_PROPERTY: {
|
||||
// auto it = properties.find(delta.property.key);
|
||||
// if (it != properties.end()) {
|
||||
// if (delta.property.value.IsNull()) {
|
||||
// // remove the property
|
||||
// properties.erase(it);
|
||||
// } else {
|
||||
// // set the value
|
||||
// it->second = delta.property.value;
|
||||
// }
|
||||
// } else if (!delta.property.value.IsNull()) {
|
||||
// properties.emplace(delta.property.key, delta.property.value);
|
||||
// }
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::DELETE_OBJECT: {
|
||||
// exists = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::RECREATE_OBJECT: {
|
||||
// deleted = false;
|
||||
// break;
|
||||
// }
|
||||
// case Delta::Action::ADD_LABEL:
|
||||
// case Delta::Action::REMOVE_LABEL:
|
||||
// case Delta::Action::ADD_IN_EDGE:
|
||||
// case Delta::Action::ADD_OUT_EDGE:
|
||||
// case Delta::Action::REMOVE_IN_EDGE:
|
||||
// case Delta::Action::REMOVE_OUT_EDGE:
|
||||
// break;
|
||||
// }
|
||||
// });
|
||||
// if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
// if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
// return std::move(properties);
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::Properties");
|
||||
}
|
||||
|
||||
Result<std::vector<std::unique_ptr<EdgeAccessor>>> DiskVertexAccessor::InEdges(
|
||||
View view, const std::vector<EdgeTypeId> &edge_types, const VertexAccessor *destination) const {
|
||||
/*
|
||||
auto *destVA = dynamic_cast<const DiskVertexAccessor *>(destination);
|
||||
MG_ASSERT(!destination || destVA, "Target VertexAccessor must be from the same storage as the storage accessor!");
|
||||
MG_ASSERT(!destVA || destVA->transaction_ == transaction_, "Invalid accessor!");
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> in_edges;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
deleted = vertex_->deleted;
|
||||
if (edge_types.empty() && !destVA) {
|
||||
in_edges = vertex_->in_edges;
|
||||
} else {
|
||||
for (const auto &item : vertex_->in_edges) {
|
||||
const auto &[edge_type, from_vertex, edge] = item;
|
||||
if (destVA && from_vertex != destVA->vertex_) continue;
|
||||
if (!edge_types.empty() && std::find(edge_types.begin(), edge_types.end(), edge_type) == edge_types.end())
|
||||
continue;
|
||||
in_edges.push_back(item);
|
||||
}
|
||||
}
|
||||
delta = vertex_->delta;
|
||||
}
|
||||
ApplyDeltasForRead(
|
||||
transaction_, delta, view, [&exists, &deleted, &in_edges, &edge_types, &destVA](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_IN_EDGE: {
|
||||
if (destVA && delta.vertex_edge.vertex != destVA->vertex_) break;
|
||||
if (!edge_types.empty() &&
|
||||
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
|
||||
break;
|
||||
// Add the edge because we don't see the removal.
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
|
||||
delta.vertex_edge.edge};
|
||||
auto it = std::find(in_edges.begin(), in_edges.end(), link);
|
||||
MG_ASSERT(it == in_edges.end(), "Invalid database state!");
|
||||
in_edges.push_back(link);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_IN_EDGE: {
|
||||
if (destVA && delta.vertex_edge.vertex != destVA->vertex_) break;
|
||||
if (!edge_types.empty() &&
|
||||
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
|
||||
break;
|
||||
// Remove the label because we don't see the addition.
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
|
||||
delta.vertex_edge.edge};
|
||||
auto it = std::find(in_edges.begin(), in_edges.end(), link);
|
||||
MG_ASSERT(it != in_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *in_edges.rbegin());
|
||||
in_edges.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (deleted) return Error::DELETED_OBJECT;
|
||||
std::vector<std::unique_ptr<EdgeAccessor>> ret;
|
||||
ret.reserve(in_edges.size());
|
||||
for (const auto &item : in_edges) {
|
||||
const auto &[edge_type, from_vertex, edge] = item;
|
||||
ret.emplace_back(std::make_unique<DiskEdgeAccessor>(edge, edge_type, from_vertex, vertex_, transaction_,
|
||||
indices_, constraints_, config_));
|
||||
}
|
||||
return std::move(ret);*/
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::InEdges");
|
||||
}
|
||||
|
||||
Result<std::vector<std::unique_ptr<EdgeAccessor>>> DiskVertexAccessor::OutEdges(
|
||||
View view, const std::vector<EdgeTypeId> &edge_types, const VertexAccessor *destination) const {
|
||||
/*auto *destVA = dynamic_cast<const DiskVertexAccessor *>(destination);
|
||||
MG_ASSERT(!destination || destVA, "Target VertexAccessor must be from the same storage as the storage accessor!");
|
||||
MG_ASSERT(!destVA || destVA->transaction_ == transaction_, "Invalid accessor!");
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
std::vector<std::tuple<EdgeTypeId, Vertex *, EdgeRef>> out_edges;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
deleted = vertex_->deleted;
|
||||
if (edge_types.empty() && !destVA) {
|
||||
out_edges = vertex_->out_edges;
|
||||
} else {
|
||||
for (const auto &item : vertex_->out_edges) {
|
||||
const auto &[edge_type, to_vertex, edge] = item;
|
||||
if (destVA && to_vertex != destVA->vertex_) continue;
|
||||
if (!edge_types.empty() && std::find(edge_types.begin(), edge_types.end(), edge_type) == edge_types.end())
|
||||
continue;
|
||||
out_edges.push_back(item);
|
||||
}
|
||||
}
|
||||
delta = vertex_->delta;
|
||||
}
|
||||
ApplyDeltasForRead(
|
||||
transaction_, delta, view, [&exists, &deleted, &out_edges, &edge_types, &destVA](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_OUT_EDGE: {
|
||||
if (destVA && delta.vertex_edge.vertex != destVA->vertex_) break;
|
||||
if (!edge_types.empty() &&
|
||||
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
|
||||
break;
|
||||
// Add the edge because we don't see the removal.
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
|
||||
delta.vertex_edge.edge};
|
||||
auto it = std::find(out_edges.begin(), out_edges.end(), link);
|
||||
MG_ASSERT(it == out_edges.end(), "Invalid database state!");
|
||||
out_edges.push_back(link);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
if (destVA && delta.vertex_edge.vertex != destVA->vertex_) break;
|
||||
if (!edge_types.empty() &&
|
||||
std::find(edge_types.begin(), edge_types.end(), delta.vertex_edge.edge_type) == edge_types.end())
|
||||
break;
|
||||
// Remove the label because we don't see the addition.
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{delta.vertex_edge.edge_type, delta.vertex_edge.vertex,
|
||||
delta.vertex_edge.edge};
|
||||
auto it = std::find(out_edges.begin(), out_edges.end(), link);
|
||||
MG_ASSERT(it != out_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *out_edges.rbegin());
|
||||
out_edges.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
exists = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (deleted) return Error::DELETED_OBJECT;
|
||||
std::vector<std::unique_ptr<EdgeAccessor>> ret;
|
||||
ret.reserve(out_edges.size());
|
||||
for (const auto &item : out_edges) {
|
||||
const auto &[edge_type, to_vertex, edge] = item;
|
||||
ret.emplace_back(std::make_unique<DiskEdgeAccessor>(edge, edge_type, vertex_, to_vertex, transaction_, indices_,
|
||||
constraints_, config_));
|
||||
}
|
||||
return std::move(ret);*/
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::OutEdges");
|
||||
}
|
||||
|
||||
Result<size_t> DiskVertexAccessor::InDegree(View view) const {
|
||||
/*bool exists = true;
|
||||
bool deleted = false;
|
||||
size_t degree = 0;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
deleted = vertex_->deleted;
|
||||
degree = vertex_->in_edges.size();
|
||||
delta = vertex_->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, °ree](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
++degree;
|
||||
break;
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
--degree;
|
||||
break;
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
exists = false;
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
deleted = false;
|
||||
break;
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return degree;
|
||||
}
|
||||
|
||||
Result<size_t> DiskVertexAccessor::OutDegree(View view) const {
|
||||
bool exists = true;
|
||||
bool deleted = false;
|
||||
size_t degree = 0;
|
||||
Delta *delta = nullptr;
|
||||
{
|
||||
std::lock_guard<utils::SpinLock> guard(vertex_->lock);
|
||||
deleted = vertex_->deleted;
|
||||
degree = vertex_->out_edges.size();
|
||||
delta = vertex_->delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction_, delta, view, [&exists, &deleted, °ree](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
++degree;
|
||||
break;
|
||||
case Delta::Action::REMOVE_OUT_EDGE:
|
||||
--degree;
|
||||
break;
|
||||
case Delta::Action::DELETE_OBJECT:
|
||||
exists = false;
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT:
|
||||
deleted = false;
|
||||
break;
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return degree;
|
||||
*/
|
||||
throw utils::NotYetImplemented("DiskVertexAccessor::OutDegree");
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage
|
||||
117
src/storage/v2/disk/vertex_accessor.hpp
Normal file
117
src/storage/v2/disk/vertex_accessor.hpp
Normal file
@@ -0,0 +1,117 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "storage/v2/vertex.hpp"
|
||||
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
class EdgeAccessor;
|
||||
class Storage;
|
||||
struct Indices;
|
||||
struct Constraints;
|
||||
|
||||
class DiskVertexAccessor final : public VertexAccessor {
|
||||
private:
|
||||
friend class DiskStorage;
|
||||
|
||||
public:
|
||||
DiskVertexAccessor(Vertex *vertex, Transaction *transaction, Indices *indices, Constraints *constraints,
|
||||
Config::Items config, bool for_deleted = false)
|
||||
: VertexAccessor(transaction, config, for_deleted),
|
||||
vertex_(vertex),
|
||||
indices_(indices),
|
||||
constraints_(constraints) {}
|
||||
|
||||
static std::unique_ptr<DiskVertexAccessor> Create(Vertex *vertex, Transaction *transaction, Indices *indices,
|
||||
Constraints *constraints, Config::Items config, View view);
|
||||
|
||||
/// @return true if the object is visible from the current transaction
|
||||
bool IsVisible(View view) const override;
|
||||
|
||||
/// Add a label and return `true` if insertion took place.
|
||||
/// `false` is returned if the label already existed.
|
||||
/// @throw std::bad_alloc
|
||||
Result<bool> AddLabel(LabelId label) override;
|
||||
|
||||
/// Remove a label and return `true` if deletion took place.
|
||||
/// `false` is returned if the vertex did not have a label already.
|
||||
/// @throw std::bad_alloc
|
||||
Result<bool> RemoveLabel(LabelId label) override;
|
||||
|
||||
Result<bool> HasLabel(LabelId label, View view) const override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error if the resulting vector exceeds
|
||||
/// std::vector::max_size().
|
||||
Result<std::vector<LabelId>> Labels(View view) const override;
|
||||
|
||||
/// Set a property value and return the old value.
|
||||
/// @throw std::bad_alloc
|
||||
Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value) override;
|
||||
|
||||
/// Set property values only if property store is empty. Returns `true` if successully set all values,
|
||||
/// `false` otherwise.
|
||||
/// @throw std::bad_alloc
|
||||
Result<bool> InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) override;
|
||||
|
||||
/// Remove all properties and return the values of the removed properties.
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::map<PropertyId, PropertyValue>> ClearProperties() override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<PropertyValue> GetProperty(PropertyId property, View view) const override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error if the resulting vector exceeds
|
||||
/// std::vector::max_size().
|
||||
Result<std::vector<std::unique_ptr<EdgeAccessor>>> InEdges(View view, const std::vector<EdgeTypeId> &edge_types,
|
||||
const VertexAccessor *destination) const override;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error if the resulting vector exceeds
|
||||
/// std::vector::max_size().
|
||||
Result<std::vector<std::unique_ptr<EdgeAccessor>>> OutEdges(View view, const std::vector<EdgeTypeId> &edge_types,
|
||||
const VertexAccessor *destination) const override;
|
||||
|
||||
Result<size_t> InDegree(View view) const override;
|
||||
|
||||
Result<size_t> OutDegree(View view) const override;
|
||||
|
||||
storage::Gid Gid() const noexcept override { return vertex_->gid; }
|
||||
|
||||
bool operator==(const VertexAccessor &other) const noexcept override {
|
||||
const auto *otherVertex = dynamic_cast<const DiskVertexAccessor *>(&other);
|
||||
if (otherVertex == nullptr) return false;
|
||||
return vertex_ == otherVertex->vertex_ && transaction_ == otherVertex->transaction_;
|
||||
}
|
||||
|
||||
bool operator!=(const VertexAccessor &other) const noexcept { return !(*this == other); }
|
||||
|
||||
private:
|
||||
Vertex *vertex_;
|
||||
Indices *indices_;
|
||||
Constraints *constraints_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,7 +19,6 @@
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/edge_ref.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -714,11 +713,11 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
// type and invalid from/to pointers because we don't know them here,
|
||||
// but that isn't an issue because we won't use that part of the API
|
||||
// here.
|
||||
auto ea = EdgeAccessor::Create(edge_ref, EdgeTypeId::FromUint(0UL), nullptr, nullptr, transaction, indices,
|
||||
constraints, items);
|
||||
auto ea =
|
||||
EdgeAccessor{edge_ref, EdgeTypeId::FromUint(0UL), nullptr, nullptr, transaction, indices, constraints, items};
|
||||
|
||||
// Get edge data.
|
||||
auto maybe_props = ea->Properties(View::OLD);
|
||||
auto maybe_props = ea.Properties(View::OLD);
|
||||
MG_ASSERT(maybe_props.HasValue(), "Invalid database state!");
|
||||
|
||||
// Store the edge.
|
||||
@@ -776,16 +775,16 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
const auto &in_edges = maybe_in_edges.GetValue();
|
||||
snapshot.WriteUint(in_edges.size());
|
||||
for (const auto &item : in_edges) {
|
||||
snapshot.WriteUint(item->Gid().AsUint());
|
||||
snapshot.WriteUint(item->FromVertex()->Gid().AsUint());
|
||||
write_mapping(item->EdgeType());
|
||||
snapshot.WriteUint(item.Gid().AsUint());
|
||||
snapshot.WriteUint(item.FromVertex().Gid().AsUint());
|
||||
write_mapping(item.EdgeType());
|
||||
}
|
||||
const auto &out_edges = maybe_out_edges.GetValue();
|
||||
snapshot.WriteUint(out_edges.size());
|
||||
for (const auto &item : out_edges) {
|
||||
snapshot.WriteUint(item->Gid().AsUint());
|
||||
snapshot.WriteUint(item->ToVertex()->Gid().AsUint());
|
||||
write_mapping(item->EdgeType());
|
||||
snapshot.WriteUint(item.Gid().AsUint());
|
||||
snapshot.WriteUint(item.ToVertex().Gid().AsUint());
|
||||
write_mapping(item.EdgeType());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -327,7 +327,10 @@ void LabelIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp) {
|
||||
}
|
||||
|
||||
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
|
||||
: self_(self), index_iterator_(index_iterator), current_vertex_accessor_(nullptr), current_vertex_(nullptr) {
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_),
|
||||
current_vertex_(nullptr) {
|
||||
AdvanceUntilValid();
|
||||
}
|
||||
|
||||
@@ -344,8 +347,8 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
}
|
||||
if (CurrentVersionHasLabel(*index_iterator_->vertex, self_->label_, self_->transaction_, self_->view_)) {
|
||||
current_vertex_ = index_iterator_->vertex;
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
current_vertex_accessor_ =
|
||||
VertexAccessor{current_vertex_, self_->transaction_, self_->indices_, self_->constraints_, self_->config_};
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -475,7 +478,10 @@ void LabelPropertyIndex::RemoveObsoleteEntries(uint64_t oldest_active_start_time
|
||||
}
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
|
||||
: self_(self), index_iterator_(index_iterator), current_vertex_accessor_(nullptr), current_vertex_(nullptr) {
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, nullptr, self_->config_),
|
||||
current_vertex_(nullptr) {
|
||||
AdvanceUntilValid();
|
||||
}
|
||||
|
||||
@@ -513,8 +519,8 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
if (CurrentVersionHasLabelProperty(*index_iterator_->vertex, self_->label_, self_->property_,
|
||||
index_iterator_->value, self_->transaction_, self_->view_)) {
|
||||
current_vertex_ = index_iterator_->vertex;
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
current_vertex_accessor_ =
|
||||
VertexAccessor(current_vertex_, self_->transaction_, self_->indices_, self_->constraints_, self_->config_);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,7 +78,7 @@ class LabelIndex {
|
||||
public:
|
||||
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
|
||||
|
||||
VertexAccessor *operator*() const { return current_vertex_accessor_.get(); }
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
bool operator==(const Iterator &other) const { return index_iterator_ == other.index_iterator_; }
|
||||
bool operator!=(const Iterator &other) const { return index_iterator_ != other.index_iterator_; }
|
||||
@@ -90,7 +90,7 @@ class LabelIndex {
|
||||
|
||||
Iterable *self_;
|
||||
utils::SkipList<Entry>::Iterator index_iterator_;
|
||||
std::unique_ptr<VertexAccessor> current_vertex_accessor_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
|
||||
@@ -181,7 +181,7 @@ class LabelPropertyIndex {
|
||||
public:
|
||||
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
|
||||
|
||||
VertexAccessor *operator*() const { return current_vertex_accessor_.get(); }
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
bool operator==(const Iterator &other) const { return index_iterator_ == other.index_iterator_; }
|
||||
bool operator!=(const Iterator &other) const { return index_iterator_ != other.index_iterator_; }
|
||||
@@ -193,7 +193,7 @@ class LabelPropertyIndex {
|
||||
|
||||
Iterable *self_;
|
||||
utils::SkipList<Entry>::Iterator index_iterator_;
|
||||
std::unique_ptr<VertexAccessor> current_vertex_accessor_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
@@ -11,9 +11,12 @@
|
||||
|
||||
#include "storage/v2/property_store.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
@@ -1219,4 +1222,37 @@ bool PropertyStore::ClearProperties() {
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string PropertyStore::StringBuffer() {
|
||||
uint64_t size = 0;
|
||||
uint8_t *data = nullptr;
|
||||
std::tie(size, data) = GetSizeData(buffer_);
|
||||
if (size % 8 != 0) { // We are storing the data in the local buffer.
|
||||
size = sizeof(buffer_) - 1;
|
||||
data = &buffer_[1];
|
||||
}
|
||||
std::string arr(size, ' ');
|
||||
for (uint i = 0; i < size; ++i) {
|
||||
arr[i] = static_cast<char>(data[i]);
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
void PropertyStore::SetBuffer(const std::string_view buffer) {
|
||||
uint64_t size = 0;
|
||||
uint8_t *data = nullptr;
|
||||
if (buffer.size() == sizeof(buffer_) - 1) { // use local buffer
|
||||
buffer_[0] = kUseLocalBuffer;
|
||||
size = buffer.size() - 1;
|
||||
data = &buffer_[1];
|
||||
} else {
|
||||
size = buffer.size();
|
||||
data = new uint8_t[size];
|
||||
SetSizeData(buffer_, size, data);
|
||||
}
|
||||
|
||||
for (uint i = 0; i < size; ++i) {
|
||||
data[i] = static_cast<uint8_t>(buffer[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage
|
||||
|
||||
@@ -71,6 +71,12 @@ class PropertyStore {
|
||||
/// @throw std::bad_alloc
|
||||
bool ClearProperties();
|
||||
|
||||
/// Return property buffer as a string
|
||||
std::string StringBuffer();
|
||||
|
||||
/// Sets buffer
|
||||
void SetBuffer(std::string_view buffer);
|
||||
|
||||
private:
|
||||
uint8_t buffer_[sizeof(uint64_t) + sizeof(uint8_t *)];
|
||||
};
|
||||
|
||||
@@ -30,10 +30,9 @@ template <typename>
|
||||
} // namespace
|
||||
|
||||
////// ReplicationClient //////
|
||||
InMemoryStorage::ReplicationClient::ReplicationClient(std::string name, InMemoryStorage *storage,
|
||||
const io::network::Endpoint &endpoint,
|
||||
const replication::ReplicationMode mode,
|
||||
const replication::ReplicationClientConfig &config)
|
||||
Storage::ReplicationClient::ReplicationClient(std::string name, Storage *storage, const io::network::Endpoint &endpoint,
|
||||
const replication::ReplicationMode mode,
|
||||
const replication::ReplicationClientConfig &config)
|
||||
: name_(std::move(name)), storage_(storage), mode_(mode) {
|
||||
if (config.ssl) {
|
||||
rpc_context_.emplace(config.ssl->key_file, config.ssl->cert_file);
|
||||
@@ -50,14 +49,14 @@ InMemoryStorage::ReplicationClient::ReplicationClient(std::string name, InMemory
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::TryInitializeClientAsync() {
|
||||
void Storage::ReplicationClient::TryInitializeClientAsync() {
|
||||
thread_pool_.AddTask([this] {
|
||||
rpc_client_->Abort();
|
||||
this->TryInitializeClientSync();
|
||||
});
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::FrequentCheck() {
|
||||
void Storage::ReplicationClient::FrequentCheck() {
|
||||
const auto is_success = std::invoke([this]() {
|
||||
try {
|
||||
auto stream{rpc_client_->Stream<replication::FrequentHeartbeatRpc>()};
|
||||
@@ -83,7 +82,7 @@ void InMemoryStorage::ReplicationClient::FrequentCheck() {
|
||||
}
|
||||
|
||||
/// @throws rpc::RpcFailedException
|
||||
void InMemoryStorage::ReplicationClient::InitializeClient() {
|
||||
void Storage::ReplicationClient::InitializeClient() {
|
||||
uint64_t current_commit_timestamp{kTimestampInitialId};
|
||||
|
||||
std::optional<std::string> epoch_id;
|
||||
@@ -135,7 +134,7 @@ void InMemoryStorage::ReplicationClient::InitializeClient() {
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::TryInitializeClientSync() {
|
||||
void Storage::ReplicationClient::TryInitializeClientSync() {
|
||||
try {
|
||||
InitializeClient();
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
@@ -146,19 +145,19 @@ void InMemoryStorage::ReplicationClient::TryInitializeClientSync() {
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::HandleRpcFailure() {
|
||||
void Storage::ReplicationClient::HandleRpcFailure() {
|
||||
spdlog::error(utils::MessageWithLink("Couldn't replicate data to {}.", name_, "https://memgr.ph/replication"));
|
||||
TryInitializeClientAsync();
|
||||
}
|
||||
|
||||
replication::SnapshotRes InMemoryStorage::ReplicationClient::TransferSnapshot(const std::filesystem::path &path) {
|
||||
replication::SnapshotRes Storage::ReplicationClient::TransferSnapshot(const std::filesystem::path &path) {
|
||||
auto stream{rpc_client_->Stream<replication::SnapshotRpc>()};
|
||||
replication::Encoder encoder(stream.GetBuilder());
|
||||
encoder.WriteFile(path);
|
||||
return stream.AwaitResponse();
|
||||
}
|
||||
|
||||
replication::WalFilesRes InMemoryStorage::ReplicationClient::TransferWalFiles(
|
||||
replication::WalFilesRes Storage::ReplicationClient::TransferWalFiles(
|
||||
const std::vector<std::filesystem::path> &wal_files) {
|
||||
MG_ASSERT(!wal_files.empty(), "Wal files list is empty!");
|
||||
auto stream{rpc_client_->Stream<replication::WalFilesRpc>(wal_files.size())};
|
||||
@@ -171,7 +170,7 @@ replication::WalFilesRes InMemoryStorage::ReplicationClient::TransferWalFiles(
|
||||
return stream.AwaitResponse();
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) {
|
||||
void Storage::ReplicationClient::StartTransactionReplication(const uint64_t current_wal_seq_num) {
|
||||
std::unique_lock guard(client_lock_);
|
||||
const auto status = replica_state_.load();
|
||||
switch (status) {
|
||||
@@ -205,8 +204,7 @@ void InMemoryStorage::ReplicationClient::StartTransactionReplication(const uint6
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::IfStreamingTransaction(
|
||||
const std::function<void(ReplicaStream &handler)> &callback) {
|
||||
void Storage::ReplicationClient::IfStreamingTransaction(const std::function<void(ReplicaStream &handler)> &callback) {
|
||||
// We can only check the state because it guarantees to be only
|
||||
// valid during a single transaction replication (if the assumption
|
||||
// that this and other transaction replication functions can only be
|
||||
@@ -226,7 +224,7 @@ void InMemoryStorage::ReplicationClient::IfStreamingTransaction(
|
||||
}
|
||||
}
|
||||
|
||||
bool InMemoryStorage::ReplicationClient::FinalizeTransactionReplication() {
|
||||
bool Storage::ReplicationClient::FinalizeTransactionReplication() {
|
||||
// We can only check the state because it guarantees to be only
|
||||
// valid during a single transaction replication (if the assumption
|
||||
// that this and other transaction replication functions can only be
|
||||
@@ -243,7 +241,7 @@ bool InMemoryStorage::ReplicationClient::FinalizeTransactionReplication() {
|
||||
}
|
||||
}
|
||||
|
||||
bool InMemoryStorage::ReplicationClient::FinalizeTransactionReplicationInternal() {
|
||||
bool Storage::ReplicationClient::FinalizeTransactionReplicationInternal() {
|
||||
MG_ASSERT(replica_stream_, "Missing stream for transaction deltas");
|
||||
try {
|
||||
auto response = replica_stream_->Finalize();
|
||||
@@ -267,7 +265,7 @@ bool InMemoryStorage::ReplicationClient::FinalizeTransactionReplicationInternal(
|
||||
return false;
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
|
||||
void Storage::ReplicationClient::RecoverReplica(uint64_t replica_commit) {
|
||||
while (true) {
|
||||
auto file_locker = storage_->file_retainer_.AddLocker();
|
||||
|
||||
@@ -329,7 +327,7 @@ void InMemoryStorage::ReplicationClient::RecoverReplica(uint64_t replica_commit)
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t InMemoryStorage::ReplicationClient::ReplicateCurrentWal() {
|
||||
uint64_t Storage::ReplicationClient::ReplicateCurrentWal() {
|
||||
const auto &wal_file = storage_->wal_file_;
|
||||
auto stream = TransferCurrentWalFile();
|
||||
stream.AppendFilename(wal_file->Path().filename());
|
||||
@@ -363,7 +361,7 @@ uint64_t InMemoryStorage::ReplicationClient::ReplicateCurrentWal() {
|
||||
/// recovery steps, so we can safely send it to the replica.
|
||||
/// We assume that the property of preserving at least 1 WAL before the snapshot
|
||||
/// is satisfied as we extract the timestamp information from it.
|
||||
std::vector<InMemoryStorage::ReplicationClient::RecoveryStep> InMemoryStorage::ReplicationClient::GetRecoverySteps(
|
||||
std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient::GetRecoverySteps(
|
||||
const uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker) {
|
||||
// First check if we can recover using the current wal file only
|
||||
// otherwise save the seq_num of the current wal file
|
||||
@@ -504,8 +502,8 @@ std::vector<InMemoryStorage::ReplicationClient::RecoveryStep> InMemoryStorage::R
|
||||
return recovery_steps;
|
||||
}
|
||||
|
||||
InMemoryStorage::TimestampInfo InMemoryStorage::ReplicationClient::GetTimestampInfo() {
|
||||
InMemoryStorage::TimestampInfo info;
|
||||
Storage::TimestampInfo Storage::ReplicationClient::GetTimestampInfo() {
|
||||
Storage::TimestampInfo info;
|
||||
info.current_timestamp_of_replica = 0;
|
||||
info.current_number_of_timestamp_behind_master = 0;
|
||||
|
||||
@@ -533,71 +531,65 @@ InMemoryStorage::TimestampInfo InMemoryStorage::ReplicationClient::GetTimestampI
|
||||
}
|
||||
|
||||
////// ReplicaStream //////
|
||||
InMemoryStorage::ReplicationClient::ReplicaStream::ReplicaStream(ReplicationClient *self,
|
||||
const uint64_t previous_commit_timestamp,
|
||||
const uint64_t current_seq_num)
|
||||
Storage::ReplicationClient::ReplicaStream::ReplicaStream(ReplicationClient *self,
|
||||
const uint64_t previous_commit_timestamp,
|
||||
const uint64_t current_seq_num)
|
||||
: self_(self),
|
||||
stream_(self_->rpc_client_->Stream<replication::AppendDeltasRpc>(previous_commit_timestamp, current_seq_num)) {
|
||||
replication::Encoder encoder{stream_.GetBuilder()};
|
||||
encoder.WriteString(self_->storage_->epoch_id_);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex,
|
||||
uint64_t final_commit_timestamp) {
|
||||
void Storage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Vertex &vertex,
|
||||
uint64_t final_commit_timestamp) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, self_->storage_->config_.items, delta, vertex,
|
||||
final_commit_timestamp);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge,
|
||||
uint64_t final_commit_timestamp) {
|
||||
void Storage::ReplicationClient::ReplicaStream::AppendDelta(const Delta &delta, const Edge &edge,
|
||||
uint64_t final_commit_timestamp) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
EncodeDelta(&encoder, &self_->storage_->name_id_mapper_, delta, edge, final_commit_timestamp);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) {
|
||||
void Storage::ReplicationClient::ReplicaStream::AppendTransactionEnd(uint64_t final_commit_timestamp) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
EncodeTransactionEnd(&encoder, final_commit_timestamp);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation,
|
||||
LabelId label,
|
||||
const std::set<PropertyId> &properties,
|
||||
uint64_t timestamp) {
|
||||
void Storage::ReplicationClient::ReplicaStream::AppendOperation(durability::StorageGlobalOperation operation,
|
||||
LabelId label, const std::set<PropertyId> &properties,
|
||||
uint64_t timestamp) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
EncodeOperation(&encoder, &self_->storage_->name_id_mapper_, operation, label, properties, timestamp);
|
||||
}
|
||||
|
||||
replication::AppendDeltasRes InMemoryStorage::ReplicationClient::ReplicaStream::Finalize() {
|
||||
return stream_.AwaitResponse();
|
||||
}
|
||||
replication::AppendDeltasRes Storage::ReplicationClient::ReplicaStream::Finalize() { return stream_.AwaitResponse(); }
|
||||
|
||||
////// CurrentWalHandler //////
|
||||
InMemoryStorage::ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self)
|
||||
Storage::ReplicationClient::CurrentWalHandler::CurrentWalHandler(ReplicationClient *self)
|
||||
: self_(self), stream_(self_->rpc_client_->Stream<replication::CurrentWalRpc>()) {}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::CurrentWalHandler::AppendFilename(const std::string &filename) {
|
||||
void Storage::ReplicationClient::CurrentWalHandler::AppendFilename(const std::string &filename) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
encoder.WriteString(filename);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) {
|
||||
void Storage::ReplicationClient::CurrentWalHandler::AppendSize(const size_t size) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
encoder.WriteUint(size);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::CurrentWalHandler::AppendFileData(utils::InputFile *file) {
|
||||
void Storage::ReplicationClient::CurrentWalHandler::AppendFileData(utils::InputFile *file) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
encoder.WriteFileData(file);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationClient::CurrentWalHandler::AppendBufferData(const uint8_t *buffer,
|
||||
const size_t buffer_size) {
|
||||
void Storage::ReplicationClient::CurrentWalHandler::AppendBufferData(const uint8_t *buffer, const size_t buffer_size) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
encoder.WriteBuffer(buffer, buffer_size);
|
||||
}
|
||||
|
||||
replication::CurrentWalRes InMemoryStorage::ReplicationClient::CurrentWalHandler::Finalize() {
|
||||
return stream_.AwaitResponse();
|
||||
}
|
||||
replication::CurrentWalRes Storage::ReplicationClient::CurrentWalHandler::Finalize() { return stream_.AwaitResponse(); }
|
||||
} // namespace memgraph::storage
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -21,7 +21,6 @@
|
||||
#include "storage/v2/delta.hpp"
|
||||
#include "storage/v2/durability/wal.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/name_id_mapper.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
@@ -29,6 +28,7 @@
|
||||
#include "storage/v2/replication/enums.hpp"
|
||||
#include "storage/v2/replication/rpc.hpp"
|
||||
#include "storage/v2/replication/serialization.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
@@ -37,9 +37,9 @@
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
class InMemoryStorage::ReplicationClient {
|
||||
class Storage::ReplicationClient {
|
||||
public:
|
||||
ReplicationClient(std::string name, InMemoryStorage *storage, const io::network::Endpoint &endpoint,
|
||||
ReplicationClient(std::string name, Storage *storage, const io::network::Endpoint &endpoint,
|
||||
replication::ReplicationMode mode, const replication::ReplicationClientConfig &config = {});
|
||||
|
||||
// Handler used for transfering the current transaction.
|
||||
@@ -123,7 +123,7 @@ class InMemoryStorage::ReplicationClient {
|
||||
|
||||
const auto &Endpoint() const { return rpc_client_->Endpoint(); }
|
||||
|
||||
InMemoryStorage::TimestampInfo GetTimestampInfo();
|
||||
Storage::TimestampInfo GetTimestampInfo();
|
||||
|
||||
private:
|
||||
[[nodiscard]] bool FinalizeTransactionReplicationInternal();
|
||||
@@ -150,7 +150,7 @@ class InMemoryStorage::ReplicationClient {
|
||||
void HandleRpcFailure();
|
||||
|
||||
std::string name_;
|
||||
InMemoryStorage *storage_;
|
||||
Storage *storage_;
|
||||
std::optional<communication::ClientContext> rpc_context_;
|
||||
std::optional<rpc::Client> rpc_client_;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,7 +19,6 @@
|
||||
#include "storage/v2/durability/snapshot.hpp"
|
||||
#include "storage/v2/durability/version.hpp"
|
||||
#include "storage/v2/durability/wal.hpp"
|
||||
#include "storage/v2/inmemory/edge_accessor.hpp"
|
||||
#include "storage/v2/replication/config.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
@@ -40,8 +39,8 @@ std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder
|
||||
};
|
||||
} // namespace
|
||||
|
||||
InMemoryStorage::ReplicationServer::ReplicationServer(InMemoryStorage *storage, io::network::Endpoint endpoint,
|
||||
const replication::ReplicationServerConfig &config)
|
||||
Storage::ReplicationServer::ReplicationServer(Storage *storage, io::network::Endpoint endpoint,
|
||||
const replication::ReplicationServerConfig &config)
|
||||
: storage_(storage) {
|
||||
// Create RPC server.
|
||||
if (config.ssl) {
|
||||
@@ -88,21 +87,21 @@ InMemoryStorage::ReplicationServer::ReplicationServer(InMemoryStorage *storage,
|
||||
rpc_server_->Start();
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::HeartbeatReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
replication::HeartbeatRes res{true, storage_->last_commit_timestamp_.load(), storage_->epoch_id_};
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::FrequentHeartbeatReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
replication::FrequentHeartbeatRes res{true};
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::AppendDeltasReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -149,7 +148,7 @@ void InMemoryStorage::ReplicationServer::AppendDeltasHandler(slk::Reader *req_re
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::SnapshotReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -213,7 +212,7 @@ void InMemoryStorage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::WalFilesReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -232,7 +231,7 @@ void InMemoryStorage::ReplicationServer::WalFilesHandler(slk::Reader *req_reader
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::CurrentWalReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -246,7 +245,7 @@ void InMemoryStorage::ReplicationServer::CurrentWalHandler(slk::Reader *req_read
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::LoadWal(replication::Decoder *decoder) {
|
||||
void Storage::ReplicationServer::LoadWal(replication::Decoder *decoder) {
|
||||
const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory;
|
||||
utils::EnsureDir(temp_wal_directory);
|
||||
auto maybe_wal_path = decoder->ReadFile(temp_wal_directory);
|
||||
@@ -289,7 +288,7 @@ void InMemoryStorage::ReplicationServer::LoadWal(replication::Decoder *decoder)
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::ReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void Storage::ReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::TimestampReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -297,26 +296,24 @@ void InMemoryStorage::ReplicationServer::TimestampHandler(slk::Reader *req_reade
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
InMemoryStorage::ReplicationServer::~ReplicationServer() {
|
||||
Storage::ReplicationServer::~ReplicationServer() {
|
||||
if (rpc_server_) {
|
||||
rpc_server_->Shutdown();
|
||||
rpc_server_->AwaitShutdown();
|
||||
}
|
||||
}
|
||||
uint64_t InMemoryStorage::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *decoder) {
|
||||
uint64_t Storage::ReplicationServer::ReadAndApplyDelta(durability::BaseDecoder *decoder) {
|
||||
auto edge_acc = storage_->edges_.access();
|
||||
auto vertex_acc = storage_->vertices_.access();
|
||||
|
||||
std::optional<std::pair<uint64_t, storage::InMemoryStorage::InMemoryAccessor *>> commit_timestamp_and_accessor;
|
||||
std::optional<std::pair<uint64_t, storage::Storage::Accessor>> commit_timestamp_and_accessor;
|
||||
auto get_transaction = [this, &commit_timestamp_and_accessor](uint64_t commit_timestamp) {
|
||||
if (!commit_timestamp_and_accessor) {
|
||||
commit_timestamp_and_accessor.emplace(commit_timestamp,
|
||||
dynamic_cast<storage::InMemoryStorage::InMemoryAccessor *>(
|
||||
storage_->Access(std::optional<IsolationLevel>{}).get()));
|
||||
commit_timestamp_and_accessor.emplace(commit_timestamp, storage_->Access());
|
||||
} else if (commit_timestamp_and_accessor->first != commit_timestamp) {
|
||||
throw utils::BasicException("Received more than one transaction!");
|
||||
}
|
||||
return commit_timestamp_and_accessor->second;
|
||||
return &commit_timestamp_and_accessor->second;
|
||||
};
|
||||
|
||||
uint64_t applied_deltas = 0;
|
||||
@@ -411,7 +408,7 @@ uint64_t InMemoryStorage::ReplicationServer::ReadAndApplyDelta(durability::BaseD
|
||||
if (edges.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
if (edges->size() != 1) throw utils::BasicException("Invalid transaction!");
|
||||
auto &edge = (*edges)[0];
|
||||
auto ret = transaction->DeleteEdge(edge.get());
|
||||
auto ret = transaction->DeleteEdge(&edge);
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
@@ -469,14 +466,14 @@ uint64_t InMemoryStorage::ReplicationServer::ReadAndApplyDelta(durability::BaseD
|
||||
// type and invalid from/to pointers because we don't know them
|
||||
// here, but that isn't an issue because we won't use that part of
|
||||
// the API here.
|
||||
auto ea = InMemoryEdgeAccessor{edge_ref,
|
||||
EdgeTypeId::FromUint(0UL),
|
||||
nullptr,
|
||||
nullptr,
|
||||
&transaction->transaction_,
|
||||
&storage_->indices_,
|
||||
&storage_->constraints_,
|
||||
storage_->config_.items};
|
||||
auto ea = EdgeAccessor{edge_ref,
|
||||
EdgeTypeId::FromUint(0UL),
|
||||
nullptr,
|
||||
nullptr,
|
||||
&transaction->transaction_,
|
||||
&storage_->indices_,
|
||||
&storage_->constraints_,
|
||||
storage_->config_.items};
|
||||
|
||||
auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
|
||||
delta.vertex_edge_set_property.value);
|
||||
@@ -488,7 +485,7 @@ uint64_t InMemoryStorage::ReplicationServer::ReadAndApplyDelta(durability::BaseD
|
||||
spdlog::trace(" Transaction end");
|
||||
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
|
||||
throw utils::BasicException("Invalid data!");
|
||||
auto ret = commit_timestamp_and_accessor->second->Commit(commit_timestamp_and_accessor->first);
|
||||
auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first);
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
commit_timestamp_and_accessor = std::nullopt;
|
||||
break;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,13 +11,13 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
class InMemoryStorage::ReplicationServer {
|
||||
class Storage::ReplicationServer {
|
||||
public:
|
||||
explicit ReplicationServer(InMemoryStorage *storage, io::network::Endpoint endpoint,
|
||||
explicit ReplicationServer(Storage *storage, io::network::Endpoint endpoint,
|
||||
const replication::ReplicationServerConfig &config);
|
||||
ReplicationServer(const ReplicationServer &) = delete;
|
||||
ReplicationServer(ReplicationServer &&) = delete;
|
||||
@@ -42,7 +42,7 @@ class InMemoryStorage::ReplicationServer {
|
||||
std::optional<communication::ServerContext> rpc_server_context_;
|
||||
std::optional<rpc::Server> rpc_server_;
|
||||
|
||||
InMemoryStorage *storage_;
|
||||
Storage *storage_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/vertex.hpp"
|
||||
|
||||
#include "storage/v2/config.hpp"
|
||||
@@ -78,6 +81,10 @@ class VertexAccessor {
|
||||
/// @throw std::bad_alloc
|
||||
virtual Result<std::map<PropertyId, PropertyValue>> Properties(View view) const = 0;
|
||||
|
||||
std::string PropertyStore() const;
|
||||
|
||||
void SetPropertyStore(std::string_view buffer) const;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error if the resulting vector exceeds
|
||||
/// std::vector::max_size().
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -21,6 +21,7 @@
|
||||
class ExpansionBenchFixture : public benchmark::Fixture {
|
||||
protected:
|
||||
std::optional<memgraph::storage::Storage> db;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db_;
|
||||
std::optional<memgraph::query::InterpreterContext> interpreter_context;
|
||||
std::optional<memgraph::query::Interpreter> interpreter;
|
||||
std::filesystem::path data_directory{std::filesystem::temp_directory_path() / "expansion-benchmark"};
|
||||
@@ -47,7 +48,7 @@ class ExpansionBenchFixture : public benchmark::Fixture {
|
||||
|
||||
MG_ASSERT(!db->CreateIndex(label).HasError());
|
||||
|
||||
interpreter_context.emplace(&*db, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
interpreter_context.emplace(&*db, &disk_db_, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
interpreter.emplace(&*interpreter_context);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -27,11 +27,13 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
|
||||
memgraph::storage::Storage db;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db;
|
||||
auto data_directory = std::filesystem::temp_directory_path() / "single_query_test";
|
||||
memgraph::utils::OnScopeExit([&data_directory] { std::filesystem::remove_all(data_directory); });
|
||||
|
||||
memgraph::license::global_license_checker.EnableTesting();
|
||||
memgraph::query::InterpreterContext interpreter_context{&db, memgraph::query::InterpreterConfig{}, data_directory};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db, &disk_db, memgraph::query::InterpreterConfig{},
|
||||
data_directory};
|
||||
memgraph::query::Interpreter interpreter{&interpreter_context};
|
||||
|
||||
ResultStreamFaker stream(&db);
|
||||
|
||||
@@ -303,6 +303,9 @@ target_link_libraries(${test_prefix}storage_v2_decoder_encoder mg-storage-v2)
|
||||
add_unit_test(storage_v2_durability.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v2_durability mg-storage-v2)
|
||||
|
||||
add_unit_test(storage_rocks.cpp)
|
||||
target_link_libraries(${test_prefix}storage_rocks mg-storage-v2)
|
||||
|
||||
add_unit_test(storage_v2_edge.cpp)
|
||||
target_link_libraries(${test_prefix}storage_v2_edge mg-storage-v2)
|
||||
|
||||
|
||||
@@ -49,8 +49,9 @@ auto ToEdgeList(const memgraph::communication::bolt::Value &v) {
|
||||
class InterpreterTest : public ::testing::Test {
|
||||
public:
|
||||
memgraph::storage::Storage db_;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db;
|
||||
std::filesystem::path data_directory{std::filesystem::temp_directory_path() / "MG_tests_unit_interpreter"};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, {}, data_directory};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, &disk_db, {}, data_directory};
|
||||
|
||||
InterpreterFaker default_interpreter{&interpreter_context};
|
||||
|
||||
@@ -1066,7 +1067,7 @@ TEST_F(InterpreterTest, AllowLoadCsvConfig) {
|
||||
"row"};
|
||||
|
||||
memgraph::query::InterpreterContext csv_interpreter_context{
|
||||
&db_, {.query = {.allow_load_csv = allow_load_csv}}, directory_manager.Path()};
|
||||
&db_, &disk_db, {.query = {.allow_load_csv = allow_load_csv}}, directory_manager.Path()};
|
||||
InterpreterFaker interpreter_faker{&csv_interpreter_context};
|
||||
for (const auto &query : queries) {
|
||||
if (allow_load_csv) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -202,8 +202,9 @@ DatabaseState GetState(memgraph::storage::Storage *db) {
|
||||
}
|
||||
|
||||
auto Execute(memgraph::storage::Storage *db, const std::string &query) {
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db;
|
||||
auto data_directory = std::filesystem::temp_directory_path() / "MG_tests_unit_query_dump";
|
||||
memgraph::query::InterpreterContext context(db, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
memgraph::query::InterpreterContext context(db, &disk_db, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
memgraph::query::Interpreter interpreter(&context);
|
||||
ResultStreamFaker stream(db);
|
||||
|
||||
@@ -757,8 +758,10 @@ TEST(DumpTest, ExecuteDumpDatabase) {
|
||||
|
||||
class StatefulInterpreter {
|
||||
public:
|
||||
explicit StatefulInterpreter(memgraph::storage::Storage *db)
|
||||
: db_(db), context_(db_, memgraph::query::InterpreterConfig{}, data_directory_), interpreter_(&context_) {}
|
||||
explicit StatefulInterpreter(memgraph::storage::Storage *db, memgraph::storage::rocks::RocksDBStorage *disk_db)
|
||||
: db_(db),
|
||||
context_(db_, disk_db, memgraph::query::InterpreterConfig{}, data_directory_),
|
||||
interpreter_(&context_) {}
|
||||
|
||||
auto Execute(const std::string &query) {
|
||||
ResultStreamFaker stream(db_);
|
||||
@@ -785,7 +788,8 @@ const std::filesystem::path StatefulInterpreter::data_directory_{std::filesystem
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST(DumpTest, ExecuteDumpDatabaseInMulticommandTransaction) {
|
||||
memgraph::storage::Storage db;
|
||||
StatefulInterpreter interpreter(&db);
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db;
|
||||
StatefulInterpreter interpreter(&db, &disk_db);
|
||||
|
||||
// Begin the transaction before the vertex is created.
|
||||
interpreter.Execute("BEGIN");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -28,6 +28,7 @@ DECLARE_bool(query_cost_planner);
|
||||
class QueryExecution : public testing::Test {
|
||||
protected:
|
||||
std::optional<memgraph::storage::Storage> db_;
|
||||
std::optional<memgraph::storage::rocks::RocksDBStorage> disk_db_;
|
||||
std::optional<memgraph::query::InterpreterContext> interpreter_context_;
|
||||
std::optional<memgraph::query::Interpreter> interpreter_;
|
||||
|
||||
@@ -35,7 +36,8 @@ class QueryExecution : public testing::Test {
|
||||
|
||||
void SetUp() {
|
||||
db_.emplace();
|
||||
interpreter_context_.emplace(&*db_, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
disk_db_.emplace();
|
||||
interpreter_context_.emplace(&*db_, &*disk_db_, memgraph::query::InterpreterConfig{}, data_directory);
|
||||
interpreter_.emplace(&*interpreter_context_);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,6 +22,7 @@
|
||||
#include "query/config.hpp"
|
||||
#include "query/interpreter.hpp"
|
||||
#include "query/stream/streams.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "test_utils.hpp"
|
||||
|
||||
@@ -55,6 +56,7 @@ class StreamsTest : public ::testing::Test {
|
||||
|
||||
protected:
|
||||
memgraph::storage::Storage db_;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db_;
|
||||
std::filesystem::path data_directory_{GetCleanDataDirectory()};
|
||||
KafkaClusterMock mock_cluster_{std::vector<std::string>{kTopicName}};
|
||||
// Though there is a Streams object in interpreter context, it makes more sense to use a separate object to test,
|
||||
@@ -62,7 +64,8 @@ class StreamsTest : public ::testing::Test {
|
||||
// Streams constructor.
|
||||
// InterpreterContext::auth_checker_ is used in the Streams object, but only in the message processing part. Because
|
||||
// these tests don't send any messages, the auth_checker_ pointer can be left as nullptr.
|
||||
memgraph::query::InterpreterContext interpreter_context_{&db_, memgraph::query::InterpreterConfig{}, data_directory_};
|
||||
memgraph::query::InterpreterContext interpreter_context_{&db_, &disk_db_, memgraph::query::InterpreterConfig{},
|
||||
data_directory_};
|
||||
std::filesystem::path streams_data_directory_{data_directory_ / "separate-dir-for-test"};
|
||||
std::optional<Streams> streams_;
|
||||
|
||||
|
||||
323
tests/unit/storage_rocks.cpp
Normal file
323
tests/unit/storage_rocks.cpp
Normal file
@@ -0,0 +1,323 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <exception>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "query/common.hpp"
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "storage/v2/delta.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/isolation_level.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
class RocksDBStorageTest : public ::testing::TestWithParam<bool> {
|
||||
public:
|
||||
~RocksDBStorageTest() { db.Clear(); }
|
||||
|
||||
protected:
|
||||
memgraph::storage::rocks::RocksDBStorage db;
|
||||
memgraph::storage::Storage storage;
|
||||
};
|
||||
|
||||
TEST_F(RocksDBStorageTest, SerializeVertexGID) {
|
||||
// empty vertices, only gid is serialized
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
std::unordered_set<uint64_t> gids;
|
||||
for (uint64_t i = 0; i < 5; ++i) {
|
||||
gids.insert(i);
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.SetGid(memgraph::storage::Gid::FromUint(i));
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto loaded_vertices = db.Vertices(dba);
|
||||
ASSERT_EQ(loaded_vertices.size(), 5);
|
||||
for (const auto &vertex_acc : loaded_vertices) {
|
||||
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) {
|
||||
// serialize vertex's gid with its single label
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
// save vertices on disk
|
||||
std::unordered_set<uint64_t> gids;
|
||||
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
|
||||
dba.NameToLabel("Ball")};
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
gids.insert(i);
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.SetGid(memgraph::storage::Gid::FromUint(i));
|
||||
impl.AddLabel(label_ids[i % 3]);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto loaded_vertices = db.Vertices(dba);
|
||||
ASSERT_EQ(loaded_vertices.size(), 5);
|
||||
for (const auto &vertex_acc : loaded_vertices) {
|
||||
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
|
||||
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
|
||||
ASSERT_EQ(labels->size(), 1);
|
||||
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
|
||||
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, SerializeVertexGIDMutlipleLabels) {
|
||||
// serialize vertex's gid with multiple labels it contains
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
// save vertices on disk
|
||||
std::unordered_set<uint64_t> gids;
|
||||
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
|
||||
dba.NameToLabel("Ball")};
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
gids.insert(i);
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.SetGid(memgraph::storage::Gid::FromUint(i));
|
||||
impl.AddLabel(label_ids[i % 3]);
|
||||
impl.AddLabel(label_ids[(i + 1) % 3]);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto loaded_vertices = db.Vertices(dba);
|
||||
ASSERT_EQ(loaded_vertices.size(), 5);
|
||||
for (const auto &vertex_acc : loaded_vertices) {
|
||||
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
|
||||
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
|
||||
ASSERT_EQ(labels->size(), 2);
|
||||
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
|
||||
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, GetVerticesByLabel) {
|
||||
// search vertices by label
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
// prepare labels
|
||||
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Player"),
|
||||
dba.NameToLabel("Ball")};
|
||||
// insert vertices
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.AddLabel(label_ids[i % 3]);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto player_vertices = db.Vertices(dba, dba.NameToLabel("Player"));
|
||||
auto ball_vertices = db.Vertices(dba, dba.NameToLabel("Ball"));
|
||||
ASSERT_EQ(player_vertices.size(), 4);
|
||||
ASSERT_EQ(ball_vertices.size(), 1);
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, GetVerticesByProperty) {
|
||||
// search vertices by property value
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
// prepare ssd properties
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> ssd_properties_1;
|
||||
ssd_properties_1.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(225.84));
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> ssd_properties_2;
|
||||
ssd_properties_2.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(226.84));
|
||||
// prepare hdd properties
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> hdd_properties_1;
|
||||
hdd_properties_1.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(125.84));
|
||||
std::vector properties{ssd_properties_1, ssd_properties_2, hdd_properties_1, hdd_properties_1};
|
||||
// insert vertices
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
auto impl = dba.InsertVertex();
|
||||
memgraph::query::MultiPropsInitChecked(&impl, properties[i]);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto ssd_vertices_1 = db.Vertices(dba, dba.NameToProperty("price"), memgraph::storage::PropertyValue(225.84));
|
||||
auto hdd_vertices = db.Vertices(dba, dba.NameToProperty("price"), memgraph::storage::PropertyValue(125.84));
|
||||
auto hdd_vertices_non_existing =
|
||||
db.Vertices(dba, dba.NameToProperty("price"), memgraph::storage::PropertyValue(125.81));
|
||||
ASSERT_EQ(ssd_vertices_1.size(), 1);
|
||||
ASSERT_EQ(hdd_vertices.size(), 2);
|
||||
ASSERT_EQ(hdd_vertices_non_existing.size(), 0);
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, DeleteVertex) {
|
||||
// auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTE);
|
||||
auto storage_dba = storage.Access();
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
|
||||
// samo 1 property stane
|
||||
properties.emplace(dba.NameToProperty("sum"), memgraph::storage::PropertyValue("2TB"));
|
||||
properties.emplace(dba.NameToProperty("same_type"), memgraph::storage::PropertyValue(true));
|
||||
// properties.emplace(dba.NameToProperty("cluster_price"), memgraph::storage::PropertyValue(2000.42));
|
||||
// create vertex
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.AddLabel(dba.NameToLabel("Player"));
|
||||
memgraph::query::MultiPropsInitChecked(&impl, properties);
|
||||
db.StoreVertex(impl);
|
||||
// find vertex should work now
|
||||
ASSERT_TRUE(db.FindVertex(std::to_string(impl.Gid().AsUint()), dba).has_value());
|
||||
db.FindVertex(std::to_string(impl.Gid().AsUint()), dba);
|
||||
// RocksDB doesn't physically delete entry so deletion will pass two times
|
||||
ASSERT_TRUE(db.DeleteVertex(impl).has_value());
|
||||
ASSERT_TRUE(db.DeleteVertex(impl).has_value());
|
||||
// second time you shouldn't be able to find the vertex
|
||||
ASSERT_FALSE(db.FindVertex(std::to_string(impl.Gid().AsUint()), dba).has_value());
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, SerializeVertexGIDProperties) {
|
||||
// serializes vertex's gid, multiple labels and properties
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
// prepare labels
|
||||
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Person"),
|
||||
dba.NameToLabel("Ball")};
|
||||
// prepare properties
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties;
|
||||
properties.emplace(dba.NameToProperty("name"), memgraph::storage::PropertyValue("disk"));
|
||||
properties.emplace(dba.NameToProperty("memory"), memgraph::storage::PropertyValue("1TB"));
|
||||
properties.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(1000.21));
|
||||
properties.emplace(dba.NameToProperty("price2"), memgraph::storage::PropertyValue(1000.212));
|
||||
// gids
|
||||
std::unordered_set<uint64_t> gids;
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
gids.insert(i);
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.SetGid(memgraph::storage::Gid::FromUint(i));
|
||||
impl.AddLabel(label_ids[i % 3]);
|
||||
impl.AddLabel(label_ids[(i + 1) % 3]);
|
||||
memgraph::query::MultiPropsInitChecked(&impl, properties);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// load vertices from disk
|
||||
auto loaded_vertices = db.Vertices(dba);
|
||||
ASSERT_EQ(loaded_vertices.size(), 5);
|
||||
for (const auto &vertex_acc : loaded_vertices) {
|
||||
ASSERT_TRUE(gids.contains(vertex_acc.Gid().AsUint()));
|
||||
// labels
|
||||
auto labels = vertex_acc.Labels(memgraph::storage::View::OLD);
|
||||
ASSERT_EQ(labels->size(), 2);
|
||||
ASSERT_TRUE(std::all_of(labels->begin(), labels->end(), [&label_ids](const auto &label_id) {
|
||||
return std::find(label_ids.begin(), label_ids.end(), label_id) != label_ids.end();
|
||||
}));
|
||||
// check properties
|
||||
auto props = vertex_acc.Properties(memgraph::storage::View::OLD);
|
||||
ASSERT_FALSE(props.HasError());
|
||||
auto prop_name = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("name"));
|
||||
auto prop_memory = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("memory"));
|
||||
auto prop_price = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("price"));
|
||||
auto prop_unexisting = vertex_acc.GetProperty(memgraph::storage::View::OLD, dba.NameToProperty("random"));
|
||||
ASSERT_TRUE(prop_name->IsString());
|
||||
ASSERT_EQ(prop_name->ValueString(), "disk");
|
||||
ASSERT_TRUE(prop_memory->IsString());
|
||||
ASSERT_EQ(prop_memory->ValueString(), "1TB");
|
||||
ASSERT_TRUE(prop_price->IsDouble());
|
||||
ASSERT_DOUBLE_EQ(prop_price->ValueDouble(), 1000.21);
|
||||
ASSERT_TRUE(prop_unexisting->IsNull());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(RocksDBStorageTest, SerializeEdge) {
|
||||
// create two vertices and edge between them
|
||||
// search by one of the vertices, return edge
|
||||
// check deserialization for both vertices and edge
|
||||
auto storage_dba = storage.Access(memgraph::storage::IsolationLevel::READ_UNCOMMITTED);
|
||||
memgraph::query::DbAccessor dba(&storage_dba);
|
||||
std::vector<memgraph::storage::LabelId> label_ids{dba.NameToLabel("Player"), dba.NameToLabel("Referee")};
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties_1;
|
||||
properties_1.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(221.84));
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> properties_2;
|
||||
properties_2.emplace(dba.NameToProperty("price"), memgraph::storage::PropertyValue(222.84));
|
||||
std::vector properties{properties_1, properties_2};
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
auto impl = dba.InsertVertex();
|
||||
impl.AddLabel(label_ids[i]);
|
||||
memgraph::query::MultiPropsInitChecked(&impl, properties[i]);
|
||||
db.StoreVertex(impl);
|
||||
}
|
||||
// prepare edge properties
|
||||
std::map<memgraph::storage::PropertyId, memgraph::storage::PropertyValue> edge_properties;
|
||||
edge_properties.emplace(dba.NameToProperty("sum"), memgraph::storage::PropertyValue("2TB"));
|
||||
edge_properties.emplace(dba.NameToProperty("same_type"), memgraph::storage::PropertyValue(true));
|
||||
edge_properties.emplace(dba.NameToProperty("cluster_price"), memgraph::storage::PropertyValue(2000.42));
|
||||
// Before inserting edge, find two vertices
|
||||
// find source vertex by the property
|
||||
auto src_vertices = db.Vertices(dba, dba.NameToProperty("price"), memgraph::storage::PropertyValue(221.84));
|
||||
ASSERT_EQ(src_vertices.size(), 1);
|
||||
auto src_vertex = src_vertices[0];
|
||||
// find destination vertex by the property
|
||||
auto dest_vertices = db.Vertices(dba, dba.NameToProperty("price"), memgraph::storage::PropertyValue(222.84));
|
||||
ASSERT_EQ(dest_vertices.size(), 1);
|
||||
auto dest_vertex = dest_vertices[0];
|
||||
// insert the edge
|
||||
uint64_t edge_gid = 2;
|
||||
auto edge_type_id = "CONNECTION";
|
||||
auto impl_edge = dba.InsertEdge(&src_vertex, &dest_vertex, dba.NameToEdgeType(edge_type_id));
|
||||
ASSERT_FALSE(impl_edge.HasError());
|
||||
(*impl_edge).SetGid(memgraph::storage::Gid::FromUint(edge_gid));
|
||||
memgraph::query::MultiPropsInitChecked(&*impl_edge, edge_properties);
|
||||
db.StoreEdge(*impl_edge);
|
||||
// Test out edges of the source vertex
|
||||
auto src_out_edges = db.OutEdges(src_vertex, dba);
|
||||
ASSERT_EQ(src_out_edges.size(), 1);
|
||||
auto src_out_edge = src_out_edges[0];
|
||||
// test from edge accessor
|
||||
auto from_out_edge_acc = src_out_edge.From();
|
||||
ASSERT_EQ(from_out_edge_acc.Gid(), src_vertex.Gid());
|
||||
ASSERT_EQ(from_out_edge_acc.Labels(memgraph::storage::View::OLD)->size(), 1);
|
||||
ASSERT_EQ(from_out_edge_acc.Labels(memgraph::storage::View::OLD)->at(0), label_ids[0]);
|
||||
ASSERT_EQ(*from_out_edge_acc.Properties(memgraph::storage::View::OLD), properties_1);
|
||||
// test to edge accessor
|
||||
auto to_out_edge_acc = src_out_edge.To();
|
||||
ASSERT_EQ(to_out_edge_acc.Gid(), dest_vertex.Gid());
|
||||
ASSERT_EQ(to_out_edge_acc.Labels(memgraph::storage::View::OLD)->size(), 1);
|
||||
ASSERT_EQ(to_out_edge_acc.Labels(memgraph::storage::View::OLD)->at(0), label_ids[1]);
|
||||
ASSERT_EQ(*to_out_edge_acc.Properties(memgraph::storage::View::OLD), properties_2);
|
||||
// test edge accessor
|
||||
ASSERT_EQ(src_out_edge.Gid().AsUint(), edge_gid);
|
||||
ASSERT_EQ(src_out_edge.EdgeType(), dba.NameToEdgeType(edge_type_id));
|
||||
ASSERT_EQ(*src_out_edge.Properties(memgraph::storage::View::OLD), edge_properties);
|
||||
// Test in edge of the destination vertex
|
||||
auto dest_in_edges = db.InEdges(dest_vertex, dba);
|
||||
ASSERT_EQ(dest_in_edges.size(), 1);
|
||||
auto dest_in_edge = dest_in_edges[0];
|
||||
// test from edge accessor
|
||||
auto from_in_edge_acc = dest_in_edge.From();
|
||||
ASSERT_EQ(from_in_edge_acc.Gid(), from_out_edge_acc.Gid());
|
||||
ASSERT_EQ(from_in_edge_acc.Labels(memgraph::storage::View::OLD)->size(), 1);
|
||||
ASSERT_EQ(from_in_edge_acc.Labels(memgraph::storage::View::OLD)->at(0),
|
||||
from_out_edge_acc.Labels(memgraph::storage::View::OLD)->at(0));
|
||||
ASSERT_EQ(*from_in_edge_acc.Properties(memgraph::storage::View::OLD),
|
||||
*from_out_edge_acc.Properties(memgraph::storage::View::OLD));
|
||||
// test in edge accessors
|
||||
auto to_in_edge_acc = dest_in_edge.To();
|
||||
ASSERT_EQ(to_in_edge_acc.Gid(), to_out_edge_acc.Gid());
|
||||
ASSERT_EQ(to_in_edge_acc.Labels(memgraph::storage::View::OLD)->size(), 1);
|
||||
ASSERT_EQ(to_in_edge_acc.Labels(memgraph::storage::View::OLD)->at(0),
|
||||
to_out_edge_acc.Labels(memgraph::storage::View::OLD)->at(0));
|
||||
ASSERT_EQ(*to_in_edge_acc.Properties(memgraph::storage::View::OLD),
|
||||
*to_out_edge_acc.Properties(memgraph::storage::View::OLD));
|
||||
// test edge accessors
|
||||
ASSERT_EQ(dest_in_edge.Gid(), src_out_edge.Gid());
|
||||
ASSERT_EQ(dest_in_edge.EdgeType(), src_out_edge.EdgeType());
|
||||
ASSERT_EQ(*dest_in_edge.Properties(memgraph::storage::View::OLD),
|
||||
*src_out_edge.Properties(memgraph::storage::View::OLD));
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,11 +11,10 @@
|
||||
|
||||
#include "storage_test_utils.hpp"
|
||||
|
||||
size_t CountVertices(memgraph::storage::InMemoryStorage::InMemoryAccessor &storage_accessor,
|
||||
memgraph::storage::View view) {
|
||||
size_t CountVertices(memgraph::storage::Storage::Accessor &storage_accessor, memgraph::storage::View view) {
|
||||
auto vertices = storage_accessor.Vertices(view);
|
||||
size_t count = 0U;
|
||||
for (auto it = vertices.begin(); it != vertices.end(); ++it, ++count)
|
||||
;
|
||||
return count;
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,8 +11,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
size_t CountVertices(memgraph::storage::InMemoryStorage::InMemoryAccessor &storage_accessor,
|
||||
memgraph::storage::View view);
|
||||
size_t CountVertices(memgraph::storage::Storage::Accessor &storage_accessor, memgraph::storage::View view);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -26,8 +26,9 @@ corresponding interpreter/.
|
||||
class TransactionQueueSimpleTest : public ::testing::Test {
|
||||
protected:
|
||||
memgraph::storage::Storage db_;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db_;
|
||||
std::filesystem::path data_directory{std::filesystem::temp_directory_path() / "MG_tests_unit_transaction_queue_intr"};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, {}, data_directory};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, &disk_db_, {}, data_directory};
|
||||
InterpreterFaker running_interpreter{&interpreter_context}, main_interpreter{&interpreter_context};
|
||||
};
|
||||
|
||||
|
||||
@@ -31,9 +31,10 @@ corresponding interpreter.
|
||||
class TransactionQueueMultipleTest : public ::testing::Test {
|
||||
protected:
|
||||
memgraph::storage::Storage db_;
|
||||
memgraph::storage::rocks::RocksDBStorage disk_db_;
|
||||
std::filesystem::path data_directory{std::filesystem::temp_directory_path() /
|
||||
"MG_tests_unit_transaction_queue_multiple_intr"};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, {}, data_directory};
|
||||
memgraph::query::InterpreterContext interpreter_context{&db_, &disk_db_, {}, data_directory};
|
||||
InterpreterFaker main_interpreter{&interpreter_context};
|
||||
std::vector<InterpreterFaker *> running_interpreters;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user