From de9280b334b8b9d21e5c44bdb55d9246440b50ae Mon Sep 17 00:00:00 2001 From: Andi Date: Mon, 16 Oct 2023 09:11:07 +0200 Subject: [PATCH] Refactor disk storage (#1347) --- src/query/plan/operator.cpp | 6 +- src/storage/v2/CMakeLists.txt | 1 + src/storage/v2/disk/durable_metadata.cpp | 173 +++++ src/storage/v2/disk/durable_metadata.hpp | 68 ++ src/storage/v2/disk/label_index.cpp | 3 +- src/storage/v2/disk/storage.cpp | 821 ++++++++------------- src/storage/v2/disk/storage.hpp | 227 +++--- src/storage/v2/disk/unique_constraints.cpp | 9 +- src/storage/v2/id_types.hpp | 2 + src/storage/v2/storage.cpp | 11 +- src/storage/v2/storage.hpp | 5 +- src/storage/v2/transaction.hpp | 2 +- src/storage/v2/vertex_accessor.cpp | 10 +- src/utils/rocksdb_serialization.hpp | 163 ++-- tests/e2e/mock_api/workloads.yaml | 7 - tests/e2e/triggers/workloads.yaml | 23 - tests/e2e/write_procedures/workloads.yaml | 15 - tests/unit/storage_rocks.cpp | 7 +- tests/unit/storage_v2_constraints.cpp | 3 +- 19 files changed, 754 insertions(+), 802 deletions(-) create mode 100644 src/storage/v2/disk/durable_metadata.cpp create mode 100644 src/storage/v2/disk/durable_metadata.hpp diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index 56024b660..227e523fb 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -1548,7 +1548,7 @@ class SingleSourceShortestPathCursor : public query::plan::Cursor { // populates the to_visit_next_ structure with expansions // from the given vertex. skips expansions that don't satisfy // the "where" condition. - auto expand_from_vertex = [this, &expand_pair, &context](const auto &vertex) { + auto expand_from_vertex = [this, &expand_pair](const auto &vertex) { if (self_.common_.direction != EdgeAtom::Direction::IN) { auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges; for (const auto &edge : out_edges) expand_pair(edge, edge.To()); @@ -1748,8 +1748,8 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor { // Populates the priority queue structure with expansions // from the given vertex. skips expansions that don't satisfy // the "where" condition. - auto expand_from_vertex = [this, &expand_pair, &context](const VertexAccessor &vertex, const TypedValue &weight, - int64_t depth) { + auto expand_from_vertex = [this, &expand_pair](const VertexAccessor &vertex, const TypedValue &weight, + int64_t depth) { if (self_.common_.direction != EdgeAtom::Direction::IN) { auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges; for (const auto &edge : out_edges) { diff --git a/src/storage/v2/CMakeLists.txt b/src/storage/v2/CMakeLists.txt index a6eaebe24..b18bbd1ed 100644 --- a/src/storage/v2/CMakeLists.txt +++ b/src/storage/v2/CMakeLists.txt @@ -25,6 +25,7 @@ add_library(mg-storage-v2 STATIC inmemory/label_index.cpp inmemory/label_property_index.cpp inmemory/unique_constraints.cpp + disk/durable_metadata.cpp disk/edge_import_mode_cache.cpp disk/storage.cpp disk/rocksdb_storage.cpp diff --git a/src/storage/v2/disk/durable_metadata.cpp b/src/storage/v2/disk/durable_metadata.cpp new file mode 100644 index 000000000..fe2c558ae --- /dev/null +++ b/src/storage/v2/disk/durable_metadata.cpp @@ -0,0 +1,173 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#include +#include +#include + +#include "kvstore/kvstore.hpp" +#include "storage/v2/config.hpp" +#include "storage/v2/disk/durable_metadata.hpp" +#include "utils/file.hpp" +#include "utils/rocksdb_serialization.hpp" +#include "utils/string.hpp" + +namespace { +constexpr const char *kLastTransactionStartTimeStamp = "last_transaction_start_timestamp"; +constexpr const char *kVertexCountDescr = "vertex_count"; +constexpr const char *kEdgeDountDescr = "edge_count"; +constexpr const char *kLabelIndexStr = "label_index"; +constexpr const char *kLabelPropertyIndexStr = "label_property_index"; +constexpr const char *kExistenceConstraintsStr = "existence_constraints"; +constexpr const char *kUniqueConstraintsStr = "unique_constraints"; +} // namespace + +namespace memgraph::storage { + +DurableMetadata::DurableMetadata(const Config &config) + : durability_kvstore_(kvstore::KVStore(config.disk.durability_directory)), config_(config) { + MG_ASSERT(utils::DirExists(config_.disk.durability_directory), + "Durability directory for saving disk metadata does not exist."); +} + +DurableMetadata::DurableMetadata(DurableMetadata &&other) noexcept + : durability_kvstore_(std::move(other.durability_kvstore_)), config_(std::move(other.config_)) {} + +void DurableMetadata::SaveBeforeClosingDB(uint64_t timestamp, uint64_t vertex_count, uint64_t edge_count) { + durability_kvstore_.Put(kLastTransactionStartTimeStamp, std::to_string(timestamp)); + durability_kvstore_.Put(kVertexCountDescr, std::to_string(vertex_count)); + durability_kvstore_.Put(kEdgeDountDescr, std::to_string(edge_count)); +} + +std::optional DurableMetadata::LoadTimestampIfExists() const { + return LoadPropertyIfExists(kLastTransactionStartTimeStamp); +} + +std::optional DurableMetadata::LoadVertexCountIfExists() const { + return LoadPropertyIfExists(kVertexCountDescr); +} + +std::optional DurableMetadata::LoadEdgeCountIfExists() const { return LoadPropertyIfExists(kEdgeDountDescr); } + +std::optional DurableMetadata::LoadPropertyIfExists(const std::string &property) const { + if (auto count = durability_kvstore_.Get(property); count.has_value()) { + auto last_count = count.value(); + uint64_t count_to_return{0U}; + if (std::from_chars(last_count.data(), last_count.data() + last_count.size(), count_to_return).ec == std::errc()) { + return count_to_return; + } + } + return {}; +} + +std::optional> DurableMetadata::LoadLabelIndexInfoIfExists() const { + return LoadInfoFromAuxiliaryStorages(kLabelIndexStr); +} + +std::optional> DurableMetadata::LoadLabelPropertyIndexInfoIfExists() const { + return LoadInfoFromAuxiliaryStorages(kLabelPropertyIndexStr); +} + +std::optional> DurableMetadata::LoadExistenceConstraintInfoIfExists() const { + return LoadInfoFromAuxiliaryStorages(kExistenceConstraintsStr); +} + +std::optional> DurableMetadata::LoadUniqueConstraintInfoIfExists() const { + return LoadInfoFromAuxiliaryStorages(kUniqueConstraintsStr); +} + +std::optional> DurableMetadata::LoadInfoFromAuxiliaryStorages( + const std::string &property) const { + if (auto maybe_props = durability_kvstore_.Get(property); maybe_props.has_value()) { + return utils::Split(maybe_props.value(), "|"); + } + return {}; +} + +bool DurableMetadata::PersistLabelIndexCreation(LabelId label) { + const auto serialized_label = label.ToString(); + if (auto label_index_store = durability_kvstore_.Get(kLabelIndexStr); label_index_store.has_value()) { + std::string &value = label_index_store.value(); + value += "|"; + value += serialized_label; + return durability_kvstore_.Put(kLabelIndexStr, value); + } + return durability_kvstore_.Put(kLabelIndexStr, serialized_label); +} + +bool DurableMetadata::PersistLabelIndexDeletion(LabelId label) { + const auto serialized_label = label.ToString(); + if (auto label_index_store = durability_kvstore_.Get(kLabelIndexStr); label_index_store.has_value()) { + const std::string &value = label_index_store.value(); + std::vector labels = utils::Split(value, "|"); + std::erase(labels, serialized_label); + if (labels.empty()) { + return durability_kvstore_.Delete(kLabelIndexStr); + } + return durability_kvstore_.Put(kLabelIndexStr, utils::Join(labels, "|")); + } + return true; +} + +bool DurableMetadata::PersistLabelPropertyIndexAndExistenceConstraintCreation(LabelId label, PropertyId property, + const std::string &key) { + const std::string label_property_pair = label.ToString() + "," + property.ToString(); + if (auto label_property_index_store = durability_kvstore_.Get(key); label_property_index_store.has_value()) { + std::string &value = label_property_index_store.value(); + value += "|"; + value += label_property_pair; + return durability_kvstore_.Put(key, value); + } + return durability_kvstore_.Put(key, label_property_pair); +} + +bool DurableMetadata::PersistLabelPropertyIndexAndExistenceConstraintDeletion(LabelId label, PropertyId property, + const std::string &key) { + const std::string label_property_pair = label.ToString() + "," + property.ToString(); + if (auto label_property_index_store = durability_kvstore_.Get(key); label_property_index_store.has_value()) { + const std::string &value = label_property_index_store.value(); + std::vector label_properties = utils::Split(value, "|"); + std::erase(label_properties, label_property_pair); + if (label_properties.empty()) { + return durability_kvstore_.Delete(key); + } + return durability_kvstore_.Put(key, utils::Join(label_properties, "|")); + } + return true; +} + +bool DurableMetadata::PersistUniqueConstraintCreation(LabelId label, const std::set &properties) { + const std::string entry = utils::GetKeyForUniqueConstraintsDurability(label, properties); + + if (auto unique_store = durability_kvstore_.Get(kUniqueConstraintsStr); unique_store.has_value()) { + std::string &value = unique_store.value(); + value += "|" + entry; + return durability_kvstore_.Put(kUniqueConstraintsStr, value); + } + return durability_kvstore_.Put(kUniqueConstraintsStr, entry); +} + +bool DurableMetadata::PersistUniqueConstraintDeletion(LabelId label, const std::set &properties) { + const std::string entry = utils::GetKeyForUniqueConstraintsDurability(label, properties); + + if (auto unique_store = durability_kvstore_.Get(kUniqueConstraintsStr); unique_store.has_value()) { + const std::string &value = unique_store.value(); + std::vector unique_constraints = utils::Split(value, "|"); + std::erase(unique_constraints, entry); + if (unique_constraints.empty()) { + return durability_kvstore_.Delete(kUniqueConstraintsStr); + } + return durability_kvstore_.Put(kUniqueConstraintsStr, utils::Join(unique_constraints, "|")); + } + return true; +} + +} // namespace memgraph::storage diff --git a/src/storage/v2/disk/durable_metadata.hpp b/src/storage/v2/disk/durable_metadata.hpp new file mode 100644 index 000000000..168cce469 --- /dev/null +++ b/src/storage/v2/disk/durable_metadata.hpp @@ -0,0 +1,68 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +#pragma once + +#include +#include +#include + +#include "kvstore/kvstore.hpp" +#include "storage/v2/config.hpp" +#include "storage/v2/id_types.hpp" + +namespace memgraph::storage { + +class DurableMetadata { + public: + explicit DurableMetadata(const Config &config); + + DurableMetadata(const DurableMetadata &) = delete; + DurableMetadata &operator=(const DurableMetadata &) = delete; + DurableMetadata &operator=(DurableMetadata &&) = delete; + + DurableMetadata(DurableMetadata &&other) noexcept; + + ~DurableMetadata() = default; + + std::optional LoadTimestampIfExists() const; + std::optional LoadVertexCountIfExists() const; + std::optional LoadEdgeCountIfExists() const; + std::optional> LoadLabelIndexInfoIfExists() const; + std::optional> LoadLabelPropertyIndexInfoIfExists() const; + std::optional> LoadExistenceConstraintInfoIfExists() const; + std::optional> LoadUniqueConstraintInfoIfExists() const; + + void SaveBeforeClosingDB(uint64_t timestamp, uint64_t vertex_count, uint64_t edge_count); + + bool PersistLabelIndexCreation(LabelId label); + + bool PersistLabelIndexDeletion(LabelId label); + + bool PersistLabelPropertyIndexAndExistenceConstraintCreation(LabelId label, PropertyId property, + const std::string &key); + + bool PersistLabelPropertyIndexAndExistenceConstraintDeletion(LabelId label, PropertyId property, + const std::string &key); + + bool PersistUniqueConstraintCreation(LabelId label, const std::set &properties); + + bool PersistUniqueConstraintDeletion(LabelId label, const std::set &properties); + + private: + std::optional LoadPropertyIfExists(const std::string &property) const; + std::optional> LoadInfoFromAuxiliaryStorages(const std::string &property) const; + + kvstore::KVStore durability_kvstore_; + Config config_; +}; + +} // namespace memgraph::storage diff --git a/src/storage/v2/disk/label_index.cpp b/src/storage/v2/disk/label_index.cpp index a43f86a5e..56986079b 100644 --- a/src/storage/v2/disk/label_index.cpp +++ b/src/storage/v2/disk/label_index.cpp @@ -190,9 +190,10 @@ bool DiskLabelIndex::DropIndex(LabelId label) { rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto it = std::unique_ptr(disk_transaction->GetIterator(ro)); + const std::string serialized_label = label.ToString(); for (it->SeekToFirst(); it->Valid(); it->Next()) { std::string key = it->key().ToString(); - if (key.starts_with(utils::SerializeIdType(label))) { + if (key.starts_with(serialized_label)) { disk_transaction->Delete(it->key().ToString()); } } diff --git a/src/storage/v2/disk/storage.cpp b/src/storage/v2/disk/storage.cpp index 77832f0f6..78ead9323 100644 --- a/src/storage/v2/disk/storage.cpp +++ b/src/storage/v2/disk/storage.cpp @@ -9,6 +9,8 @@ // by the Apache License, Version 2.0, included in the file // licenses/APL.txt. +#include "storage/v2/disk/storage.hpp" + #include #include #include @@ -35,7 +37,6 @@ #include "storage/v2/disk/label_index.hpp" #include "storage/v2/disk/label_property_index.hpp" #include "storage/v2/disk/rocksdb_storage.hpp" -#include "storage/v2/disk/storage.hpp" #include "storage/v2/disk/unique_constraints.hpp" #include "storage/v2/edge_accessor.hpp" #include "storage/v2/edge_import_mode.hpp" @@ -74,19 +75,14 @@ using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler; namespace { -constexpr const char *deserializeTimestamp = "0"; -constexpr const char *vertexHandle = "vertex"; -constexpr const char *edgeHandle = "edge"; -constexpr const char *defaultHandle = "default"; -constexpr const char *outEdgesHandle = "out_edges"; -constexpr const char *inEdgesHandle = "in_edges"; -constexpr const char *lastTransactionStartTimeStamp = "last_transaction_start_timestamp"; -constexpr const char *vertex_count_descr = "vertex_count"; -constexpr const char *edge_count_descr = "edge_count"; -constexpr const char *label_index_str = "label_index"; -constexpr const char *label_property_index_str = "label_property_index"; -constexpr const char *existence_constraints_str = "existence_constraints"; -constexpr const char *unique_constraints_str = "unique_constraints"; +constexpr const char *kDeserializeTimestamp = "0"; +constexpr const char *kVertexHandle = "vertex"; +constexpr const char *kEdgeHandle = "edge"; +constexpr const char *kDefaultHandle = "default"; +constexpr const char *kOutEdgesHandle = "out_edges"; +constexpr const char *kInEdgesHandle = "in_edges"; +constexpr const char *kLabelPropertyIndexStr = "label_property_index"; +constexpr const char *kExistenceConstraintsStr = "existence_constraints"; /// TODO: (andi) Maybe a better way of checking would be if the first delta is DELETE_DESERIALIZED /// then we now that the vertex has only been deserialized and nothing more has been done on it. @@ -201,83 +197,11 @@ bool IsPropertyValueWithinInterval(const PropertyValue &value, } // namespace -void DiskStorage::LoadTimestampIfExists() { - if (!utils::DirExists(config_.disk.durability_directory)) { - return; - } - if (auto last_timestamp_ = durability_kvstore_->Get(lastTransactionStartTimeStamp); last_timestamp_.has_value()) { - auto last_timestamp_value = last_timestamp_.value(); - std::from_chars(last_timestamp_value.data(), last_timestamp_value.data() + last_timestamp_value.size(), timestamp_); - } -} - -void DiskStorage::LoadVertexAndEdgeCountIfExists() { - if (!utils::DirExists(config_.disk.durability_directory)) { - return; - } - if (auto vertex_count = durability_kvstore_->Get(vertex_count_descr); vertex_count.has_value()) { - vertex_count_ = std::stoull(vertex_count.value()); - } - if (auto edge_count = durability_kvstore_->Get(edge_count_descr); edge_count.has_value()) { - edge_count_ = std::stoull(edge_count.value()); - } -} - -void DiskStorage::LoadIndexInfoIfExists() const { - if (utils::DirExists(config_.disk.durability_directory)) { - LoadLabelIndexInfoIfExists(); - LoadLabelPropertyIndexInfoIfExists(); - } -} - -void DiskStorage::LoadLabelIndexInfoIfExists() const { - if (auto label_index = durability_kvstore_->Get(label_index_str); label_index.has_value()) { - auto *disk_label_index = static_cast(indices_.label_index_.get()); - const std::vector labels{utils::Split(label_index.value(), "|")}; - disk_label_index->LoadIndexInfo(labels); - } -} - -void DiskStorage::LoadLabelPropertyIndexInfoIfExists() const { - if (auto label_property_index = durability_kvstore_->Get(label_property_index_str); - label_property_index.has_value()) { - auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); - const std::vector keys{utils::Split(label_property_index.value(), "|")}; - disk_label_property_index->LoadIndexInfo(keys); - } -} - -void DiskStorage::LoadConstraintsInfoIfExists() const { - if (utils::DirExists(config_.disk.durability_directory)) { - LoadExistenceConstraintInfoIfExists(); - LoadUniqueConstraintInfoIfExists(); - } -} - -void DiskStorage::LoadExistenceConstraintInfoIfExists() const { - if (auto existence_constraints = durability_kvstore_->Get(existence_constraints_str); - existence_constraints.has_value()) { - std::vector keys = utils::Split(existence_constraints.value(), "|"); - constraints_.existence_constraints_->LoadExistenceConstraints(keys); - } -} - -void DiskStorage::LoadUniqueConstraintInfoIfExists() const { - if (auto unique_constraints = durability_kvstore_->Get(unique_constraints_str); unique_constraints.has_value()) { - std::vector keys = utils::Split(unique_constraints.value(), "|"); - auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); - disk_unique_constraints->LoadUniqueConstraints(keys); - } -} - DiskStorage::DiskStorage(Config config) : Storage(config, StorageMode::ON_DISK_TRANSACTIONAL), kvstore_(std::make_unique()), - durability_kvstore_(std::make_unique(config.disk.durability_directory)) { - LoadTimestampIfExists(); - LoadVertexAndEdgeCountIfExists(); - LoadIndexInfoIfExists(); - LoadConstraintsInfoIfExists(); + durable_metadata_(config) { + LoadPersistingMetadataInfo(); kvstore_->options_.create_if_missing = true; kvstore_->options_.comparator = new ComparatorWithU64TsImpl(); kvstore_->options_.compression = rocksdb::kNoCompression; @@ -287,11 +211,11 @@ DiskStorage::DiskStorage(Config config) std::vector column_handles; std::vector column_families; if (utils::DirExists(config.disk.main_storage_directory)) { - column_families.emplace_back(vertexHandle, kvstore_->options_); - column_families.emplace_back(edgeHandle, kvstore_->options_); - column_families.emplace_back(defaultHandle, kvstore_->options_); - column_families.emplace_back(outEdgesHandle, kvstore_->options_); - column_families.emplace_back(inEdgesHandle, kvstore_->options_); + column_families.emplace_back(kVertexHandle, kvstore_->options_); + column_families.emplace_back(kEdgeHandle, kvstore_->options_); + column_families.emplace_back(kDefaultHandle, kvstore_->options_); + column_families.emplace_back(kOutEdgesHandle, kvstore_->options_); + column_families.emplace_back(kInEdgesHandle, kvstore_->options_); logging::AssertRocksDBStatus(rocksdb::TransactionDB::Open(kvstore_->options_, rocksdb::TransactionDBOptions(), config.disk.main_storage_directory, column_families, @@ -305,20 +229,19 @@ DiskStorage::DiskStorage(Config config) logging::AssertRocksDBStatus(rocksdb::TransactionDB::Open(kvstore_->options_, rocksdb::TransactionDBOptions(), config.disk.main_storage_directory, &kvstore_->db_)); logging::AssertRocksDBStatus( - kvstore_->db_->CreateColumnFamily(kvstore_->options_, vertexHandle, &kvstore_->vertex_chandle)); + kvstore_->db_->CreateColumnFamily(kvstore_->options_, kVertexHandle, &kvstore_->vertex_chandle)); logging::AssertRocksDBStatus( - kvstore_->db_->CreateColumnFamily(kvstore_->options_, edgeHandle, &kvstore_->edge_chandle)); + kvstore_->db_->CreateColumnFamily(kvstore_->options_, kEdgeHandle, &kvstore_->edge_chandle)); logging::AssertRocksDBStatus( - kvstore_->db_->CreateColumnFamily(kvstore_->options_, outEdgesHandle, &kvstore_->out_edges_chandle)); + kvstore_->db_->CreateColumnFamily(kvstore_->options_, kOutEdgesHandle, &kvstore_->out_edges_chandle)); logging::AssertRocksDBStatus( - kvstore_->db_->CreateColumnFamily(kvstore_->options_, inEdgesHandle, &kvstore_->in_edges_chandle)); + kvstore_->db_->CreateColumnFamily(kvstore_->options_, kInEdgesHandle, &kvstore_->in_edges_chandle)); } } DiskStorage::~DiskStorage() { - durability_kvstore_->Put(lastTransactionStartTimeStamp, std::to_string(timestamp_)); - durability_kvstore_->Put(vertex_count_descr, std::to_string(vertex_count_.load(std::memory_order_acquire))); - durability_kvstore_->Put(edge_count_descr, std::to_string(edge_count_.load(std::memory_order_acquire))); + durable_metadata_.SaveBeforeClosingDB(timestamp_, vertex_count_.load(std::memory_order_acquire), + edge_count_.load(std::memory_order_acquire)); logging::AssertRocksDBStatus(kvstore_->db_->DestroyColumnFamilyHandle(kvstore_->vertex_chandle)); logging::AssertRocksDBStatus(kvstore_->db_->DestroyColumnFamilyHandle(kvstore_->edge_chandle)); logging::AssertRocksDBStatus(kvstore_->db_->DestroyColumnFamilyHandle(kvstore_->out_edges_chandle)); @@ -341,23 +264,47 @@ DiskStorage::DiskAccessor::DiskAccessor(auto tag, DiskStorage *storage, Isolatio transaction_.disk_transaction_->SetReadTimestampForValidation(transaction_.start_timestamp); } -DiskStorage::DiskAccessor::DiskAccessor(DiskAccessor &&other) noexcept : Accessor(std::move(other)) { - other.is_transaction_active_ = false; - other.commit_timestamp_.reset(); -} +DiskStorage::DiskAccessor::DiskAccessor(DiskAccessor &&other) noexcept : Accessor(std::move(other)) {} DiskStorage::DiskAccessor::~DiskAccessor() { if (is_transaction_active_) { Abort(); } - FinalizeTransaction(); - transaction_.deltas.~Bond(); } -std::optional DiskStorage::DiskAccessor::LoadVertexToLabelIndexCache( - const std::string &key, const std::string &value, Delta *index_delta, +void DiskStorage::LoadPersistingMetadataInfo() { + if (auto last_timestamp = durable_metadata_.LoadTimestampIfExists(); last_timestamp.has_value()) { + timestamp_ = last_timestamp.value(); + } + if (auto vertex_count = durable_metadata_.LoadVertexCountIfExists(); vertex_count.has_value()) { + vertex_count_ = vertex_count.value(); + } + if (auto edge_count = durable_metadata_.LoadEdgeCountIfExists(); edge_count.has_value()) { + edge_count_ = edge_count.value(); + } + if (auto label_index = durable_metadata_.LoadLabelIndexInfoIfExists(); label_index.has_value()) { + auto *disk_label_index = static_cast(indices_.label_index_.get()); + disk_label_index->LoadIndexInfo(label_index.value()); + } + if (auto label_property_index = durable_metadata_.LoadLabelPropertyIndexInfoIfExists(); + label_property_index.has_value()) { + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); + disk_label_property_index->LoadIndexInfo(label_property_index.value()); + } + if (auto existence_constraints = durable_metadata_.LoadExistenceConstraintInfoIfExists(); + existence_constraints.has_value()) { + constraints_.existence_constraints_->LoadExistenceConstraints(existence_constraints.value()); + } + if (auto unique_constraints = durable_metadata_.LoadUniqueConstraintInfoIfExists(); unique_constraints.has_value()) { + auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); + disk_unique_constraints->LoadUniqueConstraints(unique_constraints.value()); + } +} + +std::optional DiskStorage::LoadVertexToLabelIndexCache( + Transaction *transaction, const std::string &key, const std::string &value, Delta *index_delta, utils::SkipList::Accessor index_accessor) { storage::Gid gid = Gid::FromString(utils::ExtractGidFromLabelIndexStorage(key)); if (ObjectExistsInCache(index_accessor, gid)) { @@ -365,14 +312,13 @@ std::optional DiskStorage::DiskAccessor::LoadVertexToLa } std::vector labels_id{utils::DeserializeLabelsFromLabelIndexStorage(key, value)}; PropertyStore properties{utils::DeserializePropertiesFromLabelIndexStorage(value)}; - auto *disk_storage = static_cast(storage_); - return disk_storage->CreateVertexFromDisk(&transaction_, index_accessor, gid, std::move(labels_id), - std::move(properties), index_delta); + return CreateVertexFromDisk(transaction, index_accessor, gid, std::move(labels_id), std::move(properties), + index_delta); } /// TODO: can be decoupled by providing as arguments extractor functions and delta. -std::optional DiskStorage::DiskAccessor::LoadVertexToLabelPropertyIndexCache( - const std::string &key, const std::string &value, Delta *index_delta, +std::optional DiskStorage::LoadVertexToLabelPropertyIndexCache( + Transaction *transaction, const std::string &key, const std::string &value, Delta *index_delta, utils::SkipList::Accessor index_accessor) { storage::Gid gid = Gid::FromString(utils::ExtractGidFromLabelPropertyIndexStorage(key)); if (ObjectExistsInCache(index_accessor, gid)) { @@ -380,40 +326,35 @@ std::optional DiskStorage::DiskAccessor::LoadVertexToLa } std::vector labels_id{utils::DeserializeLabelsFromLabelPropertyIndexStorage(key, value)}; PropertyStore properties{utils::DeserializePropertiesFromLabelPropertyIndexStorage(value)}; - auto *disk_storage = static_cast(storage_); - return disk_storage->CreateVertexFromDisk(&transaction_, index_accessor, gid, std::move(labels_id), - std::move(properties), index_delta); + return CreateVertexFromDisk(transaction, index_accessor, gid, std::move(labels_id), std::move(properties), + index_delta); } -void DiskStorage::DiskAccessor::LoadVerticesToMainMemoryCache() { - auto *disk_storage = static_cast(storage_); +void DiskStorage::LoadVerticesToMainMemoryCache(Transaction *transaction) { rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; - auto it = std::unique_ptr( - transaction_.disk_transaction_->GetIterator(ro, disk_storage->kvstore_->vertex_chandle)); + auto it = + std::unique_ptr(transaction->disk_transaction_->GetIterator(ro, kvstore_->vertex_chandle)); for (it->SeekToFirst(); it->Valid(); it->Next()) { // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator - disk_storage->LoadVertexToMainMemoryCache(&transaction_, it->key().ToString(), it->value().ToString(), - deserializeTimestamp); + LoadVertexToMainMemoryCache(transaction, it->key().ToString(), it->value().ToString(), kDeserializeTimestamp); } } -/// TODO: how to remove this /// TODO: When loading from disk, you can in some situations load from index rocksdb not the main one /// TODO: send from and to as arguments and remove so many methods -void DiskStorage::DiskAccessor::LoadVerticesFromMainStorageToEdgeImportCache() { - auto *disk_storage = static_cast(storage_); - auto cache_accessor = disk_storage->edge_import_mode_cache_->AccessToVertices(); +void DiskStorage::LoadVerticesFromMainStorageToEdgeImportCache(Transaction *transaction) { + auto cache_accessor = edge_import_mode_cache_->AccessToVertices(); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; - auto it = std::unique_ptr( - transaction_.disk_transaction_->GetIterator(ro, disk_storage->kvstore_->vertex_chandle)); + auto it = + std::unique_ptr(transaction->disk_transaction_->GetIterator(ro, kvstore_->vertex_chandle)); for (it->SeekToFirst(); it->Valid(); it->Next()) { std::string key = it->key().ToString(); @@ -423,32 +364,29 @@ void DiskStorage::DiskAccessor::LoadVerticesFromMainStorageToEdgeImportCache() { std::vector labels_id{utils::DeserializeLabelsFromMainDiskStorage(key)}; PropertyStore properties{utils::DeserializePropertiesFromMainDiskStorage(value)}; - disk_storage->CreateVertexFromDisk( - &transaction_, cache_accessor, gid, std::move(labels_id), std::move(properties), - CreateDeleteDeserializedObjectDelta(&transaction_, std::move(key), deserializeTimestamp)); + CreateVertexFromDisk(transaction, cache_accessor, gid, std::move(labels_id), std::move(properties), + CreateDeleteDeserializedObjectDelta(transaction, std::move(key), kDeserializeTimestamp)); } } -void DiskStorage::DiskAccessor::HandleMainLoadingForEdgeImportCache() { - auto *disk_storage = static_cast(storage_); - if (!disk_storage->edge_import_mode_cache_->AllVerticesScanned()) { - LoadVerticesFromMainStorageToEdgeImportCache(); - disk_storage->edge_import_mode_cache_->SetScannedAllVertices(); +void DiskStorage::HandleMainLoadingForEdgeImportCache(Transaction *transaction) { + if (!edge_import_mode_cache_->AllVerticesScanned()) { + LoadVerticesFromMainStorageToEdgeImportCache(transaction); + edge_import_mode_cache_->SetScannedAllVertices(); } } -void DiskStorage::DiskAccessor::LoadVerticesFromLabelIndexStorageToEdgeImportCache(LabelId label) { - auto *disk_storage = static_cast(storage_); - auto *disk_label_index = static_cast(disk_storage->indices_.label_index_.get()); +void DiskStorage::LoadVerticesFromLabelIndexStorageToEdgeImportCache(Transaction *transaction, LabelId label) { + auto *disk_label_index = static_cast(indices_.label_index_.get()); auto disk_index_transaction = disk_label_index->CreateRocksDBTransaction(); - auto cache_accessor = disk_storage->edge_import_mode_cache_->AccessToVertices(); + auto cache_accessor = edge_import_mode_cache_->AccessToVertices(); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - std::string label_prefix{utils::SerializeIdType(label)}; + std::string label_prefix{label.ToString()}; for (it->SeekToFirst(); it->Valid(); it->Next()) { std::string key = it->key().ToString(); @@ -459,52 +397,47 @@ void DiskStorage::DiskAccessor::LoadVerticesFromLabelIndexStorageToEdgeImportCac std::vector labels_id{utils::DeserializeLabelsFromLabelIndexStorage(key, value)}; PropertyStore properties{utils::DeserializePropertiesFromLabelIndexStorage(value)}; - disk_storage->CreateVertexFromDisk( - &transaction_, cache_accessor, gid, std::move(labels_id), std::move(properties), - CreateDeleteDeserializedObjectDelta(&transaction_, std::move(key), deserializeTimestamp)); + CreateVertexFromDisk(transaction, cache_accessor, gid, std::move(labels_id), std::move(properties), + CreateDeleteDeserializedObjectDelta(transaction, std::move(key), kDeserializeTimestamp)); } } } -void DiskStorage::DiskAccessor::HandleLoadingLabelForEdgeImportCache(LabelId label) { - auto *disk_storage = static_cast(storage_); - if (!disk_storage->edge_import_mode_cache_->VerticesWithLabelScanned(label)) { - LoadVerticesFromLabelIndexStorageToEdgeImportCache(label); +void DiskStorage::HandleLoadingLabelForEdgeImportCache(Transaction *transaction, LabelId label) { + if (!edge_import_mode_cache_->VerticesWithLabelScanned(label)) { + LoadVerticesFromLabelIndexStorageToEdgeImportCache(transaction, label); - if (!disk_storage->edge_import_mode_cache_->CreateIndex(label)) { + if (!edge_import_mode_cache_->CreateIndex(label)) { throw utils::BasicException("Failed creation of in-memory label index."); } } } -void DiskStorage::DiskAccessor::HandleLoadingLabelPropertyForEdgeImportCache(LabelId label, PropertyId property) { - auto *disk_storage = static_cast(storage_); - if (!disk_storage->edge_import_mode_cache_->VerticesWithLabelPropertyScanned(label, property)) { - LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(label, property); +void DiskStorage::HandleLoadingLabelPropertyForEdgeImportCache(Transaction *transaction, LabelId label, + PropertyId property) { + if (!edge_import_mode_cache_->VerticesWithLabelPropertyScanned(label, property)) { + LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(transaction, label, property); - if (!disk_storage->edge_import_mode_cache_->CreateIndex(label, property)) { + if (!edge_import_mode_cache_->CreateIndex(label, property)) { throw utils::BasicException("Failed creation of in-memory label-property index."); } } } /// TODO: Just extract disk_label_index and disk_label_property_index -/// TODO: put it into a EdgeImportModeCache methods -void DiskStorage::DiskAccessor::LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(LabelId label, - PropertyId property) { - auto *disk_storage = static_cast(storage_); - auto *disk_label_property_index = - static_cast(disk_storage->indices_.label_property_index_.get()); +void DiskStorage::LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(Transaction *transaction, LabelId label, + PropertyId property) { + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); auto disk_index_transaction = disk_label_property_index->CreateRocksDBTransaction(); - auto cache_accessor = disk_storage->edge_import_mode_cache_->AccessToVertices(); + auto cache_accessor = edge_import_mode_cache_->AccessToVertices(); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - const std::string label_property_prefix = utils::SerializeIdType(label) + "|" + utils::SerializeIdType(property); + const std::string label_property_prefix = label.ToString() + "|" + property.ToString(); for (it->SeekToFirst(); it->Valid(); it->Next()) { std::string key = it->key().ToString(); std::string value = it->value().ToString(); @@ -514,9 +447,8 @@ void DiskStorage::DiskAccessor::LoadVerticesFromLabelPropertyIndexStorageToEdgeI std::vector labels_id{utils::DeserializeLabelsFromLabelPropertyIndexStorage(key, value)}; PropertyStore properties{utils::DeserializePropertiesFromLabelPropertyIndexStorage(value)}; - disk_storage->CreateVertexFromDisk( - &transaction_, cache_accessor, gid, std::move(labels_id), std::move(properties), - CreateDeleteDeserializedObjectDelta(&transaction_, std::move(key), deserializeTimestamp)); + CreateVertexFromDisk(transaction, cache_accessor, gid, std::move(labels_id), std::move(properties), + CreateDeleteDeserializedObjectDelta(transaction, std::move(key), kDeserializeTimestamp)); } } } @@ -524,7 +456,7 @@ void DiskStorage::DiskAccessor::LoadVerticesFromLabelPropertyIndexStorageToEdgeI VerticesIterable DiskStorage::DiskAccessor::Vertices(View view) { auto *disk_storage = static_cast(storage_); if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { - HandleMainLoadingForEdgeImportCache(); + disk_storage->HandleMainLoadingForEdgeImportCache(&transaction_); return VerticesIterable( AllVerticesIterable(disk_storage->edge_import_mode_cache_->AccessToVertices(), storage_, &transaction_, view)); @@ -533,7 +465,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(View view) { return VerticesIterable(AllVerticesIterable(transaction_.vertices_.access(), storage_, &transaction_, view)); } - LoadVerticesToMainMemoryCache(); + disk_storage->LoadVerticesToMainMemoryCache(&transaction_); transaction_.scanned_all_vertices_ = true; return VerticesIterable(AllVerticesIterable(transaction_.vertices_.access(), storage_, &transaction_, view)); } @@ -542,7 +474,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, View view) { auto *disk_storage = static_cast(storage_); if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { - HandleLoadingLabelForEdgeImportCache(label); + disk_storage->HandleLoadingLabelForEdgeImportCache(&transaction_, label); return VerticesIterable(disk_storage->edge_import_mode_cache_->Vertices(label, view, storage_, &transaction_)); } @@ -552,8 +484,9 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, View view) { transaction_.index_deltas_storage_.emplace_back(); auto &index_deltas = transaction_.index_deltas_storage_.back(); - auto gids = MergeVerticesFromMainCacheWithLabelIndexCache(label, view, index_deltas, indexed_vertices.get()); - LoadVerticesFromDiskLabelIndex(label, gids, index_deltas, indexed_vertices.get()); + auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelIndexCache(&transaction_, label, view, index_deltas, + indexed_vertices.get()); + disk_storage->LoadVerticesFromDiskLabelIndex(&transaction_, label, gids, index_deltas, indexed_vertices.get()); return VerticesIterable(AllVerticesIterable(indexed_vertices->access(), storage_, &transaction_, view)); } @@ -561,7 +494,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, View view) { VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId property, View view) { auto *disk_storage = static_cast(storage_); if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { - HandleLoadingLabelPropertyForEdgeImportCache(label, property); + disk_storage->HandleLoadingLabelPropertyForEdgeImportCache(&transaction_, label, property); return VerticesIterable(disk_storage->edge_import_mode_cache_->Vertices(label, property, std::nullopt, std::nullopt, view, storage_, &transaction_)); @@ -578,16 +511,16 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p HasVertexProperty(vertex, property, &transaction_, view); }; - const auto gids = MergeVerticesFromMainCacheWithLabelPropertyIndexCache( - label, property, view, index_deltas, indexed_vertices.get(), label_property_filter); + const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCache( + &transaction_, label, property, view, index_deltas, indexed_vertices.get(), label_property_filter); const auto disk_label_property_filter = [](const std::string &key, const std::string &label_property_prefix, const std::unordered_set &gids, Gid curr_gid) -> bool { return key.starts_with(label_property_prefix) && !utils::Contains(gids, curr_gid); }; - LoadVerticesFromDiskLabelPropertyIndex(label, property, gids, index_deltas, indexed_vertices.get(), - disk_label_property_filter); + disk_storage->LoadVerticesFromDiskLabelPropertyIndex(&transaction_, label, property, gids, index_deltas, + indexed_vertices.get(), disk_label_property_filter); return VerticesIterable(AllVerticesIterable(indexed_vertices->access(), storage_, &transaction_, view)); } @@ -596,7 +529,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p View view) { auto *disk_storage = static_cast(storage_); if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { - HandleLoadingLabelPropertyForEdgeImportCache(label, property); + disk_storage->HandleLoadingLabelPropertyForEdgeImportCache(&transaction_, label, property); return VerticesIterable(disk_storage->edge_import_mode_cache_->Vertices( label, property, utils::MakeBoundInclusive(value), utils::MakeBoundInclusive(value), view, storage_, @@ -614,11 +547,11 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p VertexHasEqualPropertyValue(vertex, property, value, &transaction_, view); }; - const auto gids = MergeVerticesFromMainCacheWithLabelPropertyIndexCache( - label, property, view, index_deltas, indexed_vertices.get(), label_property_filter); + const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCache( + &transaction_, label, property, view, index_deltas, indexed_vertices.get(), label_property_filter); - LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup(label, property, gids, value, index_deltas, - indexed_vertices.get()); + disk_storage->LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup(&transaction_, label, property, gids, value, + index_deltas, indexed_vertices.get()); return VerticesIterable(AllVerticesIterable(indexed_vertices->access(), storage_, &transaction_, view)); } @@ -629,7 +562,7 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p View view) { auto *disk_storage = static_cast(storage_); if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { - HandleLoadingLabelPropertyForEdgeImportCache(label, property); + disk_storage->HandleLoadingLabelPropertyForEdgeImportCache(&transaction_, label, property); return VerticesIterable(disk_storage->edge_import_mode_cache_->Vertices(label, property, lower_bound, upper_bound, view, storage_, &transaction_)); @@ -640,30 +573,30 @@ VerticesIterable DiskStorage::DiskAccessor::Vertices(LabelId label, PropertyId p transaction_.index_deltas_storage_.emplace_back(); auto &index_deltas = transaction_.index_deltas_storage_.back(); - const auto gids = MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( - label, property, view, lower_bound, upper_bound, index_deltas, indexed_vertices.get()); + const auto gids = disk_storage->MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( + &transaction_, label, property, view, lower_bound, upper_bound, index_deltas, indexed_vertices.get()); - LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch(label, property, gids, lower_bound, upper_bound, index_deltas, - indexed_vertices.get()); + disk_storage->LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch( + &transaction_, label, property, gids, lower_bound, upper_bound, index_deltas, indexed_vertices.get()); return VerticesIterable(AllVerticesIterable(indexed_vertices->access(), storage_, &transaction_, view)); } /// TODO: (andi) This should probably go into some other class not the storage. All utils methods -std::unordered_set DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWithLabelIndexCache( - LabelId label, View view, std::list &index_deltas, utils::SkipList *indexed_vertices) { - auto main_cache_acc = transaction_.vertices_.access(); +std::unordered_set DiskStorage::MergeVerticesFromMainCacheWithLabelIndexCache( + Transaction *transaction, LabelId label, View view, std::list &index_deltas, + utils::SkipList *indexed_vertices) { + auto main_cache_acc = transaction->vertices_.access(); std::unordered_set gids; gids.reserve(main_cache_acc.size()); for (const auto &vertex : main_cache_acc) { gids.insert(vertex.gid); - if (VertexHasLabel(vertex, label, &transaction_, view)) { - spdlog::trace("Loaded vertex with gid: {} from main index storage to label index", - utils::SerializeIdType(vertex.gid)); + if (VertexHasLabel(vertex, label, transaction, view)) { + spdlog::trace("Loaded vertex with gid: {} from main index storage to label index", vertex.gid.ToString()); uint64_t ts = utils::GetEarliestTimestamp(vertex.delta); /// TODO: here are doing serialization and then later deserialization again -> expensive - LoadVertexToLabelIndexCache(utils::SerializeVertexAsKeyForLabelIndex(label, vertex.gid), + LoadVertexToLabelIndexCache(transaction, utils::SerializeVertexAsKeyForLabelIndex(label, vertex.gid), utils::SerializeVertexAsValueForLabelIndex(label, vertex.labels, vertex.properties), CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts), indexed_vertices->access()); @@ -672,21 +605,21 @@ std::unordered_set DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWit return gids; } -void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelIndex(LabelId label, - const std::unordered_set &gids, - std::list &index_deltas, - utils::SkipList *indexed_vertices) { - auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); +void DiskStorage::LoadVerticesFromDiskLabelIndex(Transaction *transaction, LabelId label, + const std::unordered_set &gids, + std::list &index_deltas, + utils::SkipList *indexed_vertices) { + auto *disk_label_index = static_cast(indices_.label_index_.get()); auto disk_index_transaction = disk_label_index->CreateRocksDBTransaction(); - disk_index_transaction->SetReadTimestampForValidation(transaction_.start_timestamp); + disk_index_transaction->SetReadTimestampForValidation(transaction->start_timestamp); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto index_it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - const auto serialized_label = utils::SerializeIdType(label); + const auto serialized_label = label.ToString(); for (index_it->SeekToFirst(); index_it->Valid(); index_it->Next()) { std::string key = index_it->key().ToString(); Gid curr_gid = Gid::FromString(utils::ExtractGidFromLabelIndexStorage(key)); @@ -695,27 +628,26 @@ void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelIndex(LabelId label, // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator LoadVertexToLabelIndexCache( - index_it->key().ToString(), index_it->value().ToString(), - CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), deserializeTimestamp), + transaction, index_it->key().ToString(), index_it->value().ToString(), + CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), kDeserializeTimestamp), indexed_vertices->access()); } } } -std::unordered_set DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWithLabelPropertyIndexCache( - LabelId label, PropertyId property, View view, std::list &index_deltas, +std::unordered_set DiskStorage::MergeVerticesFromMainCacheWithLabelPropertyIndexCache( + Transaction *transaction, LabelId label, PropertyId property, View view, std::list &index_deltas, utils::SkipList *indexed_vertices, const auto &label_property_filter) { - auto main_cache_acc = transaction_.vertices_.access(); + auto main_cache_acc = transaction->vertices_.access(); std::unordered_set gids; gids.reserve(main_cache_acc.size()); for (const auto &vertex : main_cache_acc) { gids.insert(vertex.gid); - /// TODO: delta support for clearing old disk keys if (label_property_filter(vertex, label, property, view)) { uint64_t ts = utils::GetEarliestTimestamp(vertex.delta); LoadVertexToLabelPropertyIndexCache( - utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid), + transaction, utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid), utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties), CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts), indexed_vertices->access()); } @@ -724,87 +656,83 @@ std::unordered_set DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWit return gids; } -void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelPropertyIndex(LabelId label, PropertyId property, - const std::unordered_set &gids, - std::list &index_deltas, - utils::SkipList *indexed_vertices, - const auto &label_property_filter) { - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); +void DiskStorage::LoadVerticesFromDiskLabelPropertyIndex(Transaction *transaction, LabelId label, PropertyId property, + const std::unordered_set &gids, + std::list &index_deltas, + utils::SkipList *indexed_vertices, + const auto &label_property_filter) { + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); auto disk_index_transaction = disk_label_property_index->CreateRocksDBTransaction(); - disk_index_transaction->SetReadTimestampForValidation(transaction_.start_timestamp); + disk_index_transaction->SetReadTimestampForValidation(transaction->start_timestamp); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto index_it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - const auto label_property_prefix = utils::SerializeIdType(label) + "|" + utils::SerializeIdType(property); + const auto label_property_prefix = label.ToString() + "|" + property.ToString(); for (index_it->SeekToFirst(); index_it->Valid(); index_it->Next()) { std::string key = index_it->key().ToString(); Gid curr_gid = Gid::FromString(utils::ExtractGidFromLabelPropertyIndexStorage(key)); - /// TODO: optimize if (label_property_filter(key, label_property_prefix, gids, curr_gid)) { // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator LoadVertexToLabelPropertyIndexCache( - index_it->key().ToString(), index_it->value().ToString(), - CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), deserializeTimestamp), + transaction, index_it->key().ToString(), index_it->value().ToString(), + CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), kDeserializeTimestamp), indexed_vertices->access()); } } } -void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup( - LabelId label, PropertyId property, const std::unordered_set &gids, const PropertyValue &value, - std::list &index_deltas, utils::SkipList *indexed_vertices) { - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); +void DiskStorage::LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup( + Transaction *transaction, LabelId label, PropertyId property, const std::unordered_set &gids, + const PropertyValue &value, std::list &index_deltas, utils::SkipList *indexed_vertices) { + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); auto disk_index_transaction = disk_label_property_index->CreateRocksDBTransaction(); - disk_index_transaction->SetReadTimestampForValidation(transaction_.start_timestamp); + disk_index_transaction->SetReadTimestampForValidation(transaction->start_timestamp); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto index_it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - const auto label_property_prefix = utils::SerializeIdType(label) + "|" + utils::SerializeIdType(property); + const auto label_property_prefix = label.ToString() + "|" + property.ToString(); for (index_it->SeekToFirst(); index_it->Valid(); index_it->Next()) { std::string key = index_it->key().ToString(); std::string it_value = index_it->value().ToString(); Gid curr_gid = Gid::FromString(utils::ExtractGidFromLabelPropertyIndexStorage(key)); - /// TODO: optimize PropertyStore properties = utils::DeserializePropertiesFromLabelPropertyIndexStorage(it_value); if (key.starts_with(label_property_prefix) && !utils::Contains(gids, curr_gid) && properties.IsPropertyEqual(property, value)) { // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator LoadVertexToLabelPropertyIndexCache( - index_it->key().ToString(), index_it->value().ToString(), - CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), deserializeTimestamp), + transaction, index_it->key().ToString(), index_it->value().ToString(), + CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key), kDeserializeTimestamp), indexed_vertices->access()); } } } -std::unordered_set -DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( - LabelId label, PropertyId property, View view, const std::optional> &lower_bound, +std::unordered_set DiskStorage::MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( + Transaction *transaction, LabelId label, PropertyId property, View view, + const std::optional> &lower_bound, const std::optional> &upper_bound, std::list &index_deltas, utils::SkipList *indexed_vertices) { - auto main_cache_acc = transaction_.vertices_.access(); + auto main_cache_acc = transaction->vertices_.access(); std::unordered_set gids; gids.reserve(main_cache_acc.size()); for (const auto &vertex : main_cache_acc) { gids.insert(vertex.gid); - auto prop_value = GetVertexProperty(vertex, property, &transaction_, view); - if (VertexHasLabel(vertex, label, &transaction_, view) && + auto prop_value = GetVertexProperty(vertex, property, transaction, view); + if (VertexHasLabel(vertex, label, transaction, view) && IsPropertyValueWithinInterval(prop_value, lower_bound, upper_bound)) { uint64_t ts = utils::GetEarliestTimestamp(vertex.delta); LoadVertexToLabelPropertyIndexCache( - utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid), + transaction, utils::SerializeVertexAsKeyForLabelPropertyIndex(label, property, vertex.gid), utils::SerializeVertexAsValueForLabelPropertyIndex(label, vertex.labels, vertex.properties), CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::nullopt, ts), indexed_vertices->access()); } @@ -812,29 +740,27 @@ DiskStorage::DiskAccessor::MergeVerticesFromMainCacheWithLabelPropertyIndexCache return gids; } -void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch( - LabelId label, PropertyId property, const std::unordered_set &gids, +void DiskStorage::LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch( + Transaction *transaction, LabelId label, PropertyId property, const std::unordered_set &gids, const std::optional> &lower_bound, const std::optional> &upper_bound, std::list &index_deltas, utils::SkipList *indexed_vertices) { - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); auto disk_index_transaction = disk_label_property_index->CreateRocksDBTransaction(); - disk_index_transaction->SetReadTimestampForValidation(transaction_.start_timestamp); + disk_index_transaction->SetReadTimestampForValidation(transaction->start_timestamp); rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; auto index_it = std::unique_ptr(disk_index_transaction->GetIterator(ro)); - const std::string label_property_prefix = utils::SerializeIdType(label) + "|" + utils::SerializeIdType(property); + const std::string label_property_prefix = label.ToString() + "|" + property.ToString(); for (index_it->SeekToFirst(); index_it->Valid(); index_it->Next()) { std::string key_str = index_it->key().ToString(); std::string it_value_str = index_it->value().ToString(); Gid curr_gid = Gid::FromString(utils::ExtractGidFromLabelPropertyIndexStorage(key_str)); /// TODO: andi this will be optimized - /// TODO: couple this condition PropertyStore properties = utils::DeserializePropertiesFromLabelPropertyIndexStorage(it_value_str); PropertyValue prop_value = properties.GetProperty(property); if (!key_str.starts_with(label_property_prefix) || utils::Contains(gids, curr_gid) || @@ -844,8 +770,8 @@ void DiskStorage::DiskAccessor::LoadVerticesFromDiskLabelPropertyIndexForInterva // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator LoadVertexToLabelPropertyIndexCache( - index_it->key().ToString(), index_it->value().ToString(), - CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key_str), deserializeTimestamp), + transaction, index_it->key().ToString(), index_it->value().ToString(), + CreateDeleteDeserializedIndexObjectDelta(index_deltas, std::move(key_str), kDeserializeTimestamp), indexed_vertices->access()); } } @@ -855,85 +781,6 @@ uint64_t DiskStorage::DiskAccessor::ApproximateVertexCount() const { return disk_storage->vertex_count_.load(std::memory_order_acquire); } -bool DiskStorage::PersistLabelIndexCreation(LabelId label) const { - if (auto label_index_store = durability_kvstore_->Get(label_index_str); label_index_store.has_value()) { - std::string &value = label_index_store.value(); - value += "|" + utils::SerializeIdType(label); - return durability_kvstore_->Put(label_index_str, value); - } - return durability_kvstore_->Put(label_index_str, utils::SerializeIdType(label)); -} - -bool DiskStorage::PersistLabelIndexDeletion(LabelId label) const { - if (auto label_index_store = durability_kvstore_->Get(label_index_str); label_index_store.has_value()) { - const std::string &value = label_index_store.value(); - std::vector labels = utils::Split(value, "|"); - std::erase(labels, utils::SerializeIdType(label)); - if (labels.empty()) { - return durability_kvstore_->Delete(label_index_str); - } - return durability_kvstore_->Put(label_index_str, utils::Join(labels, "|")); - } - return true; -} - -bool DiskStorage::PersistLabelPropertyIndexAndExistenceConstraintCreation(LabelId label, PropertyId property, - const char *key) const { - if (auto label_property_index_store = durability_kvstore_->Get(key); label_property_index_store.has_value()) { - std::string &value = label_property_index_store.value(); - value += "|" + utils::SerializeIdType(label) + "," + utils::SerializeIdType(property); - return durability_kvstore_->Put(key, value); - } - return durability_kvstore_->Put(key, utils::SerializeIdType(label) + "," + utils::SerializeIdType(property)); -} - -bool DiskStorage::PersistLabelPropertyIndexAndExistenceConstraintDeletion(LabelId label, PropertyId property, - const char *key) const { - if (auto label_property_index_store = durability_kvstore_->Get(key); label_property_index_store.has_value()) { - const std::string &value = label_property_index_store.value(); - std::vector label_properties = utils::Split(value, "|"); - std::erase(label_properties, utils::SerializeIdType(label) + "," + utils::SerializeIdType(property)); - if (label_properties.empty()) { - return durability_kvstore_->Delete(key); - } - return durability_kvstore_->Put(key, utils::Join(label_properties, "|")); - } - return true; -} - -bool DiskStorage::PersistUniqueConstraintCreation(LabelId label, const std::set &properties) const { - std::string entry = utils::SerializeIdType(label); - for (auto property : properties) { - entry += "," + utils::SerializeIdType(property); - } - - if (auto unique_store = durability_kvstore_->Get(unique_constraints_str); unique_store.has_value()) { - std::string &value = unique_store.value(); - value += "|" + entry; - return durability_kvstore_->Put(unique_constraints_str, value); - } - return durability_kvstore_->Put(unique_constraints_str, entry); -} - -bool DiskStorage::PersistUniqueConstraintDeletion(LabelId label, const std::set &properties) const { - /// TODO: move to rocksdb_serialization.hpp - std::string entry = utils::SerializeIdType(label); - for (auto property : properties) { - entry += "," + utils::SerializeIdType(property); - } - - if (auto unique_store = durability_kvstore_->Get(unique_constraints_str); unique_store.has_value()) { - const std::string &value = unique_store.value(); - std::vector unique_constraints = utils::Split(value, "|"); - std::erase(unique_constraints, entry); - if (unique_constraints.empty()) { - return durability_kvstore_->Delete(unique_constraints_str); - } - return durability_kvstore_->Put(unique_constraints_str, utils::Join(unique_constraints, "|")); - } - return true; -} - uint64_t DiskStorage::GetDiskSpaceUsage() const { uint64_t main_disk_storage_size = utils::GetDirDiskUsage(config_.disk.main_storage_directory); uint64_t index_disk_storage_size = utils::GetDirDiskUsage(config_.disk.label_index_directory) + @@ -1022,21 +869,19 @@ DiskStorage::DiskAccessor::DetachDelete(std::vector nodes, std auto *disk_storage = static_cast(storage_); for (const auto &vertex : deleted_vertices) { - transaction_.vertices_to_delete_.emplace(utils::SerializeIdType(vertex.vertex_->gid), - utils::SerializeVertex(*vertex.vertex_)); + transaction_.vertices_to_delete_.emplace(vertex.vertex_->gid.ToString(), utils::SerializeVertex(*vertex.vertex_)); transaction_.manyDeltasCache.Invalidate(vertex.vertex_); disk_storage->vertex_count_.fetch_sub(1, std::memory_order_acq_rel); } for (const auto &edge : deleted_edges) { - const std::string ser_edge_gid = utils::SerializeIdType(edge.Gid()); - const auto src_vertex_gid = utils::SerializeIdType(edge.from_vertex_->gid); - const auto dst_vertex_gid = utils::SerializeIdType(edge.to_vertex_->gid); + const std::string ser_edge_gid = edge.Gid().ToString(); + const auto src_vertex_gid = edge.from_vertex_->gid.ToString(); + const auto dst_vertex_gid = edge.to_vertex_->gid.ToString(); transaction_.edges_to_delete_.emplace(ser_edge_gid, std::make_pair(src_vertex_gid, dst_vertex_gid)); transaction_.manyDeltasCache.Invalidate(edge.from_vertex_, edge.edge_type_, EdgeDirection::OUT); transaction_.manyDeltasCache.Invalidate(edge.to_vertex_, edge.edge_type_, EdgeDirection::IN); - /// TODO: (andi) Error handling of modified edge, this just returns void transaction_.RemoveModifiedEdge(edge.Gid()); } @@ -1068,7 +913,7 @@ Result DiskStorage::DiskAccessor::CreateEdge(VertexAccessor *from, } ModifiedEdgeInfo modified_edge(Delta::Action::DELETE_OBJECT, from_vertex->gid, to_vertex->gid, edge_type, edge); - /// TODO: (andi) Not sure if here should be modified edge. + /// TODO: (andi) Change when decoupled edge creation from edge deletion. transaction_.AddModifiedEdge(gid, modified_edge); CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge); @@ -1095,56 +940,53 @@ Result DiskStorage::DiskAccessor::EdgeSetTo(EdgeAccessor * /*edge* return Error::NONEXISTENT_OBJECT; } -/// TODO: this method should also delete the old key -bool DiskStorage::DiskAccessor::WriteVertexToVertexColumnFamily(const Vertex &vertex) { - MG_ASSERT(commit_timestamp_.has_value(), "Writing vertex to disk but commit timestamp not set."); - auto *disk_storage = static_cast(storage_); +bool DiskStorage::WriteVertexToVertexColumnFamily(Transaction *transaction, const Vertex &vertex) { + MG_ASSERT(transaction->commit_timestamp, "Writing vertex to disk but commit timestamp not set."); + auto commit_ts = transaction->commit_timestamp->load(std::memory_order_relaxed); const auto ser_vertex = utils::SerializeVertex(vertex); - auto status = transaction_.disk_transaction_->Put(disk_storage->kvstore_->vertex_chandle, ser_vertex, + auto status = transaction->disk_transaction_->Put(kvstore_->vertex_chandle, ser_vertex, utils::SerializeProperties(vertex.properties)); if (status.ok()) { - spdlog::trace("rocksdb: Saved vertex with key {} and ts {} to vertex column family", ser_vertex, - *commit_timestamp_); + spdlog::trace("rocksdb: Saved vertex with key {} and ts {} to vertex column family", ser_vertex, commit_ts); return true; } - spdlog::error("rocksdb: Failed to save vertex with key {} and ts {} to vertex column family", ser_vertex, - *commit_timestamp_); + spdlog::error("rocksdb: Failed to save vertex with key {} and ts {} to vertex column family", ser_vertex, commit_ts); return false; } -bool DiskStorage::DiskAccessor::WriteEdgeToEdgeColumnFamily(const std::string &serialized_edge_key, - const std::string &serialized_edge_value) { - MG_ASSERT(commit_timestamp_.has_value(), "Writing edge to disk but commit timestamp not set."); - auto *disk_storage = static_cast(storage_); - rocksdb::Status status = transaction_.disk_transaction_->Put(disk_storage->kvstore_->edge_chandle, - serialized_edge_key, serialized_edge_value); +bool DiskStorage::WriteEdgeToEdgeColumnFamily(Transaction *transaction, const std::string &serialized_edge_key, + const std::string &serialized_edge_value) { + MG_ASSERT(transaction->commit_timestamp, "Writing edge to disk but commit timestamp not set."); + auto commit_ts = transaction->commit_timestamp->load(std::memory_order_relaxed); + rocksdb::Status status = + transaction->disk_transaction_->Put(kvstore_->edge_chandle, serialized_edge_key, serialized_edge_value); if (status.ok()) { - spdlog::trace("rocksdb: Saved edge {} with ts {} to edge column family", serialized_edge_key, *commit_timestamp_); + spdlog::trace("rocksdb: Saved edge {} with ts {} to edge column family", serialized_edge_key, commit_ts); return true; } - spdlog::error("rocksdb: Failed to save edge {} with ts {} to edge column family", serialized_edge_key, - *commit_timestamp_); + spdlog::error("rocksdb: Failed to save edge {} with ts {} to edge column family", serialized_edge_key, commit_ts); return false; } -bool DiskStorage::DiskAccessor::WriteEdgeToConnectivityIndex(const std::string &vertex_gid, const std::string &edge_gid, - rocksdb::ColumnFamilyHandle *handle, std::string mode) { - MG_ASSERT(commit_timestamp_.has_value(), "Writing edge to disk but commit timestamp not set."); +/// NOLINTNEXTLINE(readability-convert-member-functions-to-static) +bool DiskStorage::WriteEdgeToConnectivityIndex(Transaction *transaction, const std::string &vertex_gid, + const std::string &edge_gid, rocksdb::ColumnFamilyHandle *handle, + std::string mode) { + MG_ASSERT(transaction->commit_timestamp, "Writing edge to disk but commit timestamp not set."); std::string value; - const auto put_status = std::invoke([this, handle, &value, &vertex_gid, &edge_gid]() { + const auto put_status = std::invoke([transaction, handle, &value, &vertex_gid, &edge_gid]() { rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; - if (transaction_.disk_transaction_->Get(ro, handle, vertex_gid, &value).IsNotFound()) { - return transaction_.disk_transaction_->Put(handle, vertex_gid, edge_gid); + if (transaction->disk_transaction_->Get(ro, handle, vertex_gid, &value).IsNotFound()) { + return transaction->disk_transaction_->Put(handle, vertex_gid, edge_gid); } - return transaction_.disk_transaction_->Put(handle, vertex_gid, value + "," + edge_gid); + return transaction->disk_transaction_->Put(handle, vertex_gid, value + "," + edge_gid); }); - /// TODO: (andi) Error handling in a separate method if (put_status.ok()) { spdlog::trace("rocksdb: Saved edge {} to {} edges connectivity index for vertex {}", edge_gid, mode, vertex_gid); return true; @@ -1155,14 +997,12 @@ bool DiskStorage::DiskAccessor::WriteEdgeToConnectivityIndex(const std::string & return false; } -bool DiskStorage::DiskAccessor::DeleteVertexFromDisk(const std::string &vertex_gid, const std::string &vertex) { - auto *disk_storage = static_cast(storage_); +bool DiskStorage::DeleteVertexFromDisk(Transaction *transaction, const std::string &vertex_gid, + const std::string &vertex) { /// TODO: (andi) This should be atomic delete. - auto vertex_del_status = transaction_.disk_transaction_->Delete(disk_storage->kvstore_->vertex_chandle, vertex); - auto vertex_out_conn_status = - transaction_.disk_transaction_->Delete(disk_storage->kvstore_->out_edges_chandle, vertex_gid); - auto vertex_in_conn_status = - transaction_.disk_transaction_->Delete(disk_storage->kvstore_->in_edges_chandle, vertex_gid); + auto vertex_del_status = transaction->disk_transaction_->Delete(kvstore_->vertex_chandle, vertex); + auto vertex_out_conn_status = transaction->disk_transaction_->Delete(kvstore_->out_edges_chandle, vertex_gid); + auto vertex_in_conn_status = transaction->disk_transaction_->Delete(kvstore_->in_edges_chandle, vertex_gid); if (vertex_del_status.ok() && vertex_out_conn_status.ok() && vertex_in_conn_status.ok()) { spdlog::trace("rocksdb: Deleted vertex with key {}", vertex); @@ -1172,9 +1012,8 @@ bool DiskStorage::DiskAccessor::DeleteVertexFromDisk(const std::string &vertex_g return false; } -bool DiskStorage::DiskAccessor::DeleteEdgeFromEdgeColumnFamily(const std::string &edge_gid) { - auto *disk_storage = static_cast(storage_); - if (!transaction_.disk_transaction_->Delete(disk_storage->kvstore_->edge_chandle, edge_gid).ok()) { +bool DiskStorage::DeleteEdgeFromEdgeColumnFamily(Transaction *transaction, const std::string &edge_gid) { + if (!transaction->disk_transaction_->Delete(kvstore_->edge_chandle, edge_gid).ok()) { spdlog::error("rocksdb: Failed to delete edge {}", edge_gid); return false; } @@ -1185,23 +1024,22 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromEdgeColumnFamily(const std::string /// TODO: (andi) This is currently not optimal as it will for each edge deserialize all neighborhood edges /// and then remove the edge from the neighborhood edges. This can be optimized by saving vertices together with // deleted edges and then modifying the deletion procedure. This is currently bad if we have some supernode. -bool DiskStorage::DiskAccessor::DeleteEdgeFromDisk(const std::string &edge_gid, const std::string &src_vertex_gid, - const std::string &dst_vertex_gid) { +bool DiskStorage::DeleteEdgeFromDisk(Transaction *transaction, const std::string &edge_gid, + const std::string &src_vertex_gid, const std::string &dst_vertex_gid) { /// TODO: (andi) Should be atomic deletion. - if (!DeleteEdgeFromEdgeColumnFamily(edge_gid)) { + if (!DeleteEdgeFromEdgeColumnFamily(transaction, edge_gid)) { return false; } - auto *disk_storage = static_cast(storage_); - if (!transaction_.vertices_to_delete_.contains(src_vertex_gid)) { - if (!DeleteEdgeFromConnectivityIndex(src_vertex_gid, edge_gid, disk_storage->kvstore_->out_edges_chandle, "OUT")) { + if (!transaction->vertices_to_delete_.contains(src_vertex_gid)) { + if (!DeleteEdgeFromConnectivityIndex(transaction, src_vertex_gid, edge_gid, kvstore_->out_edges_chandle, "OUT")) { spdlog::error("rocksdb: Failed to delete edge with key {}", edge_gid); return false; } spdlog::trace("rocksdb: Deleted edge with key {} from out edges of vertex", edge_gid, src_vertex_gid); } - if (!transaction_.vertices_to_delete_.contains(dst_vertex_gid)) { - if (!DeleteEdgeFromConnectivityIndex(dst_vertex_gid, edge_gid, disk_storage->kvstore_->in_edges_chandle, "IN")) { + if (!transaction->vertices_to_delete_.contains(dst_vertex_gid)) { + if (!DeleteEdgeFromConnectivityIndex(transaction, dst_vertex_gid, edge_gid, kvstore_->in_edges_chandle, "IN")) { spdlog::error("rocksdb: Failed to delete edge with key {}", edge_gid); return false; } @@ -1211,16 +1049,17 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromDisk(const std::string &edge_gid, return true; } -bool DiskStorage::DiskAccessor::DeleteEdgeFromConnectivityIndex(const std::string &vertex_gid, - const std::string &edge_gid, - rocksdb::ColumnFamilyHandle *handle, std::string mode) { +/// NOLINTNEXTLINE(readability-convert-member-functions-to-static) +bool DiskStorage::DeleteEdgeFromConnectivityIndex(Transaction *transaction, const std::string &vertex_gid, + const std::string &edge_gid, rocksdb::ColumnFamilyHandle *handle, + std::string mode) { rocksdb::ReadOptions ro; - std::string strTs = utils::StringTimestamp(transaction_.start_timestamp); + std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); ro.timestamp = &ts; std::string edges; - auto edges_status = transaction_.disk_transaction_->Get(ro, handle, vertex_gid, &edges); + auto edges_status = transaction->disk_transaction_->Get(ro, handle, vertex_gid, &edges); if (!edges_status.ok()) { /// NOTE: Edge could be created and deleted in the same txn, so no need to fail explicitly. spdlog::error("rocksdb: Failed to find {} edges collection of vertex {}.", mode, vertex_gid); @@ -1230,7 +1069,7 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromConnectivityIndex(const std::strin std::vector edges_vec = utils::Split(edges, ","); MG_ASSERT(std::erase(edges_vec, edge_gid) > 0U, "Edge must be in the edges collection of vertex"); if (edges_vec.empty()) { - if (!transaction_.disk_transaction_->Delete(handle, vertex_gid).ok()) { + if (!transaction->disk_transaction_->Delete(handle, vertex_gid).ok()) { spdlog::error("rocksdb: Failed to delete edge {} from edges connectivity index for vertex {}", edge_gid, vertex_gid); return false; @@ -1238,7 +1077,7 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromConnectivityIndex(const std::strin return true; } - if (!transaction_.disk_transaction_->Put(handle, vertex_gid, utils::Join(edges_vec, ",")).ok()) { + if (!transaction->disk_transaction_->Put(handle, vertex_gid, utils::Join(edges_vec, ",")).ok()) { spdlog::error("rocksdb: Failed to delete edge {} from edges connectivity index for vertex {}", edge_gid, vertex_gid); return false; @@ -1246,16 +1085,14 @@ bool DiskStorage::DiskAccessor::DeleteEdgeFromConnectivityIndex(const std::strin return true; } -[[nodiscard]] utils::BasicResult -DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( +[[nodiscard]] utils::BasicResult DiskStorage::CheckVertexConstraintsBeforeCommit( const Vertex &vertex, std::vector> &unique_storage) const { - if (auto existence_constraint_validation_result = storage_->constraints_.existence_constraints_->Validate(vertex); + if (auto existence_constraint_validation_result = constraints_.existence_constraints_->Validate(vertex); existence_constraint_validation_result.has_value()) { return StorageManipulationError{existence_constraint_validation_result.value()}; } - auto *disk_unique_constraints = - static_cast(storage_->constraints_.unique_constraints_.get()); + auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); if (auto unique_constraint_validation_result = disk_unique_constraints->Validate(vertex, unique_storage); unique_constraint_validation_result.has_value()) { return StorageManipulationError{unique_constraint_validation_result.value()}; @@ -1263,14 +1100,13 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushVertices( - const auto &vertex_acc, std::vector> &unique_storage) { - auto *disk_unique_constraints = - static_cast(storage_->constraints_.unique_constraints_.get()); - auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); +[[nodiscard]] utils::BasicResult DiskStorage::FlushVertices( + Transaction *transaction, const auto &vertex_acc, std::vector> &unique_storage) { + auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); + auto *disk_label_index = static_cast(indices_.label_index_.get()); + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); + auto commit_ts = transaction->commit_timestamp->load(std::memory_order_relaxed); for (const Vertex &vertex : vertex_acc) { if (!VertexNeedsToBeSerialized(vertex)) { continue; @@ -1285,18 +1121,18 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( /// NOTE: this deletion has to come before writing, otherwise RocksDB thinks that all entries are deleted if (auto maybe_old_disk_key = utils::GetOldDiskKeyOrNull(vertex.delta); maybe_old_disk_key.has_value()) { - if (!DeleteVertexFromDisk(utils::SerializeIdType(vertex.gid), maybe_old_disk_key.value())) { + if (!DeleteVertexFromDisk(transaction, vertex.gid.ToString(), maybe_old_disk_key.value())) { return StorageManipulationError{SerializationError{}}; } } - if (!WriteVertexToVertexColumnFamily(vertex)) { + if (!WriteVertexToVertexColumnFamily(transaction, vertex)) { return StorageManipulationError{SerializationError{}}; } - if (!disk_unique_constraints->SyncVertexToUniqueConstraintsStorage(vertex, *commit_timestamp_) || - !disk_label_index->SyncVertexToLabelIndexStorage(vertex, *commit_timestamp_) || - !disk_label_property_index->SyncVertexToLabelPropertyIndexStorage(vertex, *commit_timestamp_)) { + if (!disk_unique_constraints->SyncVertexToUniqueConstraintsStorage(vertex, commit_ts) || + !disk_label_index->SyncVertexToLabelIndexStorage(vertex, commit_ts) || + !disk_label_property_index->SyncVertexToLabelPropertyIndexStorage(vertex, commit_ts)) { return StorageManipulationError{SerializationError{}}; } } @@ -1304,28 +1140,27 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::ClearDanglingVertices() { - auto *disk_unique_constraints = - static_cast(storage_->constraints_.unique_constraints_.get()); - auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); +[[nodiscard]] utils::BasicResult DiskStorage::ClearDanglingVertices( + Transaction *transaction) { + auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); + auto *disk_label_index = static_cast(indices_.label_index_.get()); + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); - if (!disk_unique_constraints->DeleteVerticesWithRemovedConstraintLabel(transaction_.start_timestamp, - *commit_timestamp_) || - !disk_label_index->DeleteVerticesWithRemovedIndexingLabel(transaction_.start_timestamp, *commit_timestamp_) || - !disk_label_property_index->DeleteVerticesWithRemovedIndexingLabel(transaction_.start_timestamp, - *commit_timestamp_)) { + auto commit_ts = transaction->commit_timestamp->load(std::memory_order_relaxed); + if (!disk_unique_constraints->DeleteVerticesWithRemovedConstraintLabel(transaction->start_timestamp, commit_ts) || + !disk_label_index->DeleteVerticesWithRemovedIndexingLabel(transaction->start_timestamp, commit_ts) || + !disk_label_property_index->DeleteVerticesWithRemovedIndexingLabel(transaction->start_timestamp, commit_ts)) { return StorageManipulationError{SerializationError{}}; } return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushIndexCache() { +[[nodiscard]] utils::BasicResult DiskStorage::FlushIndexCache( + Transaction *transaction) { std::vector> unique_storage; - for (const auto &vec : transaction_.index_storage_) { - if (auto vertices_res = FlushVertices(vec->access(), unique_storage); vertices_res.HasError()) { + for (const auto &vec : transaction->index_storage_) { + if (auto vertices_res = FlushVertices(transaction, vec->access(), unique_storage); vertices_res.HasError()) { return vertices_res.GetError(); } } @@ -1333,18 +1168,18 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedVertices() { - auto *disk_unique_constraints = - static_cast(storage_->constraints_.unique_constraints_.get()); - auto *disk_label_index = static_cast(storage_->indices_.label_index_.get()); - auto *disk_label_property_index = - static_cast(storage_->indices_.label_property_index_.get()); +[[nodiscard]] utils::BasicResult DiskStorage::FlushDeletedVertices( + Transaction *transaction) { + auto *disk_unique_constraints = static_cast(constraints_.unique_constraints_.get()); + auto *disk_label_index = static_cast(indices_.label_index_.get()); + auto *disk_label_property_index = static_cast(indices_.label_property_index_.get()); - for (const auto &[vertex_gid, serialized_vertex_to_delete] : transaction_.vertices_to_delete_) { - if (!DeleteVertexFromDisk(vertex_gid, serialized_vertex_to_delete) || - !disk_unique_constraints->ClearDeletedVertex(vertex_gid, *commit_timestamp_) || - !disk_label_index->ClearDeletedVertex(vertex_gid, *commit_timestamp_) || - !disk_label_property_index->ClearDeletedVertex(vertex_gid, *commit_timestamp_)) { + auto commit_ts = transaction->commit_timestamp->load(std::memory_order_relaxed); + for (const auto &[vertex_gid, serialized_vertex_to_delete] : transaction->vertices_to_delete_) { + if (!DeleteVertexFromDisk(transaction, vertex_gid, serialized_vertex_to_delete) || + !disk_unique_constraints->ClearDeletedVertex(vertex_gid, commit_ts) || + !disk_label_index->ClearDeletedVertex(vertex_gid, commit_ts) || + !disk_label_property_index->ClearDeletedVertex(vertex_gid, commit_ts)) { return StorageManipulationError{SerializationError{}}; } } @@ -1352,10 +1187,11 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( return {}; } -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushDeletedEdges() { - for (const auto &[edge_to_delete, vertices] : transaction_.edges_to_delete_) { +[[nodiscard]] utils::BasicResult DiskStorage::FlushDeletedEdges( + Transaction *transaction) { + for (const auto &[edge_to_delete, vertices] : transaction->edges_to_delete_) { const auto &[src_vertex_id, dst_vertex_id] = vertices; - if (!DeleteEdgeFromDisk(edge_to_delete, src_vertex_id, dst_vertex_id)) { + if (!DeleteEdgeFromDisk(transaction, edge_to_delete, src_vertex_id, dst_vertex_id)) { return StorageManipulationError{SerializationError{}}; } } @@ -1367,21 +1203,21 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( /// std::map /// std::map /// Here we also do flushing of too many things, we don't need to serialize edges in read-only txn, check that... -[[nodiscard]] utils::BasicResult DiskStorage::DiskAccessor::FlushModifiedEdges( - const auto &edge_acc) { - auto *disk_storage = static_cast(storage_); - for (const auto &modified_edge : transaction_.modified_edges_) { - const std::string edge_gid = utils::SerializeIdType(modified_edge.first); +[[nodiscard]] utils::BasicResult DiskStorage::FlushModifiedEdges( + Transaction *transaction, const auto &edge_acc) { + for (const auto &modified_edge : transaction->modified_edges_) { + const std::string edge_gid = modified_edge.first.ToString(); const Delta::Action root_action = modified_edge.second.delta_action; - const auto src_vertex_gid = utils::SerializeIdType(modified_edge.second.src_vertex_gid); - const auto dst_vertex_gid = utils::SerializeIdType(modified_edge.second.dest_vertex_gid); + const auto src_vertex_gid = modified_edge.second.src_vertex_gid.ToString(); + const auto dst_vertex_gid = modified_edge.second.dest_vertex_gid.ToString(); - if (!storage_->config_.items.properties_on_edges) { + if (!config_.items.properties_on_edges) { /// If the object was created then flush it, otherwise since properties on edges are false /// edge wasn't modified for sure. if (root_action == Delta::Action::DELETE_OBJECT && - !WriteEdgeToEdgeColumnFamily(edge_gid, utils::SerializeEdgeAsValue(src_vertex_gid, dst_vertex_gid, - modified_edge.second.edge_type_id))) { + !WriteEdgeToEdgeColumnFamily( + transaction, edge_gid, + utils::SerializeEdgeAsValue(src_vertex_gid, dst_vertex_gid, modified_edge.second.edge_type_id))) { return StorageManipulationError{SerializationError{}}; } } else { @@ -1389,7 +1225,8 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( // If the edge was deserialized, only properties can be modified -> key stays the same as when deserialized // so we can delete it. // This is done to avoid storing multiple versions of the same data. - if (root_action == Delta::Action::DELETE_DESERIALIZED_OBJECT && !DeleteEdgeFromEdgeColumnFamily(edge_gid)) { + if (root_action == Delta::Action::DELETE_DESERIALIZED_OBJECT && + !DeleteEdgeFromEdgeColumnFamily(transaction, edge_gid)) { return StorageManipulationError{SerializationError{}}; } @@ -1399,14 +1236,14 @@ DiskStorage::DiskAccessor::CheckVertexConstraintsBeforeCommit( /// TODO: (andi) I think this is not wrong but it would be better to use AtomicWrites across column families. if (!WriteEdgeToEdgeColumnFamily( - edge_gid, + transaction, edge_gid, utils::SerializeEdgeAsValue(src_vertex_gid, dst_vertex_gid, modified_edge.second.edge_type_id, &*edge))) { return StorageManipulationError{SerializationError{}}; } } if (root_action == Delta::Action::DELETE_OBJECT && - (!WriteEdgeToConnectivityIndex(src_vertex_gid, edge_gid, disk_storage->kvstore_->out_edges_chandle, "OUT") || - !WriteEdgeToConnectivityIndex(dst_vertex_gid, edge_gid, disk_storage->kvstore_->in_edges_chandle, "IN"))) { + (!WriteEdgeToConnectivityIndex(transaction, src_vertex_gid, edge_gid, kvstore_->out_edges_chandle, "OUT") || + !WriteEdgeToConnectivityIndex(transaction, dst_vertex_gid, edge_gid, kvstore_->in_edges_chandle, "IN"))) { return StorageManipulationError{SerializationError{}}; } } @@ -1444,7 +1281,6 @@ VertexAccessor DiskStorage::CreateVertexFromDisk(Transaction *transaction, utils } std::optional DiskStorage::FindVertex(storage::Gid gid, Transaction *transaction, View view) { - /// TODO: (andi) Abstract to a method GetActiveAccessor auto acc = edge_import_status_ == EdgeImportMode::ACTIVE ? edge_import_mode_cache_->AccessToVertices() : transaction->vertices_.access(); auto vertex_it = acc.find(gid); @@ -1467,11 +1303,10 @@ std::optional DiskStorage::FindVertex(storage::Gid gid, Transact transaction->disk_transaction_->GetIterator(read_opts, kvstore_->vertex_chandle)); for (it->SeekToFirst(); it->Valid(); it->Next()) { std::string key = it->key().ToString(); - /// TODO: (andi) Here no need to create new string, string_view would suffice if (Gid::FromString(utils::ExtractGidFromKey(key)) == gid) { // We should pass it->timestamp().ToString() instead of "0" // This is hack until RocksDB will support timestamp() in WBWI iterator - return LoadVertexToMainMemoryCache(transaction, key, it->value().ToString(), deserializeTimestamp); + return LoadVertexToMainMemoryCache(transaction, key, it->value().ToString(), kDeserializeTimestamp); } } return std::nullopt; @@ -1508,10 +1343,8 @@ std::optional DiskStorage::CreateEdgeFromDisk(const VertexAccessor ModifiedEdgeInfo modified_edge(Delta::Action::DELETE_DESERIALIZED_OBJECT, from_vertex->gid, to_vertex->gid, edge_type, edge); if (transaction->AddModifiedEdge(gid, modified_edge)) { - spdlog::trace("Edge {} added to out edges of vertex with gid {}", utils::SerializeIdType(gid), - from_vertex->gid.AsUint()); - spdlog::trace("Edge {} added to in edges of vertex with gid {}", utils::SerializeIdType(gid), - to_vertex->gid.AsUint()); + spdlog::trace("Edge {} added to out edges of vertex with gid {}", gid.ToString(), from_vertex->gid.AsUint()); + spdlog::trace("Edge {} added to in edges of vertex with gid {}", gid.ToString(), to_vertex->gid.AsUint()); from_vertex->out_edges.emplace_back(edge_type, to_vertex, edge); to_vertex->in_edges.emplace_back(edge_type, from_vertex, edge); transaction->manyDeltasCache.Invalidate(from_vertex, edge_type, EdgeDirection::OUT); @@ -1528,7 +1361,7 @@ std::vector DiskStorage::OutEdges(const VertexAccessor *src_vertex /// Check whether the vertex is deleted in the current tx only if View::NEW is requested if (view == View::NEW && src_vertex->vertex_->deleted) return {}; - const std::string src_vertex_gid = utils::SerializeIdType(src_vertex->Gid()); + const std::string src_vertex_gid = src_vertex->Gid().ToString(); rocksdb::ReadOptions ro; std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); @@ -1555,7 +1388,7 @@ std::vector DiskStorage::OutEdges(const VertexAccessor *src_vertex if (!edge_types.empty() && !utils::Contains(edge_types, edge_type_id)) continue; auto edge_gid = Gid::FromString(edge_gid_str); - auto properties_str = config_.items.properties_on_edges ? utils::GetViewOfFourthPartOfSplit(edge_val_str, '|') : ""; + auto properties_str = config_.items.properties_on_edges ? utils::GetPropertiesFromEdgeValue(edge_val_str) : ""; const auto edge = std::invoke([this, destination, &edge_val_str, transaction, view, src_vertex, edge_type_id, edge_gid, &properties_str, &edge_gid_str]() { @@ -1568,7 +1401,7 @@ std::vector DiskStorage::OutEdges(const VertexAccessor *src_vertex return std::optional{}; return CreateEdgeFromDisk(src_vertex, &*dst_vertex, transaction, edge_type_id, edge_gid, properties_str, - edge_gid_str, deserializeTimestamp); + edge_gid_str, kDeserializeTimestamp); } /// This is needed for filtering /// Second check not needed I think @@ -1577,7 +1410,7 @@ std::vector DiskStorage::OutEdges(const VertexAccessor *src_vertex } return CreateEdgeFromDisk(src_vertex, destination, transaction, edge_type_id, edge_gid, properties_str, - edge_gid_str, deserializeTimestamp); + edge_gid_str, kDeserializeTimestamp); }); if (edge.has_value()) result.emplace_back(*edge); } @@ -1591,7 +1424,7 @@ std::vector DiskStorage::InEdges(const VertexAccessor *dst_vertex, /// Check whether the vertex is deleted in the current tx only if View::NEW is requested if (view == View::NEW && dst_vertex->vertex_->deleted) return {}; - const std::string dst_vertex_gid = utils::SerializeIdType(dst_vertex->Gid()); + const std::string dst_vertex_gid = dst_vertex->Gid().ToString(); rocksdb::ReadOptions ro; std::string strTs = utils::StringTimestamp(transaction->start_timestamp); rocksdb::Slice ts(strTs); @@ -1618,7 +1451,7 @@ std::vector DiskStorage::InEdges(const VertexAccessor *dst_vertex, if (!edge_types.empty() && !utils::Contains(edge_types, edge_type_id)) continue; auto edge_gid = Gid::FromString(edge_gid_str); - auto properties_str = utils::GetViewOfFourthPartOfSplit(edge_val_str, '|'); + auto properties_str = utils::GetPropertiesFromEdgeValue(edge_val_str); const auto edge = std::invoke([this, source, &edge_val_str, transaction, view, dst_vertex, edge_type_id, edge_gid, &properties_str, &edge_gid_str]() { @@ -1631,14 +1464,14 @@ std::vector DiskStorage::InEdges(const VertexAccessor *dst_vertex, return std::optional{}; return CreateEdgeFromDisk(&*src_vertex, dst_vertex, transaction, edge_type_id, edge_gid, properties_str, - edge_gid_str, deserializeTimestamp); + edge_gid_str, kDeserializeTimestamp); } /// TODO: (andi) 2nd check not needed I think if (src_vertex_gid != source->Gid() || source->vertex_->deleted) { return std::optional{}; } return CreateEdgeFromDisk(source, dst_vertex, transaction, edge_type_id, edge_gid, properties_str, edge_gid_str, - deserializeTimestamp); + kDeserializeTimestamp); }); if (edge.has_value()) result.emplace_back(*edge); @@ -1713,26 +1546,26 @@ utils::BasicResult DiskStorage::DiskAccessor::Co for (const auto &md_delta : transaction_.md_deltas) { switch (md_delta.action) { case MetadataDelta::Action::LABEL_INDEX_CREATE: { - if (!disk_storage->PersistLabelIndexCreation(md_delta.label)) { + if (!disk_storage->durable_metadata_.PersistLabelIndexCreation(md_delta.label)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::LABEL_PROPERTY_INDEX_CREATE: { const auto &info = md_delta.label_property; - if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintCreation(info.label, info.property, - label_property_index_str)) { + if (!disk_storage->durable_metadata_.PersistLabelPropertyIndexAndExistenceConstraintCreation( + info.label, info.property, kLabelPropertyIndexStr)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::LABEL_INDEX_DROP: { - if (!disk_storage->PersistLabelIndexDeletion(md_delta.label)) { + if (!disk_storage->durable_metadata_.PersistLabelIndexDeletion(md_delta.label)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::LABEL_PROPERTY_INDEX_DROP: { const auto &info = md_delta.label_property; - if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintDeletion(info.label, info.property, - label_property_index_str)) { + if (!disk_storage->durable_metadata_.PersistLabelPropertyIndexAndExistenceConstraintDeletion( + info.label, info.property, kLabelPropertyIndexStr)) { return StorageManipulationError{PersistenceError{}}; } } break; @@ -1750,27 +1583,27 @@ utils::BasicResult DiskStorage::DiskAccessor::Co } break; case MetadataDelta::Action::EXISTENCE_CONSTRAINT_CREATE: { const auto &info = md_delta.label_property; - if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintCreation(info.label, info.property, - existence_constraints_str)) { + if (!disk_storage->durable_metadata_.PersistLabelPropertyIndexAndExistenceConstraintCreation( + info.label, info.property, kExistenceConstraintsStr)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::EXISTENCE_CONSTRAINT_DROP: { const auto &info = md_delta.label_property; - if (!disk_storage->PersistLabelPropertyIndexAndExistenceConstraintDeletion(info.label, info.property, - existence_constraints_str)) { + if (!disk_storage->durable_metadata_.PersistLabelPropertyIndexAndExistenceConstraintDeletion( + info.label, info.property, kExistenceConstraintsStr)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::UNIQUE_CONSTRAINT_CREATE: { const auto &info = md_delta.label_properties; - if (!disk_storage->PersistUniqueConstraintCreation(info.label, info.properties)) { + if (!disk_storage->durable_metadata_.PersistUniqueConstraintCreation(info.label, info.properties)) { return StorageManipulationError{PersistenceError{}}; } } break; case MetadataDelta::Action::UNIQUE_CONSTRAINT_DROP: { const auto &info = md_delta.label_properties; - if (!disk_storage->PersistUniqueConstraintDeletion(info.label, info.properties)) { + if (!disk_storage->durable_metadata_.PersistUniqueConstraintDeletion(info.label, info.properties)) { return StorageManipulationError{PersistenceError{}}; } } break; @@ -1787,43 +1620,47 @@ utils::BasicResult DiskStorage::DiskAccessor::Co transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); if (edge_import_mode_active) { - if (auto res = FlushModifiedEdges(disk_storage->edge_import_mode_cache_->AccessToEdges()); res.HasError()) { + if (auto res = + disk_storage->FlushModifiedEdges(&transaction_, disk_storage->edge_import_mode_cache_->AccessToEdges()); + res.HasError()) { Abort(); return res; } - if (auto del_edges_res = FlushDeletedEdges(); del_edges_res.HasError()) { + if (auto del_edges_res = disk_storage->FlushDeletedEdges(&transaction_); del_edges_res.HasError()) { Abort(); return del_edges_res.GetError(); } } else { std::vector> unique_storage; - if (auto vertices_flush_res = FlushVertices(transaction_.vertices_.access(), unique_storage); + if (auto vertices_flush_res = + disk_storage->FlushVertices(&transaction_, transaction_.vertices_.access(), unique_storage); vertices_flush_res.HasError()) { Abort(); return vertices_flush_res.GetError(); } - if (auto del_vertices_res = FlushDeletedVertices(); del_vertices_res.HasError()) { + if (auto del_vertices_res = disk_storage->FlushDeletedVertices(&transaction_); del_vertices_res.HasError()) { Abort(); return del_vertices_res.GetError(); } - if (auto modified_edges_res = FlushModifiedEdges(transaction_.edges_.access()); modified_edges_res.HasError()) { + if (auto modified_edges_res = disk_storage->FlushModifiedEdges(&transaction_, transaction_.edges_.access()); + modified_edges_res.HasError()) { Abort(); return modified_edges_res.GetError(); } - if (auto del_edges_res = FlushDeletedEdges(); del_edges_res.HasError()) { + if (auto del_edges_res = disk_storage->FlushDeletedEdges(&transaction_); del_edges_res.HasError()) { Abort(); return del_edges_res.GetError(); } - if (auto clear_dangling_res = ClearDanglingVertices(); clear_dangling_res.HasError()) { + if (auto clear_dangling_res = disk_storage->ClearDanglingVertices(&transaction_); clear_dangling_res.HasError()) { Abort(); return clear_dangling_res.GetError(); } - if (auto index_flush_res = FlushIndexCache(); index_flush_res.HasError()) { + if (auto index_flush_res = disk_storage->FlushIndexCache(&transaction_); index_flush_res.HasError()) { Abort(); return index_flush_res.GetError(); } @@ -1857,7 +1694,7 @@ std::vector> DiskStorage::SerializeVerticesF ro.timestamp = &ts; auto it = std::unique_ptr(kvstore_->db_->NewIterator(ro, kvstore_->vertex_chandle)); - const std::string serialized_label = utils::SerializeIdType(label); + const std::string serialized_label = label.ToString(); for (it->SeekToFirst(); it->Valid(); it->Next()) { const std::string key_str = it->key().ToString(); if (const std::vector labels_str = utils::ExtractLabelsFromMainDiskStorage(key_str); @@ -1865,8 +1702,7 @@ std::vector> DiskStorage::SerializeVerticesF std::vector labels = utils::DeserializeLabelsFromMainDiskStorage(key_str); PropertyStore property_store = utils::DeserializePropertiesFromMainDiskStorage(it->value().ToStringView()); vertices_to_be_indexed.emplace_back( - utils::SerializeVertexAsKeyForLabelIndex(utils::SerializeIdType(label), - utils::ExtractGidFromMainDiskStorage(key_str)), + utils::SerializeVertexAsKeyForLabelIndex(label.ToString(), utils::ExtractGidFromMainDiskStorage(key_str)), utils::SerializeVertexAsValueForLabelIndex(label, labels, property_store)); } } @@ -1883,7 +1719,7 @@ std::vector> DiskStorage::SerializeVerticesF ro.timestamp = &ts; auto it = std::unique_ptr(kvstore_->db_->NewIterator(ro, kvstore_->vertex_chandle)); - const std::string serialized_label = utils::SerializeIdType(label); + const std::string serialized_label = label.ToString(); for (it->SeekToFirst(); it->Valid(); it->Next()) { const std::string key_str = it->key().ToString(); PropertyStore property_store = utils::DeserializePropertiesFromMainDiskStorage(it->value().ToString()); @@ -1891,8 +1727,7 @@ std::vector> DiskStorage::SerializeVerticesF utils::Contains(labels_str, serialized_label) && property_store.HasProperty(property)) { std::vector labels = utils::DeserializeLabelsFromMainDiskStorage(key_str); vertices_to_be_indexed.emplace_back( - utils::SerializeVertexAsKeyForLabelPropertyIndex(utils::SerializeIdType(label), - utils::SerializeIdType(property), + utils::SerializeVertexAsKeyForLabelPropertyIndex(label.ToString(), property.ToString(), utils::ExtractGidFromMainDiskStorage(key_str)), utils::SerializeVertexAsValueForLabelPropertyIndex(label, labels, property_store)); } @@ -1954,7 +1789,6 @@ void DiskStorage::DiskAccessor::UpdateObjectsCountOnAbort() { } } -/// TODO: what to do with all that? void DiskStorage::DiskAccessor::Abort() { MG_ASSERT(is_transaction_active_, "The transaction is already terminated!"); // NOTE: On abort we need to delete disk transaction because after storage remove we couldn't remove @@ -1970,12 +1804,10 @@ void DiskStorage::DiskAccessor::Abort() { } void DiskStorage::DiskAccessor::FinalizeTransaction() { - /// TODO: (andi) Check the login in InMemoryStorage. if (commit_timestamp_) { auto *disk_storage = static_cast(storage_); - bool edge_import_mode_active = disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE; - if (edge_import_mode_active) { + if (disk_storage->edge_import_status_ == EdgeImportMode::ACTIVE) { auto &committed_transactions = disk_storage->edge_import_mode_cache_->GetCommittedTransactions(); committed_transactions.WithLock( [&](auto &committed_txs) { committed_txs.emplace_back(std::move(transaction_)); }); @@ -2113,7 +1945,6 @@ Transaction DiskStorage::CreateTransaction(IsolationLevel isolation_level, Stora { std::lock_guard guard(engine_lock_); transaction_id = transaction_id_++; - /// TODO: when we introduce replication to the disk storage, take care of start_timestamp start_timestamp = timestamp_++; edge_import_mode_active = edge_import_status_ == EdgeImportMode::ACTIVE; } diff --git a/src/storage/v2/disk/storage.hpp b/src/storage/v2/disk/storage.hpp index dc89a9641..ddaeaac49 100644 --- a/src/storage/v2/disk/storage.hpp +++ b/src/storage/v2/disk/storage.hpp @@ -13,6 +13,7 @@ #include "kvstore/kvstore.hpp" #include "storage/v2/constraints/constraint_violation.hpp" +#include "storage/v2/disk/durable_metadata.hpp" #include "storage/v2/disk/edge_import_mode_cache.hpp" #include "storage/v2/disk/rocksdb_storage.hpp" #include "storage/v2/edge_import_mode.hpp" @@ -46,55 +47,6 @@ class DiskStorage final : public Storage { explicit DiskAccessor(auto tag, DiskStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode); - /// TODO: const methods? - void LoadVerticesToMainMemoryCache(); - - void LoadVerticesFromMainStorageToEdgeImportCache(); - - void HandleMainLoadingForEdgeImportCache(); - - void LoadVerticesFromLabelIndexStorageToEdgeImportCache(LabelId label); - - void HandleLoadingLabelForEdgeImportCache(LabelId label); - - void LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(LabelId label, PropertyId property); - - void HandleLoadingLabelPropertyForEdgeImportCache(LabelId label, PropertyId property); - - std::unordered_set MergeVerticesFromMainCacheWithLabelIndexCache(LabelId label, View view, - std::list &index_deltas, - utils::SkipList *indexed_vertices); - - void LoadVerticesFromDiskLabelIndex(LabelId label, const std::unordered_set &gids, - std::list &index_deltas, utils::SkipList *indexed_vertices); - - std::unordered_set MergeVerticesFromMainCacheWithLabelPropertyIndexCache( - LabelId label, PropertyId property, View view, std::list &index_deltas, - utils::SkipList *indexed_vertices, const auto &label_property_filter); - - void LoadVerticesFromDiskLabelPropertyIndex(LabelId label, PropertyId property, - const std::unordered_set &gids, - std::list &index_deltas, - utils::SkipList *indexed_vertices, - const auto &label_property_filter); - - void LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup(LabelId label, PropertyId property, - const std::unordered_set &gids, - const PropertyValue &value, - std::list &index_deltas, - utils::SkipList *indexed_vertices); - - std::unordered_set MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( - LabelId label, PropertyId property, View view, const std::optional> &lower_bound, - const std::optional> &upper_bound, std::list &index_deltas, - utils::SkipList *indexed_vertices); - - void LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch( - LabelId label, PropertyId property, const std::unordered_set &gids, - const std::optional> &lower_bound, - const std::optional> &upper_bound, std::list &index_deltas, - utils::SkipList *indexed_vertices); - public: DiskAccessor(const DiskAccessor &) = delete; DiskAccessor &operator=(const DiskAccessor &) = delete; @@ -197,17 +149,6 @@ class DiskStorage final : public Storage { void FinalizeTransaction() override; - std::optional LoadVertexToLabelIndexCache( - const std::string &key, const std::string &value, Delta *index_delta, - utils::SkipList::Accessor index_accessor); - - std::optional LoadVertexToLabelPropertyIndexCache( - const std::string &key, const std::string &value, Delta *index_delta, - utils::SkipList::Accessor index_accessor); - - std::optional DeserializeEdge(const rocksdb::Slice &key, const rocksdb::Slice &value, - const rocksdb::Slice &ts); - utils::BasicResult CreateIndex(LabelId label) override; utils::BasicResult CreateIndex(LabelId label, PropertyId property) override; @@ -227,52 +168,89 @@ class DiskStorage final : public Storage { UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label, const std::set &properties) override; - - private: - /// Flushes vertices and edges to the disk with the commit timestamp. - /// At the time of calling, the commit_timestamp_ must already exist. - /// After this method, the vertex and edge caches are cleared. - - [[nodiscard]] utils::BasicResult FlushIndexCache(); - - [[nodiscard]] utils::BasicResult FlushDeletedVertices(); - - [[nodiscard]] utils::BasicResult FlushDeletedEdges(); - - [[nodiscard]] utils::BasicResult FlushVertices( - const auto &vertex_acc, std::vector> &unique_storage); - - [[nodiscard]] utils::BasicResult FlushModifiedEdges(const auto &edge_acc); - - [[nodiscard]] utils::BasicResult ClearDanglingVertices(); - - [[nodiscard]] utils::BasicResult CheckVertexConstraintsBeforeCommit( - const Vertex &vertex, std::vector> &unique_storage) const; - - bool WriteVertexToVertexColumnFamily(const Vertex &vertex); - bool WriteEdgeToEdgeColumnFamily(const std::string &serialized_edge_key, const std::string &serialized_edge_value); - - bool WriteEdgeToConnectivityIndex(const std::string &vertex_gid, const std::string &edge_gid, - rocksdb::ColumnFamilyHandle *handle, std::string mode); - - bool DeleteVertexFromDisk(const std::string &vertex_gid, const std::string &vertex); - - bool DeleteEdgeFromEdgeColumnFamily(const std::string &edge_gid); - bool DeleteEdgeFromDisk(const std::string &edge_gid, const std::string &src_vertex_gid, - const std::string &dst_vertex_gid); - bool DeleteEdgeFromConnectivityIndex(const std::string &vertex_gid, const std::string &edge_gid, - rocksdb::ColumnFamilyHandle *handle, std::string mode); }; std::unique_ptr Access(std::optional override_isolation_level) override; std::unique_ptr UniqueAccess(std::optional override_isolation_level) override; - /// TODO: (andi) Methods working with rocksdb are scattered around DiskStorage and DiskStorage::DiskAccessor - /// Two options: - /// 1. move everything under DiskStorage level - /// 2. propagate DiskStorage::DiskAccessor to vertex and edge accessor. - /// Out of scope of this PR + /// Flushing methods + [[nodiscard]] utils::BasicResult FlushIndexCache(Transaction *transaction); + + [[nodiscard]] utils::BasicResult FlushVertices( + Transaction *transaction, const auto &vertex_acc, std::vector> &unique_storage); + + [[nodiscard]] utils::BasicResult CheckVertexConstraintsBeforeCommit( + const Vertex &vertex, std::vector> &unique_storage) const; + + [[nodiscard]] utils::BasicResult FlushDeletedVertices(Transaction *transaction); + [[nodiscard]] utils::BasicResult FlushDeletedEdges(Transaction *transaction); + [[nodiscard]] utils::BasicResult FlushModifiedEdges(Transaction *transaction, + const auto &edge_acc); + [[nodiscard]] utils::BasicResult ClearDanglingVertices(Transaction *transaction); + + /// Writing methods + bool WriteVertexToVertexColumnFamily(Transaction *transaction, const Vertex &vertex); + bool WriteEdgeToEdgeColumnFamily(Transaction *transaction, const std::string &serialized_edge_key, + const std::string &serialized_edge_value); + bool WriteEdgeToConnectivityIndex(Transaction *transaction, const std::string &vertex_gid, + const std::string &edge_gid, rocksdb::ColumnFamilyHandle *handle, std::string mode); + bool DeleteVertexFromDisk(Transaction *transaction, const std::string &vertex_gid, const std::string &vertex); + bool DeleteEdgeFromEdgeColumnFamily(Transaction *transaction, const std::string &edge_gid); + bool DeleteEdgeFromDisk(Transaction *transaction, const std::string &edge_gid, const std::string &src_vertex_gid, + const std::string &dst_vertex_gid); + bool DeleteEdgeFromConnectivityIndex(Transaction *transaction, const std::string &vertex_gid, + const std::string &edge_gid, rocksdb::ColumnFamilyHandle *handle, + std::string mode); + + void LoadVerticesToMainMemoryCache(Transaction *transaction); + + /// Edge import mode methods + void LoadVerticesFromMainStorageToEdgeImportCache(Transaction *transaction); + void HandleMainLoadingForEdgeImportCache(Transaction *transaction); + + /// Indices methods + /// Label-index + void LoadVerticesFromLabelIndexStorageToEdgeImportCache(Transaction *transaction, LabelId label); + void HandleLoadingLabelForEdgeImportCache(Transaction *transaction, LabelId label); + void LoadVerticesFromDiskLabelIndex(Transaction *transaction, LabelId label, + const std::unordered_set &gids, std::list &index_deltas, + utils::SkipList *indexed_vertices); + std::optional LoadVertexToLabelIndexCache( + Transaction *transaction, const std::string &key, const std::string &value, Delta *index_delta, + utils::SkipList::Accessor index_accessor); + std::unordered_set MergeVerticesFromMainCacheWithLabelIndexCache(Transaction *transaction, LabelId label, + View view, std::list &index_deltas, + utils::SkipList *indexed_vertices); + + /// Label-property-index + void LoadVerticesFromLabelPropertyIndexStorageToEdgeImportCache(Transaction *transaction, LabelId label, + PropertyId property); + void HandleLoadingLabelPropertyForEdgeImportCache(Transaction *transaction, LabelId label, PropertyId property); + std::unordered_set MergeVerticesFromMainCacheWithLabelPropertyIndexCache( + Transaction *transaction, LabelId label, PropertyId property, View view, std::list &index_deltas, + utils::SkipList *indexed_vertices, const auto &label_property_filter); + void LoadVerticesFromDiskLabelPropertyIndex(Transaction *transaction, LabelId label, PropertyId property, + const std::unordered_set &gids, + std::list &index_deltas, utils::SkipList *indexed_vertices, + const auto &label_property_filter); + std::optional LoadVertexToLabelPropertyIndexCache( + Transaction *transaction, const std::string &key, const std::string &value, Delta *index_delta, + utils::SkipList::Accessor index_accessor); + void LoadVerticesFromDiskLabelPropertyIndexWithPointValueLookup( + Transaction *transaction, LabelId label, PropertyId property, const std::unordered_set &gids, + const PropertyValue &value, std::list &index_deltas, utils::SkipList *indexed_vertices); + std::unordered_set MergeVerticesFromMainCacheWithLabelPropertyIndexCacheForIntervalSearch( + Transaction *transaction, LabelId label, PropertyId property, View view, + const std::optional> &lower_bound, + const std::optional> &upper_bound, std::list &index_deltas, + utils::SkipList *indexed_vertices); + void LoadVerticesFromDiskLabelPropertyIndexForIntervalSearch( + Transaction *transaction, LabelId label, PropertyId property, const std::unordered_set &gids, + const std::optional> &lower_bound, + const std::optional> &upper_bound, std::list &index_deltas, + utils::SkipList *indexed_vertices); + VertexAccessor CreateVertexFromDisk(Transaction *transaction, utils::SkipList::Accessor &accessor, storage::Gid gid, std::vector label_ids, PropertyStore properties, Delta *delta); @@ -280,7 +258,6 @@ class DiskStorage final : public Storage { std::optional LoadVertexToMainMemoryCache(Transaction *transaction, const std::string &key, const std::string &value, std::string &&ts); - /// TODO: (andi) I don't think View is necessary std::optional FindVertex(Gid gid, Transaction *transaction, View view); std::optional CreateEdgeFromDisk(const VertexAccessor *from, const VertexAccessor *to, @@ -288,12 +265,10 @@ class DiskStorage final : public Storage { std::string_view properties, const std::string &old_disk_key, std::string &&ts); - /// TODO: (andi) Maybe const std::vector OutEdges(const VertexAccessor *src_vertex, const std::vector &possible_edge_types, const VertexAccessor *destination, Transaction *transaction, View view); - /// TODO: (andi) Maybe const std::vector InEdges(const VertexAccessor *dst_vertex, const std::vector &possible_edge_types, const VertexAccessor *source, Transaction *transaction, View view); @@ -307,39 +282,10 @@ class DiskStorage final : public Storage { EdgeImportMode GetEdgeImportMode() const; private: - void LoadIndexInfoIfExists() const; - - /// TODO (andi): Maybe good to separate these methods and durability kvstore into a separate class - bool PersistLabelIndexCreation(LabelId label) const; - - bool PersistLabelIndexDeletion(LabelId label) const; - - void LoadLabelIndexInfoIfExists() const; - - bool PersistLabelPropertyIndexAndExistenceConstraintCreation(LabelId label, PropertyId property, - const char *key) const; - - bool PersistLabelPropertyIndexAndExistenceConstraintDeletion(LabelId label, PropertyId property, - const char *key) const; - - void LoadLabelPropertyIndexInfoIfExists() const; - - void LoadConstraintsInfoIfExists() const; - - void LoadExistenceConstraintInfoIfExists() const; - - bool PersistUniqueConstraintCreation(LabelId label, const std::set &properties) const; - - bool PersistUniqueConstraintDeletion(LabelId label, const std::set &properties) const; - - void LoadUniqueConstraintInfoIfExists() const; + void LoadPersistingMetadataInfo(); uint64_t GetDiskSpaceUsage() const; - void LoadTimestampIfExists(); - - void LoadVertexAndEdgeCountIfExists(); - [[nodiscard]] std::optional CheckExistingVerticesBeforeCreatingExistenceConstraint( LabelId label, PropertyId property) const; @@ -355,29 +301,26 @@ class DiskStorage final : public Storage { void FreeMemory(std::unique_lock /*lock*/) override {} - void PrepareForNewEpoch(std::string prev_epoch) override { + void PrepareForNewEpoch(std::string /*prev_epoch*/) override { throw utils::BasicException("Disk storage mode does not support replication."); } uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); - EdgeImportMode edge_import_status_{EdgeImportMode::INACTIVE}; - std::unique_ptr edge_import_mode_cache_{nullptr}; - - auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig &config) + auto CreateReplicationClient(const memgraph::replication::ReplicationClientConfig & /*config*/) -> std::unique_ptr override { throw utils::BasicException("Disk storage mode does not support replication."); } - auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig &config) + auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig & /*config*/) -> std::unique_ptr override { throw utils::BasicException("Disk storage mode does not support replication."); } - private: std::unique_ptr kvstore_; - std::unique_ptr durability_kvstore_; - + DurableMetadata durable_metadata_; + EdgeImportMode edge_import_status_{EdgeImportMode::INACTIVE}; + std::unique_ptr edge_import_mode_cache_{nullptr}; std::atomic vertex_count_{0}; }; diff --git a/src/storage/v2/disk/unique_constraints.cpp b/src/storage/v2/disk/unique_constraints.cpp index 9519e4357..e0ec3cf82 100644 --- a/src/storage/v2/disk/unique_constraints.cpp +++ b/src/storage/v2/disk/unique_constraints.cpp @@ -33,7 +33,7 @@ bool IsVertexUnderConstraint(const Vertex &vertex, const LabelId &constraint_lab bool IsDifferentVertexWithSameConstraintLabel(const std::string &key, const Gid gid, const LabelId constraint_label) { const std::vector vertex_parts = utils::Split(key, "|"); - if (std::string local_gid = vertex_parts[1]; local_gid == utils::SerializeIdType(gid)) { + if (std::string local_gid = vertex_parts[1]; local_gid == gid.ToString()) { return false; } return utils::DeserializeConstraintLabelFromUniqueConstraintStorage(key) == constraint_label; @@ -45,7 +45,7 @@ bool IsDifferentVertexWithSameConstraintLabel(const std::string &key, const Gid for (const auto &[vertex_gid, constraints] : transaction_entries) { for (const auto &[constraint_label, constraint_properties] : constraints) { auto key_to_delete = utils::SerializeVertexAsKeyForUniqueConstraint(constraint_label, constraint_properties, - utils::SerializeIdType(vertex_gid)); + vertex_gid.ToString()); if (auto status = disk_transaction.Delete(key_to_delete); !status.ok()) { return false; } @@ -217,8 +217,7 @@ bool DiskUniqueConstraints::SyncVertexToUniqueConstraintsStorage(const Vertex &v kvstore_->db_->BeginTransaction(rocksdb::WriteOptions(), rocksdb::TransactionOptions())); if (auto maybe_old_disk_key = utils::GetOldDiskKeyOrNull(vertex.delta); maybe_old_disk_key.has_value()) { - spdlog::trace("Found old disk key {} for vertex {}", maybe_old_disk_key.value(), - utils::SerializeIdType(vertex.gid)); + spdlog::trace("Found old disk key {} for vertex {}", maybe_old_disk_key.value(), vertex.gid.ToString()); if (auto status = disk_transaction->Delete(maybe_old_disk_key.value()); !status.ok()) { return false; } @@ -227,7 +226,7 @@ bool DiskUniqueConstraints::SyncVertexToUniqueConstraintsStorage(const Vertex &v for (const auto &[constraint_label, constraint_properties] : constraints_) { if (IsVertexUnderConstraint(vertex, constraint_label, constraint_properties)) { auto key = utils::SerializeVertexAsKeyForUniqueConstraint(constraint_label, constraint_properties, - utils::SerializeIdType(vertex.gid)); + vertex.gid.ToString()); auto value = utils::SerializeVertexAsValueForUniqueConstraint(constraint_label, vertex.labels, vertex.properties); if (!disk_transaction->Put(key, value).ok()) { return false; diff --git a/src/storage/v2/id_types.hpp b/src/storage/v2/id_types.hpp index 7579e4b66..2f9577246 100644 --- a/src/storage/v2/id_types.hpp +++ b/src/storage/v2/id_types.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ namespace memgraph::storage { uint64_t AsUint() const { return id_; } \ int64_t AsInt() const { return utils::MemcpyCast(id_); } \ static name FromString(std::string_view id) { return name{utils::ParseStringToUint64(id)}; } \ + std::string ToString() const { return std::to_string(id_); } \ \ private: \ uint64_t id_; \ diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 913c62f7c..81c86c2ae 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -27,13 +27,6 @@ #include "utils/typeinfo.hpp" #include "utils/uuid.hpp" -namespace memgraph::metrics { -extern const Event SnapshotCreationLatency_us; - -extern const Event ActiveLabelIndices; -extern const Event ActiveLabelPropertyIndices; -} // namespace memgraph::metrics - namespace memgraph::storage { class InMemoryStorage; @@ -59,7 +52,9 @@ Storage::Storage(Config config, StorageMode storage_mode) storage_mode_(storage_mode), indices_(config, storage_mode), constraints_(config, storage_mode), - id_(config.name) {} + id_(config.name) { + spdlog::info("Created database with {} storage mode.", StorageModeToString(storage_mode)); +} Storage::Accessor::Accessor(SharedAccess /* tag */, Storage *storage, IsolationLevel isolation_level, StorageMode storage_mode) diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 1a9d7567a..936322c21 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -301,8 +301,8 @@ class Storage { virtual auto CreateReplicationServer(const memgraph::replication::ReplicationServerConfig &config) -> std::unique_ptr = 0; - auto ReplicasInfo() { return repl_storage_state_.ReplicasInfo(); } - auto GetReplicaState(std::string_view name) -> std::optional { + auto ReplicasInfo() const { return repl_storage_state_.ReplicasInfo(); } + auto GetReplicaState(std::string_view name) const -> std::optional { return repl_storage_state_.GetReplicaState(name); } @@ -310,7 +310,6 @@ class Storage { memgraph::replication::ReplicationState repl_state_; ReplicationStorageState repl_storage_state_; - public: // Main storage lock. // Accessors take a shared lock when starting, so it is possible to block // creation of new accessors by taking a unique lock. This is used when doing diff --git a/src/storage/v2/transaction.hpp b/src/storage/v2/transaction.hpp index be4db2661..69565b902 100644 --- a/src/storage/v2/transaction.hpp +++ b/src/storage/v2/transaction.hpp @@ -84,7 +84,7 @@ struct Transaction { return modified_edges_.emplace(gid, modified_edge).second; } - void RemoveModifiedEdge(const Gid &gid) { modified_edges_.erase(gid); } + bool RemoveModifiedEdge(const Gid &gid) { return modified_edges_.erase(gid) > 0U; } uint64_t transaction_id; uint64_t start_timestamp; diff --git a/src/storage/v2/vertex_accessor.cpp b/src/storage/v2/vertex_accessor.cpp index bcb08c315..3d70a0051 100644 --- a/src/storage/v2/vertex_accessor.cpp +++ b/src/storage/v2/vertex_accessor.cpp @@ -460,8 +460,7 @@ auto VertexAccessor::BuildResultWithDisk(edge_store const &in_memory_edges, std: /// TODO: (andi) Maybe this check can be done in build_result without damaging anything else. std::erase_if(ret, [transaction = this->transaction_, view](const EdgeAccessor &edge_acc) { return !edge_acc.IsVisible(view) || !edge_acc.FromVertex().IsVisible(view) || - !edge_acc.ToVertex().IsVisible(view) || - transaction->edges_to_delete_.contains(utils::SerializeIdType(edge_acc.Gid())); + !edge_acc.ToVertex().IsVisible(view) || transaction->edges_to_delete_.contains(edge_acc.Gid().ToString()); }); std::unordered_set in_mem_edges_set; in_mem_edges_set.reserve(ret.size()); @@ -470,7 +469,7 @@ auto VertexAccessor::BuildResultWithDisk(edge_store const &in_memory_edges, std: } for (const auto &disk_edge_acc : disk_edges) { - auto const edge_gid_str = utils::SerializeIdType(disk_edge_acc.Gid()); + auto const edge_gid_str = disk_edge_acc.Gid().ToString(); if (in_mem_edges_set.contains(disk_edge_acc.Gid()) || (view == View::NEW && transaction_->edges_to_delete_.contains(edge_gid_str))) { continue; @@ -496,7 +495,6 @@ Result VertexAccessor::InEdges(View view, const std:: bool edges_modified_in_tx = !vertex_->in_edges.empty(); disk_edges = disk_storage->InEdges(this, edge_types, destination, transaction_, view); - /// DiskStorage & View::OLD if (view == View::OLD && !edges_modified_in_tx) { return EdgesVertexAccessorResult{.edges = disk_edges, .expanded_count = static_cast(disk_edges.size())}; } @@ -563,7 +561,6 @@ Result VertexAccessor::InEdges(View view, const std:: if (!exists) return Error::NONEXISTENT_OBJECT; if (deleted) return Error::DELETED_OBJECT; - /// DiskStorage & View::NEW if (transaction_->IsDiskStorage()) { return EdgesVertexAccessorResult{.edges = BuildResultWithDisk(in_edges, disk_edges, view, "IN"), .expanded_count = expanded_count}; @@ -587,7 +584,7 @@ Result VertexAccessor::OutEdges(View view, const std: bool edges_modified_in_tx = !vertex_->out_edges.empty(); disk_edges = disk_storage->OutEdges(this, edge_types, destination, transaction_, view); - /// DiskStorage & View::OLD + if (view == View::OLD && !edges_modified_in_tx) { return EdgesVertexAccessorResult{.edges = disk_edges, .expanded_count = static_cast(disk_edges.size())}; } @@ -652,7 +649,6 @@ Result VertexAccessor::OutEdges(View view, const std: if (!exists) return Error::NONEXISTENT_OBJECT; if (deleted) return Error::DELETED_OBJECT; - /// DiskStorage & View::NEW if (transaction_->IsDiskStorage()) { return EdgesVertexAccessorResult{.edges = BuildResultWithDisk(out_edges, disk_edges, view, "OUT"), .expanded_count = expanded_count}; diff --git a/src/utils/rocksdb_serialization.hpp b/src/utils/rocksdb_serialization.hpp index 9af21d839..6871f1a69 100644 --- a/src/utils/rocksdb_serialization.hpp +++ b/src/utils/rocksdb_serialization.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "storage/v2/edge_accessor.hpp" #include "storage/v2/id_types.hpp" @@ -31,7 +32,6 @@ namespace memgraph::utils { static constexpr const char *outEdgeDirection = "0"; static constexpr const char *inEdgeDirection = "1"; -namespace { struct StartEndPositions { size_t start; size_t end; @@ -58,28 +58,6 @@ inline std::string_view FindPartOfStringView(const std::string_view str, const c return startEndPos.Valid() ? str.substr(startEndPos.start, startEndPos.Size()) : str; } -inline std::string_view GetViewOfFirstPartOfSplit(const std::string_view src, const char delimiter) { - return FindPartOfStringView(src, delimiter, 1); -} - -inline std::string_view GetViewOfSecondPartOfSplit(const std::string_view src, const char delimiter) { - return FindPartOfStringView(src, delimiter, 2); -} - -inline std::string_view GetViewOfThirdPartOfSplit(const std::string_view src, const char delimiter) { - return FindPartOfStringView(src, delimiter, 3); -} - -inline std::string_view GetViewOfFourthPartOfSplit(const std::string_view src, const char delimiter) { - return FindPartOfStringView(src, delimiter, 4); -} - -} // namespace - -/// TODO: try to move this to hpp files so that we can follow jump on readings - -inline std::string SerializeIdType(const auto &id) { return std::to_string(id.AsUint()); } - inline bool SerializedVertexHasLabels(const std::string &labels) { return !labels.empty(); } template @@ -92,7 +70,7 @@ inline std::vector TransformIDsToString(const TCollection &col) { std::vector transformed_col; transformed_col.reserve(col.size()); for (const auto &elem : col) { - transformed_col.emplace_back(SerializeIdType(elem)); + transformed_col.emplace_back(elem.ToString()); } return transformed_col; } @@ -106,29 +84,22 @@ inline std::vector TransformFromStringLabels(std::vector &labels) { - if (labels.empty()) { - return ""; - } - std::string result = labels[0]; - std::string ser_labels = - std::accumulate(std::next(labels.begin()), labels.end(), result, - [](const std::string &join, const auto &label_id) { return join + "," + label_id; }); - return ser_labels; -} +inline std::string SerializeLabels(const std::vector &labels) { return utils::Join(labels, ","); } inline std::string SerializeProperties(const storage::PropertyStore &properties) { return properties.StringBuffer(); } -/// TODO: andi Probably it is better to add delimiter between label,property and the rest of labels -/// TODO: reuse PutIndexingLabelAndPropertiesFirst inline std::string PutIndexingLabelAndPropertyFirst(const std::string &indexing_label, const std::string &indexing_property, const std::vector &vertex_labels) { - std::string result = indexing_label + "," + indexing_property; + std::string result; + result += indexing_label; + result += ","; + result += indexing_property; + for (const auto &label : vertex_labels) { if (label != indexing_label) { - result += "," + label; + result += ","; + result += label; } } return result; @@ -138,33 +109,43 @@ inline std::string PutIndexingLabelAndPropertiesFirst(const std::string &target_ const std::vector &target_properties) { std::string result = target_label; for (const auto &target_property : target_properties) { - result += "," + target_property; + result += ","; + result += target_property; } return result; } inline storage::Gid ExtractSrcVertexGidFromEdgeValue(const std::string value) { - const std::string_view src_vertex_gid_str = GetViewOfFirstPartOfSplit(value, '|'); - return storage::Gid::FromString(src_vertex_gid_str); + return storage::Gid::FromString(FindPartOfStringView(value, '|', 1)); } inline storage::Gid ExtractDstVertexGidFromEdgeValue(const std::string value) { - const std::string_view dst_vertex_gid_str = GetViewOfSecondPartOfSplit(value, '|'); - return storage::Gid::FromString(dst_vertex_gid_str); + return storage::Gid::FromString(FindPartOfStringView(value, '|', 2)); } inline storage::EdgeTypeId ExtractEdgeTypeIdFromEdgeValue(const std::string_view value) { - const std::string_view edge_type_str = GetViewOfThirdPartOfSplit(value, '|'); - return storage::EdgeTypeId::FromString(edge_type_str); + return storage::EdgeTypeId::FromString(FindPartOfStringView(value, '|', 3)); +} + +inline std::string_view GetPropertiesFromEdgeValue(const std::string_view value) { + return FindPartOfStringView(value, '|', 4); } inline std::string SerializeEdgeAsValue(const std::string &src_vertex_gid, const std::string &dst_vertex_gid, const storage::EdgeTypeId &edge_type, const storage::Edge *edge = nullptr) { - auto tmp = src_vertex_gid + "|" + dst_vertex_gid + "|" + SerializeIdType(edge_type) + "|"; + std::string edge_type_str = edge_type.ToString(); + std::string result; + result.reserve(src_vertex_gid.size() + 3 + dst_vertex_gid.size() + edge_type_str.size()); + result += src_vertex_gid; + result += "|"; + result += dst_vertex_gid; + result += "|"; + result += edge_type_str; + result += "|"; if (edge) { - return tmp + utils::SerializeProperties(edge->properties); + return result + utils::SerializeProperties(edge->properties); } - return tmp; + return result; } inline std::string SerializeVertexAsValueForAuxiliaryStorages(storage::LabelId label_to_remove, @@ -181,19 +162,15 @@ inline std::string SerializeVertexAsValueForAuxiliaryStorages(storage::LabelId l return result + SerializeProperties(property_store); } -inline std::string ExtractGidFromKey(const std::string &key) { - return std::string(GetViewOfSecondPartOfSplit(key, '|')); -} +inline std::string_view ExtractGidFromKey(const std::string &key) { return FindPartOfStringView(key, '|', 2); } inline storage::PropertyStore DeserializePropertiesFromAuxiliaryStorages(const std::string &value) { - const std::string_view properties_str = GetViewOfSecondPartOfSplit(value, '|'); - return storage::PropertyStore::CreateFromBuffer(properties_str); + return storage::PropertyStore::CreateFromBuffer(FindPartOfStringView(value, '|', 2)); } inline std::string SerializeVertex(const storage::Vertex &vertex) { std::string result = utils::SerializeLabels(TransformIDsToString(vertex.labels)) + "|"; - result += utils::SerializeIdType(vertex.gid); - return result; + return result + vertex.gid.ToString(); } inline std::vector DeserializeLabelsFromMainDiskStorage(const std::string &key) { @@ -205,26 +182,36 @@ inline std::vector DeserializeLabelsFromMainDiskStorage(const } inline std::vector ExtractLabelsFromMainDiskStorage(const std::string &key) { - return utils::Split(GetViewOfFirstPartOfSplit(key, '|'), ","); + return utils::Split(FindPartOfStringView(key, '|', 1), ","); } inline storage::PropertyStore DeserializePropertiesFromMainDiskStorage(const std::string_view value) { return storage::PropertyStore::CreateFromBuffer(value); } -inline std::string ExtractGidFromMainDiskStorage(const std::string &key) { return ExtractGidFromKey(key); } +inline std::string_view ExtractGidFromMainDiskStorage(const std::string &key) { return ExtractGidFromKey(key); } -inline std::string ExtractGidFromUniqueConstraintStorage(const std::string &key) { return ExtractGidFromKey(key); } +inline std::string_view ExtractGidFromUniqueConstraintStorage(const std::string &key) { return ExtractGidFromKey(key); } + +inline std::string GetKeyForUniqueConstraintsDurability(storage::LabelId label, + const std::set &properties) { + std::string entry; + entry += label.ToString(); + for (auto property : properties) { + entry += ","; + entry += property.ToString(); + } + return entry; +} -/// Serialize vertex to string as a key in unique constraint index KV store. -/// target_label, target_property_1, target_property_2, ... GID | -/// commit_timestamp inline std::string SerializeVertexAsKeyForUniqueConstraint(const storage::LabelId &constraint_label, const std::set &constraint_properties, - const std::string &gid) { - auto key_for_indexing = PutIndexingLabelAndPropertiesFirst(SerializeIdType(constraint_label), - TransformIDsToString(constraint_properties)); - return key_for_indexing + "|" + gid; + std::string_view gid) { + auto key_for_indexing = + PutIndexingLabelAndPropertiesFirst(constraint_label.ToString(), TransformIDsToString(constraint_properties)); + key_for_indexing += "|"; + key_for_indexing += gid; + return key_for_indexing; } inline std::string SerializeVertexAsValueForUniqueConstraint(const storage::LabelId &constraint_label, @@ -234,31 +221,29 @@ inline std::string SerializeVertexAsValueForUniqueConstraint(const storage::Labe } inline storage::LabelId DeserializeConstraintLabelFromUniqueConstraintStorage(const std::string &key) { - const std::string_view firstPartKey = GetViewOfFirstPartOfSplit(key, '|'); - const std::string_view constraint_key = GetViewOfFirstPartOfSplit(firstPartKey, ','); - /// TODO: andi Change this to deserialization method directly into the LabelId class - uint64_t labelID = 0; - const char *endOfConstraintKey = constraint_key.data() + constraint_key.size(); - auto [ptr, ec] = std::from_chars(constraint_key.data(), endOfConstraintKey, labelID); - if (ec != std::errc() || ptr != endOfConstraintKey) { - throw std::invalid_argument("Failed to deserialize label id from unique constraint storage"); - } - return storage::LabelId::FromUint(labelID); + const std::string_view firstPartKey = FindPartOfStringView(key, '|', 1); + const std::string_view constraint_key = FindPartOfStringView(firstPartKey, ',', 1); + return storage::LabelId::FromString(constraint_key); } inline storage::PropertyStore DeserializePropertiesFromUniqueConstraintStorage(const std::string &value) { return DeserializePropertiesFromAuxiliaryStorages(value); } -inline std::string SerializeVertexAsKeyForLabelIndex(const std::string &indexing_label, const std::string &gid) { - return indexing_label + "|" + gid; +inline std::string SerializeVertexAsKeyForLabelIndex(const std::string &indexing_label, std::string_view gid) { + std::string result; + result.reserve(indexing_label.size() + 1 + gid.size()); + result += indexing_label; + result += "|"; + result += gid; + return result; } inline std::string SerializeVertexAsKeyForLabelIndex(storage::LabelId label, storage::Gid gid) { - return SerializeVertexAsKeyForLabelIndex(SerializeIdType(label), utils::SerializeIdType(gid)); + return SerializeVertexAsKeyForLabelIndex(label.ToString(), gid.ToString()); } -inline std::string ExtractGidFromLabelIndexStorage(const std::string &key) { return ExtractGidFromKey(key); } +inline std::string_view ExtractGidFromLabelIndexStorage(const std::string &key) { return ExtractGidFromKey(key); } inline std::string SerializeVertexAsValueForLabelIndex(storage::LabelId indexing_label, const std::vector &vertex_labels, @@ -268,7 +253,7 @@ inline std::string SerializeVertexAsValueForLabelIndex(storage::LabelId indexing inline std::vector DeserializeLabelsFromIndexStorage(const std::string &key, const std::string &value) { - std::string labels_str{GetViewOfFirstPartOfSplit(value, '|')}; + std::string labels_str{FindPartOfStringView(value, '|', 1)}; std::vector labels{TransformFromStringLabels(utils::Split(labels_str, ","))}; std::string indexing_label = key.substr(0, key.find('|')); labels.emplace_back(storage::LabelId::FromString(indexing_label)); @@ -286,14 +271,20 @@ inline storage::PropertyStore DeserializePropertiesFromLabelIndexStorage(const s inline std::string SerializeVertexAsKeyForLabelPropertyIndex(const std::string &indexing_label, const std::string &indexing_property, - const std::string &gid) { - return indexing_label + "|" + indexing_property + "|" + gid; + std::string_view gid) { + std::string result; + result.reserve(indexing_label.size() + 2 + indexing_property.size() + gid.size()); + result += indexing_label; + result += "|"; + result += indexing_property; + result += "|"; + result += gid; + return result; } inline std::string SerializeVertexAsKeyForLabelPropertyIndex(storage::LabelId label, storage::PropertyId property, storage::Gid gid) { - return SerializeVertexAsKeyForLabelPropertyIndex(SerializeIdType(label), SerializeIdType(property), - utils::SerializeIdType(gid)); + return SerializeVertexAsKeyForLabelPropertyIndex(label.ToString(), property.ToString(), gid.ToString()); } inline std::string SerializeVertexAsValueForLabelPropertyIndex(storage::LabelId indexing_label, @@ -303,7 +294,7 @@ inline std::string SerializeVertexAsValueForLabelPropertyIndex(storage::LabelId } inline std::string ExtractGidFromLabelPropertyIndexStorage(const std::string &key) { - return std::string(GetViewOfThirdPartOfSplit(key, '|')); + return std::string(FindPartOfStringView(key, '|', 3)); } inline std::vector DeserializeLabelsFromLabelPropertyIndexStorage(const std::string &key, diff --git a/tests/e2e/mock_api/workloads.yaml b/tests/e2e/mock_api/workloads.yaml index 65691f6e4..9f6b43c65 100644 --- a/tests/e2e/mock_api/workloads.yaml +++ b/tests/e2e/mock_api/workloads.yaml @@ -100,10 +100,3 @@ workloads: proc: "tests/e2e/mock_api/procedures/" args: ["mock_api/test_compare_mock.py"] <<: *compare_mock_in_memory_cluster - - # Disk storage doesn't work with compare mock - # - name: "test-compare-mock on disk" - # binary: "tests/e2e/pytest_runner.sh" - # proc: "tests/e2e/mock_api/procedures/" - # args: ["mock_api/test_compare_mock.py"] - # <<: *compare_mock_disk_cluster diff --git a/tests/e2e/triggers/workloads.yaml b/tests/e2e/triggers/workloads.yaml index a69c606d4..81d6b464a 100644 --- a/tests/e2e/triggers/workloads.yaml +++ b/tests/e2e/triggers/workloads.yaml @@ -81,26 +81,3 @@ workloads: args: ["--bolt-port", *bolt_port] proc: "tests/e2e/triggers/procedures/" <<: *storage_properties_edges_true_disk_cluster - - # - name: "ON UPDATE Triggers for disk storage" - # binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_update" - # args: ["--bolt-port", *bolt_port] - # proc: "tests/e2e/triggers/procedures/" - # <<: *storage_properties_edges_true_disk_cluster - - # - name: "ON DELETE Triggers Storage Properties On Edges True for disk storage" - # binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_delete" - # args: ["--bolt-port", *bolt_port] - # proc: "tests/e2e/triggers/procedures/" - # <<: *storage_properties_edges_true_disk_cluster - - # - name: "Triggers privilege check for disk storage" - # binary: "tests/e2e/triggers/memgraph__e2e__triggers__privileges" - # args: ["--bolt-port", *bolt_port] - # <<: *storage_properties_edges_true_disk_cluster - - # - name: "ON DELETE Triggers Storage Properties On Edges False for disk storage" - # binary: "tests/e2e/pytest_runner.sh" - # proc: "tests/e2e/triggers/procedures/" - # args: ["triggers/triggers_properties_false.py"] - # <<: *storage_properties_edges_false_disk_cluster diff --git a/tests/e2e/write_procedures/workloads.yaml b/tests/e2e/write_procedures/workloads.yaml index 89d60b853..3e913cbf4 100644 --- a/tests/e2e/write_procedures/workloads.yaml +++ b/tests/e2e/write_procedures/workloads.yaml @@ -33,18 +33,3 @@ workloads: proc: "tests/e2e/write_procedures/procedures/" args: ["write_procedures/read_subgraph.py"] <<: *in_memory_cluster - - # TODO: has to be addressed. - # - name: "Write procedures simple on disk" - # binary: "tests/e2e/pytest_runner.sh" - # proc: "tests/e2e/write_procedures/procedures/" - # args: ["write_procedures/simple_write.py"] - # <<: *disk_cluster - - # TODO: Has to be addressed but currently some problem with disk storage and edges. - # Edge case and requires refactoring of bulk detach delete. - # - name: "Graph projection procedures on disk" - # binary: "tests/e2e/pytest_runner.sh" - # proc: "tests/e2e/write_procedures/procedures/" - # args: ["write_procedures/read_subgraph.py"] - # <<: *disk_cluster diff --git a/tests/unit/storage_rocks.cpp b/tests/unit/storage_rocks.cpp index aeb290769..42890c383 100644 --- a/tests/unit/storage_rocks.cpp +++ b/tests/unit/storage_rocks.cpp @@ -60,7 +60,7 @@ TEST_F(RocksDBStorageTest, SerializeVertexGID) { auto acc = storage->Access(); auto vertex = acc->CreateVertex(); auto gid = vertex.Gid(); - ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), "|" + memgraph::utils::SerializeIdType(gid)); + ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), "|" + gid.ToString()); } TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) { @@ -71,9 +71,8 @@ TEST_F(RocksDBStorageTest, SerializeVertexGIDLabels) { ASSERT_FALSE(vertex.AddLabel(ser_player_label).HasError()); ASSERT_FALSE(vertex.AddLabel(ser_user_label).HasError()); auto gid = vertex.Gid(); - ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), std::to_string(ser_player_label.AsInt()) + "," + - std::to_string(ser_user_label.AsInt()) + "|" + - memgraph::utils::SerializeIdType(gid)); + ASSERT_EQ(memgraph::utils::SerializeVertex(*vertex.vertex_), + ser_player_label.ToString() + "," + ser_user_label.ToString() + "|" + gid.ToString()); } TEST_F(RocksDBStorageTest, SerializePropertiesLocalBuffer) { diff --git a/tests/unit/storage_v2_constraints.cpp b/tests/unit/storage_v2_constraints.cpp index 990ecfbdd..4a60c5330 100644 --- a/tests/unit/storage_v2_constraints.cpp +++ b/tests/unit/storage_v2_constraints.cpp @@ -765,8 +765,7 @@ TYPED_TEST(ConstraintsTest, UniqueConstraintsLabelAlteration) { gid1 = vertex1.Gid(); gid2 = vertex2.Gid(); - spdlog::debug("Vertex1 gid: {} Vertex2 gid: {}\n", memgraph::utils::SerializeIdType(gid1), - memgraph::utils::SerializeIdType(gid2)); + spdlog::debug("Vertex1 gid: {} Vertex2 gid: {}\n", gid1.ToString(), gid2.ToString()); ASSERT_NO_ERROR(vertex1.AddLabel(this->label2)); ASSERT_NO_ERROR(vertex1.SetProperty(this->prop1, PropertyValue(1)));