diff --git a/src/query/db_accessor.hpp b/src/query/db_accessor.hpp index e370082ad..f9595b589 100644 --- a/src/query/db_accessor.hpp +++ b/src/query/db_accessor.hpp @@ -440,9 +440,12 @@ class DbAccessor final { } std::vector> ClearIndexStats() { + accessor_->ClearLabelIndexStats(); return accessor_->ClearIndexStats(); } + std::vector ClearLabelIndexStats() { return accessor_->ClearLabelIndexStats(); } + std::vector> DeleteIndexStatsForLabels( const std::span labels) { return accessor_->DeleteIndexStatsForLabels(labels); diff --git a/src/query/plan/cost_estimator.hpp b/src/query/plan/cost_estimator.hpp index f9b71b0d8..15684299a 100644 --- a/src/query/plan/cost_estimator.hpp +++ b/src/query/plan/cost_estimator.hpp @@ -15,9 +15,20 @@ #include "query/parameters.hpp" #include "query/plan/operator.hpp" #include "query/typed_value.hpp" +#include "utils/algorithm.hpp" namespace memgraph::query::plan { +struct SymbolStatistics { + std::string name; + int64_t cardinality; + double degree; +}; + +struct Scope { + std::unordered_map symbol_stats; +}; + /** * Query plan execution time cost estimator, for comparing and choosing optimal * execution plans. @@ -82,7 +93,10 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { using HierarchicalLogicalOperatorVisitor::PreVisit; CostEstimator(TDbAccessor *db_accessor, const Parameters ¶meters) - : db_accessor_(db_accessor), parameters(parameters) {} + : db_accessor_(db_accessor), parameters(parameters), scope_(Scope()) {} + + CostEstimator(TDbAccessor *db_accessor, const Parameters ¶meters, Scope scope) + : db_accessor_(db_accessor), parameters(parameters), scope_(scope) {} bool PostVisit(ScanAll &) override { cardinality_ *= db_accessor_->VerticesCount(); @@ -92,6 +106,15 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { } bool PostVisit(ScanAllByLabel &scan_all_by_label) override { + auto index_stats = db_accessor_->GetLabelIndexStats(scan_all_by_label.label_); + if (index_stats) { + scope_.symbol_stats[scan_all_by_label.output_symbol_.name()] = SymbolStatistics{ + .name = scan_all_by_label.output_symbol_.name(), + .cardinality = index_stats->count, + .degree = index_stats->avg_degree, + }; + } + cardinality_ *= db_accessor_->VerticesCount(scan_all_by_label.label_); // ScanAll performs some work for every element that is produced IncrementCost(CostParam::kScanAllByLabel); @@ -152,6 +175,18 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { // TODO: Cost estimate ScanAllById? + bool PostVisit(Expand &expand) override { + auto card_param = CardParam::kExpand; + if (utils::Contains(scope_.symbol_stats, expand.input_symbol_.name())) { + card_param = scope_.symbol_stats[expand.input_symbol_.name()].degree; + } + + cardinality_ *= card_param; + IncrementCost(CostParam::kExpand); + + return true; + } + // For the given op first increments the cardinality and then cost. #define POST_VISIT_CARD_FIRST(NAME) \ bool PostVisit(NAME &) override { \ @@ -160,7 +195,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { return true; \ } - POST_VISIT_CARD_FIRST(Expand); POST_VISIT_CARD_FIRST(ExpandVariable); #undef POST_VISIT_CARD_FIRST @@ -256,6 +290,7 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor { // accessor used for cardinality estimates in ScanAll and ScanAllByLabel TDbAccessor *db_accessor_; const Parameters ¶meters; + Scope scope_; void IncrementCost(double param) { cost_ += param * cardinality_; } diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 874398f63..509fbb2a7 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -28,6 +28,7 @@ #include "query/plan/operator.hpp" #include "query/plan/preprocess.hpp" #include "storage/v2/indices.hpp" +#include "utils/algorithm.hpp" DECLARE_int64(query_vertex_count_to_expand_existing); @@ -39,10 +40,15 @@ namespace impl { // given expression tree. Expression *RemoveAndExpressions(Expression *expr, const std::unordered_set &exprs_to_remove); +struct SymbolStatistics { + std::string name; + int64_t cardinality; + double degree; +}; + struct Scope { bool in_optional{false}; - std::map degree; - std::map cardinality; + std::unordered_map symbol_stats; }; template @@ -110,6 +116,18 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { ScanAll dst_scan(expand.input(), expand.common_.node_symbol, expand.view_); auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing); if (indexed_scan) { + if (utils::Contains(scope_.symbol_stats, expand.common_.node_symbol.name()) && + utils::Contains(scope_.symbol_stats, expand.input_symbol_.name())) { + auto src_cost = scope_.symbol_stats[expand.input_symbol_.name()].degree; + auto dest_cost = scope_.symbol_stats[expand.common_.node_symbol.name()].cardinality; + + if (dest_cost < src_cost) { + expand.set_input(std::move(indexed_scan)); + expand.common_.existing_node = true; + } + return true; + } + expand.set_input(std::move(indexed_scan)); expand.common_.existing_node = true; } @@ -710,8 +728,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { auto unwind_operator = std::make_unique(input, prop_filter.value_, symbol); auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_)); if (index_stats) { - scope_.degree[node_symbol.name()] = index_stats->avg_degree; - scope_.cardinality[node_symbol.name()] = index_stats->count; + scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{ + .name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree}; } return std::make_unique( std::move(unwind_operator), node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_), @@ -719,8 +737,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { } else if (prop_filter.type_ == PropertyFilter::Type::IS_NOT_NULL) { auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_)); if (index_stats) { - scope_.degree[node_symbol.name()] = index_stats->avg_degree; - scope_.cardinality[node_symbol.name()] = index_stats->count; + scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{ + .name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree}; } return std::make_unique(input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_), prop_filter.property_.name, @@ -729,8 +747,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { MG_ASSERT(prop_filter.value_, "Property filter should either have bounds or a value expression."); auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_)); if (index_stats) { - scope_.degree[node_symbol.name()] = index_stats->avg_degree; - scope_.cardinality[node_symbol.name()] = index_stats->count; + scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{ + .name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree}; } return std::make_unique(input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_), @@ -750,8 +768,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end()); auto index_stats = db_->GetLabelIndexStats(GetLabel(label)); if (index_stats) { - scope_.degree[node_symbol.name()] = index_stats->avg_degree; - scope_.cardinality[node_symbol.name()] = index_stats->count; + scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{ + .name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree}; } return std::make_unique(input, node_symbol, GetLabel(label), view); } diff --git a/src/storage/v2/indices.cpp b/src/storage/v2/indices.cpp index 42ec8ae5d..99e0dda32 100644 --- a/src/storage/v2/indices.cpp +++ b/src/storage/v2/indices.cpp @@ -837,6 +837,15 @@ std::vector> LabelPropertyIndex::DeleteIndexStats return deleted_indexes; } +std::vector LabelIndex::ClearIndexStats() { + std::vector deleted_indexes; + deleted_indexes.reserve(stats_.size()); + std::transform(stats_.begin(), stats_.end(), std::back_inserter(deleted_indexes), + [](const auto &elem) { return elem.first; }); + stats_.clear(); + return deleted_indexes; +} + std::vector> LabelPropertyIndex::ClearIndexStats() { std::vector> deleted_indexes; deleted_indexes.reserve(stats_.size()); diff --git a/src/storage/v2/indices.hpp b/src/storage/v2/indices.hpp index a2ce30e1e..860c1e875 100644 --- a/src/storage/v2/indices.hpp +++ b/src/storage/v2/indices.hpp @@ -133,6 +133,8 @@ class LabelIndex { std::optional GetIndexStats(const storage::LabelId &label) const; + std::vector ClearIndexStats(); + void Clear() { index_.clear(); } void RunGC(); diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 12bb427d4..157795a41 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -280,6 +280,8 @@ class Storage final { return storage_->indices_.label_property_index.ClearIndexStats(); } + std::vector ClearLabelIndexStats() { return storage_->indices_.label_index.ClearIndexStats(); } + std::vector> DeleteIndexStatsForLabels(const std::span labels) { std::vector> deleted_indexes; std::for_each(labels.begin(), labels.end(), [this, &deleted_indexes](const auto &label_str) {