tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Introduce degree in cost estimator and index lookup expansions

Browse Source
This commit is contained in:
Josip Mrden
2023-05-11 14:35:37 +02:00
parent b60aebaf56
commit 54bb5ed50d
6 changed files with 81 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/query/db_accessor.hpp
View File

@@ -440,9 +440,12 @@ class DbAccessor final {
}
std::vector<std::pair<storage::LabelId, storage::PropertyId>> ClearIndexStats() {
accessor_->ClearLabelIndexStats();
return accessor_->ClearIndexStats();
}
std::vector<storage::LabelId> ClearLabelIndexStats() { return accessor_->ClearLabelIndexStats(); }
std::vector<std::pair<storage::LabelId, storage::PropertyId>> DeleteIndexStatsForLabels(
const std::span<std::string> labels) {
return accessor_->DeleteIndexStatsForLabels(labels);

39
src/query/plan/cost_estimator.hpp
View File

@@ -15,9 +15,20 @@
#include "query/parameters.hpp"
#include "query/plan/operator.hpp"
#include "query/typed_value.hpp"
#include "utils/algorithm.hpp"
namespace memgraph::query::plan {
struct SymbolStatistics {
std::string name;
int64_t cardinality;
double degree;
};
struct Scope {
std::unordered_map<std::string, SymbolStatistics> symbol_stats;
};
/**
* Query plan execution time cost estimator, for comparing and choosing optimal
* execution plans.
@@ -82,7 +93,10 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
using HierarchicalLogicalOperatorVisitor::PreVisit;
CostEstimator(TDbAccessor *db_accessor, const Parameters &parameters)
: db_accessor_(db_accessor), parameters(parameters) {}
: db_accessor_(db_accessor), parameters(parameters), scope_(Scope()) {}
CostEstimator(TDbAccessor *db_accessor, const Parameters &parameters, Scope scope)
: db_accessor_(db_accessor), parameters(parameters), scope_(scope) {}
bool PostVisit(ScanAll &) override {
cardinality_ *= db_accessor_->VerticesCount();
@@ -92,6 +106,15 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
}
bool PostVisit(ScanAllByLabel &scan_all_by_label) override {
auto index_stats = db_accessor_->GetLabelIndexStats(scan_all_by_label.label_);
if (index_stats) {
scope_.symbol_stats[scan_all_by_label.output_symbol_.name()] = SymbolStatistics{
.name = scan_all_by_label.output_symbol_.name(),
.cardinality = index_stats->count,
.degree = index_stats->avg_degree,
};
}
cardinality_ *= db_accessor_->VerticesCount(scan_all_by_label.label_);
// ScanAll performs some work for every element that is produced
IncrementCost(CostParam::kScanAllByLabel);
@@ -152,6 +175,18 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
// TODO: Cost estimate ScanAllById?
bool PostVisit(Expand &expand) override {
auto card_param = CardParam::kExpand;
if (utils::Contains(scope_.symbol_stats, expand.input_symbol_.name())) {
card_param = scope_.symbol_stats[expand.input_symbol_.name()].degree;
}
cardinality_ *= card_param;
IncrementCost(CostParam::kExpand);
return true;
}
// For the given op first increments the cardinality and then cost.
#define POST_VISIT_CARD_FIRST(NAME) \
bool PostVisit(NAME &) override { \
@@ -160,7 +195,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
return true; \
}
POST_VISIT_CARD_FIRST(Expand);
POST_VISIT_CARD_FIRST(ExpandVariable);
#undef POST_VISIT_CARD_FIRST
@@ -256,6 +290,7 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
// accessor used for cardinality estimates in ScanAll and ScanAllByLabel
TDbAccessor *db_accessor_;
const Parameters &parameters;
Scope scope_;
void IncrementCost(double param) { cost_ += param * cardinality_; }

38
src/query/plan/rewrite/index_lookup.hpp
View File

@@ -28,6 +28,7 @@
#include "query/plan/operator.hpp"
#include "query/plan/preprocess.hpp"
#include "storage/v2/indices.hpp"
#include "utils/algorithm.hpp"
DECLARE_int64(query_vertex_count_to_expand_existing);
@@ -39,10 +40,15 @@ namespace impl {
// given expression tree.
Expression *RemoveAndExpressions(Expression *expr, const std::unordered_set<Expression *> &exprs_to_remove);
struct SymbolStatistics {
std::string name;
int64_t cardinality;
double degree;
};
struct Scope {
bool in_optional{false};
std::map<std::string, double> degree;
std::map<std::string, uint64_t> cardinality;
std::unordered_map<std::string, SymbolStatistics> symbol_stats;
};
template <class TDbAccessor>
@@ -110,6 +116,18 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
ScanAll dst_scan(expand.input(), expand.common_.node_symbol, expand.view_);
auto indexed_scan = GenScanByIndex(dst_scan, FLAGS_query_vertex_count_to_expand_existing);
if (indexed_scan) {
if (utils::Contains(scope_.symbol_stats, expand.common_.node_symbol.name()) &&
utils::Contains(scope_.symbol_stats, expand.input_symbol_.name())) {
auto src_cost = scope_.symbol_stats[expand.input_symbol_.name()].degree;
auto dest_cost = scope_.symbol_stats[expand.common_.node_symbol.name()].cardinality;
if (dest_cost < src_cost) {
expand.set_input(std::move(indexed_scan));
expand.common_.existing_node = true;
}
return true;
}
expand.set_input(std::move(indexed_scan));
expand.common_.existing_node = true;
}
@@ -710,8 +728,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
auto unwind_operator = std::make_unique<Unwind>(input, prop_filter.value_, symbol);
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{
.name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree};
}
return std::make_unique<ScanAllByLabelPropertyValue>(
std::move(unwind_operator), node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_),
@@ -719,8 +737,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
} else if (prop_filter.type_ == PropertyFilter::Type::IS_NOT_NULL) {
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{
.name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree};
}
return std::make_unique<ScanAllByLabelProperty>(input, node_symbol, GetLabel(found_index->label),
GetProperty(prop_filter.property_), prop_filter.property_.name,
@@ -729,8 +747,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
MG_ASSERT(prop_filter.value_, "Property filter should either have bounds or a value expression.");
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{
.name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree};
}
return std::make_unique<ScanAllByLabelPropertyValue>(input, node_symbol, GetLabel(found_index->label),
GetProperty(prop_filter.property_),
@@ -750,8 +768,8 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
auto index_stats = db_->GetLabelIndexStats(GetLabel(label));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
scope_.symbol_stats[node_symbol.name()] = SymbolStatistics{
.name = node_symbol.name(), .cardinality = index_stats->count, .degree = index_stats->avg_degree};
}
return std::make_unique<ScanAllByLabel>(input, node_symbol, GetLabel(label), view);
}

9
src/storage/v2/indices.cpp
View File

@@ -837,6 +837,15 @@ std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::DeleteIndexStats
return deleted_indexes;
}
std::vector<LabelId> LabelIndex::ClearIndexStats() {
std::vector<LabelId> deleted_indexes;
deleted_indexes.reserve(stats_.size());
std::transform(stats_.begin(), stats_.end(), std::back_inserter(deleted_indexes),
[](const auto &elem) { return elem.first; });
stats_.clear();
return deleted_indexes;
}
std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ClearIndexStats() {
std::vector<std::pair<LabelId, PropertyId>> deleted_indexes;
deleted_indexes.reserve(stats_.size());

2
src/storage/v2/indices.hpp
View File

@@ -133,6 +133,8 @@ class LabelIndex {
std::optional<storage::LabelIndexStats> GetIndexStats(const storage::LabelId &label) const;
std::vector<LabelId> ClearIndexStats();
void Clear() { index_.clear(); }
void RunGC();

2
src/storage/v2/storage.hpp
View File

@@ -280,6 +280,8 @@ class Storage final {
return storage_->indices_.label_property_index.ClearIndexStats();
}
std::vector<LabelId> ClearLabelIndexStats() { return storage_->indices_.label_index.ClearIndexStats(); }
std::vector<std::pair<LabelId, PropertyId>> DeleteIndexStatsForLabels(const std::span<std::string> labels) {
std::vector<std::pair<LabelId, PropertyId>> deleted_indexes;
std::for_each(labels.begin(), labels.end(), [this, &deleted_indexes](const auto &label_str) {