tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add LabelIndexStats

Browse Source
This commit is contained in:
Josip Mrden
2023-05-11 09:52:25 +02:00
parent b2477663b7
commit b60aebaf56
7 changed files with 110 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/query/db_accessor.hpp
View File

@@ -430,6 +430,10 @@ class DbAccessor final {
return accessor_->LabelPropertyIndexExists(label, prop);
}
std::optional<storage::LabelIndexStats> GetLabelIndexStats(const storage::LabelId &label) const {
return accessor_->GetLabelIndexStats(label);
}
std::optional<storage::IndexStats> GetIndexStats(const storage::LabelId &label,
const storage::PropertyId &property) const {
return accessor_->GetIndexStats(label, property);
@@ -444,6 +448,10 @@ class DbAccessor final {
return accessor_->DeleteIndexStatsForLabels(labels);
}
void SetLabelIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats) {
accessor_->SetLabelIndexStats(label, stats);
}
void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property,
const storage::IndexStats &stats) {
accessor_->SetIndexStats(label, property, stats);

39
src/query/interpreter.cpp
View File

@@ -53,6 +53,7 @@
#include "query/typed_value.hpp"
#include "storage/v2/edge.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/indices.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
@@ -1456,6 +1457,18 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
std::map<LPIndex, std::map<storage::PropertyValue, int64_t>> counter;
std::map<LPIndex, uint64_t> vertex_degree_counter;
// Preprocess labels in label indexes to avoid later checks
std::vector<storage::LabelId> label_indices_info = execution_db_accessor->ListAllIndices().label;
if (labels[0] != kAsterisk) {
for (auto it = label_indices_info.cbegin(); it != label_indices_info.cend();) {
if (std::find(labels.begin(), labels.end(), execution_db_accessor->LabelToName(*it)) == labels.end()) {
it = label_indices_info.erase(it);
} else {
++it;
}
}
}
// Preprocess labels to avoid later checks
std::vector<LPIndex> indices_info = execution_db_accessor->ListAllIndices().label_property;
if (labels[0] != kAsterisk) {
@@ -1467,6 +1480,23 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
}
}
}
// Iterate over all indexed vertices
std::for_each(label_indices_info.begin(), label_indices_info.end(),
[execution_db_accessor](const storage::LabelId &index_info) {
auto vertices = execution_db_accessor->Vertices(storage::View::OLD, index_info);
int64_t no_vertices = 0;
auto total_degree = 0;
std::for_each(vertices.begin(), vertices.end(), [&total_degree, &no_vertices](const auto &vertex) {
no_vertices++;
total_degree += *vertex.OutDegree(storage::View::OLD) + *vertex.InDegree(storage::View::OLD);
});
auto average_degree = (double)total_degree / no_vertices;
execution_db_accessor->SetLabelIndexStats(
index_info, storage::LabelIndexStats{.count = no_vertices, .avg_degree = average_degree});
});
// Iterate over all indexed vertices
std::for_each(indices_info.begin(), indices_info.end(),
[execution_db_accessor, &counter, &vertex_degree_counter](const LPIndex &index_info) {
@@ -1498,10 +1528,11 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
return prev_result + utils::ChiSquaredValue(value_entry.second, avg_group_size);
});
double average_degree = (double)vertex_degree_counter[label_property] / count_property_value;
execution_db_accessor->SetIndexStats(
label_property.first, label_property.second,
storage::IndexStats{
.statistic = chi_squared_stat, .avg_group_size = avg_group_size, .avg_degree = average_degree});
execution_db_accessor->SetIndexStats(label_property.first, label_property.second,
storage::IndexStats{.count = count_property_value,
.statistic = chi_squared_stat,
.avg_group_size = avg_group_size,
.avg_degree = average_degree});
// Save result
result.emplace_back(execution_db_accessor->LabelToName(label_property.first));
result.emplace_back(execution_db_accessor->PropertyToName(label_property.second));

37
src/query/plan/rewrite/index_lookup.hpp
View File

@@ -39,11 +39,20 @@ namespace impl {
// given expression tree.
Expression *RemoveAndExpressions(Expression *expr, const std::unordered_set<Expression *> &exprs_to_remove);
struct Scope {
bool in_optional{false};
std::map<std::string, double> degree;
std::map<std::string, uint64_t> cardinality;
};
template <class TDbAccessor>
class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
public:
IndexLookupRewriter(SymbolTable *symbol_table, AstStorage *ast_storage, TDbAccessor *db)
: symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db) {}
: symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), scope_(Scope()) {}
IndexLookupRewriter(SymbolTable *symbol_table, AstStorage *ast_storage, TDbAccessor *db, Scope scope)
: symbol_table_(symbol_table), ast_storage_(ast_storage), db_(db), scope_(scope) {}
using HierarchicalLogicalOperatorVisitor::PostVisit;
using HierarchicalLogicalOperatorVisitor::PreVisit;
@@ -155,6 +164,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
}
bool PreVisit(Optional &op) override {
scope_.in_optional = true;
prev_ops_.push_back(&op);
op.input()->Accept(*this);
RewriteBranch(&op.optional_);
@@ -162,6 +172,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
}
bool PostVisit(Optional &) override {
scope_.in_optional = false;
prev_ops_.pop_back();
return true;
}
@@ -489,6 +500,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
// Expressions which no longer need a plain Filter operator.
std::unordered_set<Expression *> filter_exprs_for_removal_;
std::vector<LogicalOperator *> prev_ops_;
Scope scope_;
struct LabelPropertyIndex {
LabelIx label;
@@ -511,7 +523,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
}
void RewriteBranch(std::shared_ptr<LogicalOperator> *branch) {
IndexLookupRewriter<TDbAccessor> rewriter(symbol_table_, ast_storage_, db_);
IndexLookupRewriter<TDbAccessor> rewriter(symbol_table_, ast_storage_, db_, scope_);
(*branch)->Accept(rewriter);
if (rewriter.new_root_) {
*branch = rewriter.new_root_;
@@ -623,6 +635,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
}
return found;
}
// Creates a ScanAll by the best possible index for the `node_symbol`. If the node
// does not have at least a label, no indexed lookup can be created and
// `nullptr` is returned. The operator is chained after `input`. Optional
@@ -695,15 +708,30 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
auto *expression = ast_storage_->Create<Identifier>(symbol.name_);
expression->MapTo(symbol);
auto unwind_operator = std::make_unique<Unwind>(input, prop_filter.value_, symbol);
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
}
return std::make_unique<ScanAllByLabelPropertyValue>(
std::move(unwind_operator), node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_),
prop_filter.property_.name, expression, view);
} else if (prop_filter.type_ == PropertyFilter::Type::IS_NOT_NULL) {
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
}
return std::make_unique<ScanAllByLabelProperty>(input, node_symbol, GetLabel(found_index->label),
GetProperty(prop_filter.property_), prop_filter.property_.name,
view);
} else {
MG_ASSERT(prop_filter.value_, "Property filter should either have bounds or a value expression.");
auto index_stats = db_->GetIndexStats(GetLabel(found_index->label), GetProperty(prop_filter.property_));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
}
return std::make_unique<ScanAllByLabelPropertyValue>(input, node_symbol, GetLabel(found_index->label),
GetProperty(prop_filter.property_),
prop_filter.property_.name, prop_filter.value_, view);
@@ -720,6 +748,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
std::vector<Expression *> removed_expressions;
filters_.EraseLabelFilter(node_symbol, label, &removed_expressions);
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
auto index_stats = db_->GetLabelIndexStats(GetLabel(label));
if (index_stats) {
scope_.degree[node_symbol.name()] = index_stats->avg_degree;
scope_.cardinality[node_symbol.name()] = index_stats->count;
}
return std::make_unique<ScanAllByLabel>(input, node_symbol, GetLabel(label), view);
}
};

4
src/query/plan/vertex_count_cache.hpp
View File

@@ -78,6 +78,10 @@ class VertexCountCache {
return db_->LabelPropertyIndexExists(label, property);
}
std::optional<storage::LabelIndexStats> GetLabelIndexStats(const storage::LabelId &label) const {
return db_->GetLabelIndexStats(label);
}
std::optional<storage::IndexStats> GetIndexStats(const storage::LabelId &label,
const storage::PropertyId &property) const {
return db_->GetIndexStats(label, property);

9
src/storage/v2/indices.cpp
View File

@@ -478,6 +478,15 @@ void LabelIndex::RunGC() {
}
}
void LabelIndex::SetIndexStats(const storage::LabelId &label, const LabelIndexStats &stats) { stats_[label] = stats; }
std::optional<LabelIndexStats> LabelIndex::GetIndexStats(const storage::LabelId &label) const {
if (auto it = stats_.find(label); it != stats_.end()) {
return it->second;
}
return {};
}
bool LabelPropertyIndex::Entry::operator<(const Entry &rhs) {
if (value < rhs.value) {
return true;

11
src/storage/v2/indices.hpp
View File

@@ -31,6 +31,11 @@ struct Constraints;
using ParalellizedIndexCreationInfo =
std::pair<std::vector<std::pair<Gid, uint64_t>> /*vertex_recovery_info*/, uint64_t /*thread_count*/>;
struct LabelIndexStats {
int64_t count;
double avg_degree;
};
class LabelIndex {
private:
struct Entry {
@@ -124,18 +129,24 @@ class LabelIndex {
return it->second.size();
}
void SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats);
std::optional<storage::LabelIndexStats> GetIndexStats(const storage::LabelId &label) const;
void Clear() { index_.clear(); }
void RunGC();
private:
std::map<LabelId, utils::SkipList<Entry>> index_;
std::map<LabelId, storage::LabelIndexStats> stats_;
Indices *indices_;
Constraints *constraints_;
Config::Items config_;
};
struct IndexStats {
int64_t count;
double statistic, avg_group_size, avg_degree;
};

8
src/storage/v2/storage.hpp
View File

@@ -267,6 +267,10 @@ class Storage final {
return storage_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
}
std::optional<storage::LabelIndexStats> GetLabelIndexStats(const storage::LabelId &label) const {
return storage_->indices_.label_index.GetIndexStats(label);
}
std::optional<storage::IndexStats> GetIndexStats(const storage::LabelId &label,
const storage::PropertyId &property) const {
return storage_->indices_.label_property_index.GetIndexStats(label, property);
@@ -287,6 +291,10 @@ class Storage final {
return deleted_indexes;
}
void SetLabelIndexStats(const storage::LabelId &label, const LabelIndexStats &stats) {
storage_->indices_.label_index.SetIndexStats(label, stats);
}
void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property, const IndexStats &stats) {
storage_->indices_.label_property_index.SetIndexStats(label, property, stats);
}