From 8b849575c1a77d4c54f085fdd7a440acc1ea893f Mon Sep 17 00:00:00 2001 From: Josip Mrden Date: Tue, 20 Jun 2023 13:11:19 +0200 Subject: [PATCH] Add typed value results for label indexes --- src/query/constants.hpp | 2 +- src/query/interpreter.cpp | 120 +++++++++++++++--------- src/query/plan/rewrite/index_lookup.hpp | 5 +- src/storage/v2/indices.hpp | 2 +- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/src/query/constants.hpp b/src/query/constants.hpp index 1529fa2ca..2ee036957 100644 --- a/src/query/constants.hpp +++ b/src/query/constants.hpp @@ -17,5 +17,5 @@ namespace memgraph::query { inline constexpr uint16_t kDefaultReplicationPort = 10000; inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; inline const std::string kAsterisk = "*"; -inline constexpr uint16_t kDeleteStatisticsNumResults = 7; +inline constexpr uint16_t kComputeStatisticsNumResults = 7; } // namespace memgraph::query diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index d8887d780..676bb5ec2 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -1518,9 +1518,9 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat const std::span labels, DbAccessor *execution_db_accessor) { using LPIndex = std::pair; - std::vector> results; - std::map> counter; + std::map> label_property_counter; std::map vertex_degree_counter; + auto view = storage::View::OLD; // Preprocess labels in label indexes to avoid later checks std::vector label_indices_info = execution_db_accessor->ListAllIndices().label; @@ -1534,54 +1534,54 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat } } - // Preprocess labels to avoid later checks - std::vector indices_info = execution_db_accessor->ListAllIndices().label_property; + // Preprocess labels in label property indexes to avoid later checks + std::vector label_property_indices_info = execution_db_accessor->ListAllIndices().label_property; if (labels[0] != kAsterisk) { - for (auto it = indices_info.cbegin(); it != indices_info.cend();) { + for (auto it = label_property_indices_info.cbegin(); it != label_property_indices_info.cend();) { if (std::find(labels.begin(), labels.end(), execution_db_accessor->LabelToName(it->first)) == labels.end()) { - it = indices_info.erase(it); + it = label_property_indices_info.erase(it); } else { ++it; } } } - // Iterate over all indexed vertices + std::vector> label_stats; + // Iterate over all label indexed vertices std::for_each(label_indices_info.begin(), label_indices_info.end(), - [execution_db_accessor](const storage::LabelId &index_info) { + [execution_db_accessor, view, &label_stats](const storage::LabelId &index_info) { auto vertices = execution_db_accessor->Vertices(storage::View::OLD, index_info); int64_t no_vertices = 0; auto total_degree = 0; - std::for_each(vertices.begin(), vertices.end(), [&total_degree, &no_vertices](const auto &vertex) { - no_vertices++; - total_degree += *vertex.OutDegree(storage::View::OLD) + *vertex.InDegree(storage::View::OLD); - }); + std::for_each(vertices.begin(), vertices.end(), + [&total_degree, &no_vertices, &view](const auto &vertex) { + no_vertices++; + total_degree += *vertex.OutDegree(view) + *vertex.InDegree(view); + }); auto average_degree = (double)total_degree / no_vertices; - execution_db_accessor->SetIndexStats( - index_info, storage::LabelIndexStats{.count = no_vertices, .avg_degree = average_degree}); + auto index_stats = storage::LabelIndexStats{.count = no_vertices, .avg_degree = average_degree}; + execution_db_accessor->SetIndexStats(index_info, index_stats); + label_stats.push_back(std::make_pair(index_info, index_stats)); }); - // Iterate over all indexed vertices - std::for_each(indices_info.begin(), indices_info.end(), - [execution_db_accessor, &counter, &vertex_degree_counter](const LPIndex &index_info) { - auto vertices = - execution_db_accessor->Vertices(storage::View::OLD, index_info.first, index_info.second); - std::for_each(vertices.begin(), vertices.end(), - [&index_info, &counter, &vertex_degree_counter](const auto &vertex) { - counter[index_info][*vertex.GetProperty(storage::View::OLD, index_info.second)]++; - vertex_degree_counter[index_info] += - *vertex.OutDegree(storage::View::OLD) + *vertex.InDegree(storage::View::OLD); - }); - }); - - results.reserve(counter.size()); + // Iterate over all label property indexed vertices std::for_each( - counter.begin(), counter.end(), - [&results, execution_db_accessor, &vertex_degree_counter](const auto &counter_entry) { + label_property_indices_info.begin(), label_property_indices_info.end(), + [execution_db_accessor, &label_property_counter, &vertex_degree_counter, view](const LPIndex &index_info) { + auto vertices = execution_db_accessor->Vertices(view, index_info.first, index_info.second); + std::for_each(vertices.begin(), vertices.end(), + [&index_info, &label_property_counter, &vertex_degree_counter, &view](const auto &vertex) { + label_property_counter[index_info][*vertex.GetProperty(view, index_info.second)]++; + vertex_degree_counter[index_info] += *vertex.OutDegree(view) + *vertex.InDegree(view); + }); + }); + + std::vector> label_property_stats; + std::for_each( + label_property_counter.begin(), label_property_counter.end(), + [execution_db_accessor, &vertex_degree_counter, &label_property_stats](const auto &counter_entry) { const auto &[label_property, values_map] = counter_entry; - std::vector result; - result.reserve(kDeleteStatisticsNumResults); // Extract info int64_t count_property_value = std::accumulate( values_map.begin(), values_map.end(), 0, @@ -1593,21 +1593,49 @@ std::vector> AnalyzeGraphQueryHandler::AnalyzeGraphCreat return prev_result + utils::ChiSquaredValue(value_entry.second, avg_group_size); }); double average_degree = (double)vertex_degree_counter[label_property] / count_property_value; - execution_db_accessor->SetIndexStats(label_property.first, label_property.second, - storage::LabelPropertyIndexStats{.count = count_property_value, - .statistic = chi_squared_stat, - .avg_group_size = avg_group_size, - .avg_degree = average_degree}); - // Save result - result.emplace_back(execution_db_accessor->LabelToName(label_property.first)); - result.emplace_back(execution_db_accessor->PropertyToName(label_property.second)); - result.emplace_back(count_property_value); - result.emplace_back(static_cast(values_map.size())); - result.emplace_back(avg_group_size); - result.emplace_back(chi_squared_stat); - result.emplace_back(average_degree); - results.push_back(std::move(result)); + + auto index_stats = + storage::LabelPropertyIndexStats{.count = count_property_value, + .distinct_values_count = static_cast(values_map.size()), + .statistic = chi_squared_stat, + .avg_group_size = avg_group_size, + .avg_degree = average_degree}; + execution_db_accessor->SetIndexStats(label_property.first, label_property.second, index_stats); + label_property_stats.push_back(std::make_pair(label_property, index_stats)); }); + + std::vector> results; + results.reserve(label_stats.size() + label_property_stats.size()); + + std::for_each(label_stats.begin(), label_stats.end(), [execution_db_accessor, &results](const auto &stat_entry) { + std::vector result; + result.reserve(kComputeStatisticsNumResults); + + result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first)); + result.emplace_back(TypedValue()); + result.emplace_back(stat_entry.second.count); + result.emplace_back(TypedValue()); + result.emplace_back(TypedValue()); + result.emplace_back(TypedValue()); + result.emplace_back(stat_entry.second.avg_degree); + results.push_back(std::move(result)); + }); + + std::for_each(label_property_stats.begin(), label_property_stats.end(), + [execution_db_accessor, &results](const auto &stat_entry) { + std::vector result; + result.reserve(kComputeStatisticsNumResults); + + result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first.first)); + result.emplace_back(execution_db_accessor->PropertyToName(stat_entry.first.second)); + result.emplace_back(stat_entry.second.count); + result.emplace_back(stat_entry.second.distinct_values_count); + result.emplace_back(stat_entry.second.avg_group_size); + result.emplace_back(stat_entry.second.statistic); + result.emplace_back(stat_entry.second.avg_degree); + results.push_back(std::move(result)); + }); + return results; } diff --git a/src/query/plan/rewrite/index_lookup.hpp b/src/query/plan/rewrite/index_lookup.hpp index 7c285dc94..1edc720ee 100644 --- a/src/query/plan/rewrite/index_lookup.hpp +++ b/src/query/plan/rewrite/index_lookup.hpp @@ -570,7 +570,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { // 10x less than the other one, always choose the smaller one. Otherwise, choose the index with smallest average group // size based on key distribution. If average group size is equal, choose the index that has distribution closer to // uniform distribution. Conditions based on average group size and key distribution can be only taken into account if - // the user has run `ANALYZE GRAPH` query before If the index cannot be found, nullopt is returned. + // the user has run `ANALYZE GRAPH` query before. If the index cannot be found, nullopt is returned. std::optional FindBestLabelPropertyIndex(const Symbol &symbol, const std::unordered_set &bound_symbols) { auto are_bound = [&bound_symbols](const auto &used_symbols) { @@ -630,8 +630,7 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor { }; int64_t vertex_count = db_->VerticesCount(GetLabel(label), GetProperty(property)); - std::optional new_stats = - db_->GetIndexStats(GetLabel(label), GetProperty(property)); + auto new_stats = db_->GetIndexStats(GetLabel(label), GetProperty(property)); // Conditions, from more to less important: // the index with 10x less vertices is better. diff --git a/src/storage/v2/indices.hpp b/src/storage/v2/indices.hpp index 78e27a953..a73e48f02 100644 --- a/src/storage/v2/indices.hpp +++ b/src/storage/v2/indices.hpp @@ -148,7 +148,7 @@ class LabelIndex { }; struct LabelPropertyIndexStats { - int64_t count; + int64_t count, distinct_values_count; double statistic, avg_group_size, avg_degree; };