tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add vertex degree to index statistics (#1026)

Browse Source 
Add graph analysis of vertex degrees when doing ANALYZE GRAPH.
This commit is contained in:
Josipmrden
2023-06-27 18:06:20 +02:00
committed by GitHub
parent 261aa4f49b
commit 84721f7e0a
19 changed files with 686 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/benchmark/query/planner.cpp
View File

@@ -131,7 +131,7 @@ static void BM_PlanAndEstimateIndexedMatching(benchmark::State &state) {
auto plans = memgraph::query::plan::MakeLogicalPlanForSingleQuery<memgraph::query::plan::VariableStartPlanner>(
query_parts, &ctx);
for (auto plan : plans) {
memgraph::query::plan::EstimatePlanCost(&dba, parameters, *plan);
memgraph::query::plan::EstimatePlanCost(&dba, symbol_table, parameters, *plan);
}
}
}
@@ -161,7 +161,7 @@ static void BM_PlanAndEstimateIndexedMatchingWithCachedCounts(benchmark::State &
auto plans = memgraph::query::plan::MakeLogicalPlanForSingleQuery<memgraph::query::plan::VariableStartPlanner>(
query_parts, &ctx);
for (auto plan : plans) {
memgraph::query::plan::EstimatePlanCost(&vertex_counts, parameters, *plan);
memgraph::query::plan::EstimatePlanCost(&vertex_counts, symbol_table, parameters, *plan);
}
}
}

12
tests/e2e/analyze_graph/common.py
View File

@@ -13,6 +13,7 @@ import typing
import mgclient
import pytest
from gqlalchemy import Memgraph
def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]:
@@ -27,3 +28,14 @@ def connect(**kwargs) -> mgclient.Connection:
yield connection
cursor = connection.cursor()
execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n")
@pytest.fixture
def memgraph(**kwargs) -> Memgraph:
memgraph = Memgraph()
yield memgraph
memgraph.drop_database()
memgraph.execute("analyze graph delete statistics;")
memgraph.drop_indexes()

218
tests/e2e/analyze_graph/optimize_indexes.py
View File

@@ -12,7 +12,10 @@
import sys
import pytest
from common import connect, execute_and_fetch_all
from common import connect, execute_and_fetch_all, memgraph
QUERY_PLAN = "QUERY PLAN"
# E2E tests for checking query semantic
# ------------------------------------
@@ -96,8 +99,8 @@ def test_analyze_full_graph(analyze_query, connect):
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 50, 5, 10, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 50, 5, 10, 0, 0)
# After analyzing graph, id1 index should be chosen because it has smaller average group size
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
@@ -131,8 +134,8 @@ def test_cardinality_different_avg_group_size_uniform_dist(connect):
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 20, 5, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 20, 5, 0, 0)
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
(f" * Filter",),
@@ -161,8 +164,8 @@ def test_cardinality_same_avg_group_size_uniform_dist_diff_vertex_count(connect)
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 50, 50, 1, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 100, 1, 0, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 50, 50, 1, 0, 0)
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
(f" * Filter",),
@@ -191,8 +194,8 @@ def test_large_diff_in_num_vertices_v1(connect):
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 1000, 1000, 1, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 99, 1, 99, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 1000, 1000, 1, 0, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 99, 1, 99, 0, 0)
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
(f" * Filter",),
@@ -221,8 +224,8 @@ def test_large_diff_in_num_vertices_v2(connect):
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 99, 1, 99, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 1000, 1000, 1, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 99, 1, 99, 0, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 1000, 1000, 1, 0, 0)
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
(f" * Filter",),
@@ -261,8 +264,8 @@ def test_same_avg_group_size_diff_distribution(connect):
else:
first_index = 1
# Check results
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 5, 20, 32.5)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 5, 20, 0)
assert analyze_graph_results[first_index] == ("Label", "id1", 100, 5, 20, 32.5, 0)
assert analyze_graph_results[1 - first_index] == ("Label", "id2", 100, 5, 20, 0, 0)
expected_explain_after_analysis = [
(f" * Produce {{n}}",),
(f" * Filter",),
@@ -278,5 +281,194 @@ def test_same_avg_group_size_diff_distribution(connect):
execute_and_fetch_all(cursor, "DROP INDEX ON :Label(id2);")
def test_given_supernode_when_expanding_then_expand_other_way_around(memgraph):
memgraph.execute("FOREACH (i in range(1, 1000) | CREATE (:Node {id: i}));")
memgraph.execute("CREATE (:SuperNode {id: 1});")
memgraph.execute("CREATE INDEX ON :SuperNode(id);")
memgraph.execute("CREATE INDEX ON :SuperNode;")
memgraph.execute("CREATE INDEX ON :Node(id);")
memgraph.execute("CREATE INDEX ON :Node;")
memgraph.execute("match (n:Node) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);")
query = "explain match (n:Node) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);"
expected_explain = [
f" * EmptyResult",
f" * Merge",
f" |\\ On Match",
f" | * Expand (s)-[anon3:HAS_REL_TO]->(n)",
f" | * Once",
f" |\\ On Create",
f" | * CreateExpand (n)<-[anon3:HAS_REL_TO]-(s)",
f" | * Once",
f" * ScanAllByLabel (n :Node)",
f" * ScanAllByLabelPropertyValue (s :SuperNode {{id}})",
f" * Once",
]
result_without_analysis = list(memgraph.execute_and_fetch(query))
result_without_analysis = [x[QUERY_PLAN] for x in result_without_analysis]
assert expected_explain == result_without_analysis
memgraph.execute("analyze graph;")
expected_explain = [
x.replace(f" | * Expand (s)-[anon3:HAS_REL_TO]->(n)", f" | * Expand (n)<-[anon3:HAS_REL_TO]-(s)")
for x in expected_explain
]
result_with_analysis = list(memgraph.execute_and_fetch(query))
result_with_analysis = [x[QUERY_PLAN] for x in result_with_analysis]
assert expected_explain == result_with_analysis
def test_given_supernode_when_subquery_then_carry_information_to_subquery(memgraph):
memgraph.execute("FOREACH (i in range(1, 1000) | CREATE (:Node {id: i}));")
memgraph.execute("FOREACH (i in range(1, 1000) | CREATE (:Node2 {id: i}));")
memgraph.execute("CREATE (:SuperNode {id: 1});")
memgraph.execute("CREATE INDEX ON :SuperNode(id);")
memgraph.execute("CREATE INDEX ON :SuperNode;")
memgraph.execute("CREATE INDEX ON :Node(id);")
memgraph.execute("CREATE INDEX ON :Node;")
memgraph.execute("CREATE INDEX ON :Node2(id);")
memgraph.execute("CREATE INDEX ON :Node2;")
memgraph.execute("match (n:Node) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);")
memgraph.execute("match (n:Node2) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);")
query = (
"explain match (n:Node) match (s:SuperNode {id: 1}) call { with n, s merge (n)<-[:HAS_REL_TO]-(s) } return 1"
)
expected_explain = [
f" * Produce {{0}}",
f" * Accumulate",
f" * Accumulate",
f" * Apply",
f" |\\ ",
f" | * EmptyResult",
f" | * Merge",
f" | |\\ On Match",
f" | | * Expand (s)-[anon3:HAS_REL_TO]->(n)",
f" | | * Once",
f" | |\\ On Create",
f" | | * CreateExpand (n)<-[anon3:HAS_REL_TO]-(s)",
f" | | * Once",
f" | * Produce {{n, s}}",
f" | * Once",
f" * ScanAllByLabel (n :Node)",
f" * ScanAllByLabelPropertyValue (s :SuperNode {{id}})",
f" * Once",
]
result_without_analysis = list(memgraph.execute_and_fetch(query))
result_without_analysis = [x[QUERY_PLAN] for x in result_without_analysis]
assert expected_explain == result_without_analysis
memgraph.execute("analyze graph;")
expected_explain = [
x.replace(f" | | * Expand (s)-[anon3:HAS_REL_TO]->(n)", f" | | * Expand (n)<-[anon3:HAS_REL_TO]-(s)")
for x in expected_explain
]
result_with_analysis = list(memgraph.execute_and_fetch(query))
result_with_analysis = [x[QUERY_PLAN] for x in result_with_analysis]
assert expected_explain == result_with_analysis
def test_given_supernode_when_subquery_and_union_then_carry_information(memgraph):
memgraph.execute("FOREACH (i in range(1, 1000) | CREATE (:Node {id: i}));")
memgraph.execute("FOREACH (i in range(1, 1000) | CREATE (:Node2 {id: i}));")
memgraph.execute("CREATE (:SuperNode {id: 1});")
memgraph.execute("CREATE INDEX ON :SuperNode(id);")
memgraph.execute("CREATE INDEX ON :SuperNode;")
memgraph.execute("CREATE INDEX ON :Node(id);")
memgraph.execute("CREATE INDEX ON :Node;")
memgraph.execute("CREATE INDEX ON :Node2(id);")
memgraph.execute("CREATE INDEX ON :Node2;")
memgraph.execute("match (n:Node) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);")
memgraph.execute("match (n:Node2) match (s:SuperNode {id: 1}) merge (n)<-[:HAS_REL_TO]-(s);")
query = "explain match (n:Node) match (s:SuperNode {id: 1}) call { with n, s merge (n)<-[:HAS_REL_TO]-(s) } return s union all match (n:Node) match (s:SuperNode {id: 1}) call { with n, s merge (n)<-[:HAS_REL_TO]-(s) } return s;"
expected_explain = [
f" * Union {{s : s}}",
f" |\\ ",
f" | * Produce {{s}}",
f" | * Accumulate",
f" | * Accumulate",
f" | * Apply",
f" | |\\ ",
f" | | * EmptyResult",
f" | | * Merge",
f" | | |\\ On Match",
f" | | | * Expand (s)-[anon7:HAS_REL_TO]->(n)",
f" | | | * Once",
f" | | |\\ On Create",
f" | | | * CreateExpand (n)<-[anon7:HAS_REL_TO]-(s)",
f" | | | * Once",
f" | | * Produce {{n, s}}",
f" | | * Once",
f" | * ScanAllByLabel (n :Node)",
f" | * ScanAllByLabelPropertyValue (s :SuperNode {{id}})",
f" | * Once",
f" * Produce {{s}}",
f" * Accumulate",
f" * Accumulate",
f" * Apply",
f" |\\ ",
f" | * EmptyResult",
f" | * Merge",
f" | |\\ On Match",
f" | | * Expand (s)-[anon3:HAS_REL_TO]->(n)",
f" | | * Once",
f" | |\\ On Create",
f" | | * CreateExpand (n)<-[anon3:HAS_REL_TO]-(s)",
f" | | * Once",
f" | * Produce {{n, s}}",
f" | * Once",
f" * ScanAllByLabel (n :Node)",
f" * ScanAllByLabelPropertyValue (s :SuperNode {{id}})",
f" * Once",
]
result_without_analysis = list(memgraph.execute_and_fetch(query))
result_without_analysis = [x[QUERY_PLAN] for x in result_without_analysis]
assert expected_explain == result_without_analysis
memgraph.execute("analyze graph;")
expected_explain = [
x.replace(f" | | * Expand (s)-[anon3:HAS_REL_TO]->(n)", f" | | * Expand (n)<-[anon3:HAS_REL_TO]-(s)")
for x in expected_explain
]
expected_explain = [
x.replace(f" | | | * Expand (s)-[anon7:HAS_REL_TO]->(n)", f" | | | * Expand (n)<-[anon7:HAS_REL_TO]-(s)")
for x in expected_explain
]
result_with_analysis = list(memgraph.execute_and_fetch(query))
result_with_analysis = [x[QUERY_PLAN] for x in result_with_analysis]
assert expected_explain == result_with_analysis
def test_given_empty_graph_when_analyzing_graph_return_zero_degree(memgraph):
memgraph.execute("CREATE INDEX ON :Node;")
label_stats = next(memgraph.execute_and_fetch("analyze graph;"))
expected_analysis = {
"label": "Node",
"property": None,
"num estimation nodes": 0,
"num groups": None,
"avg group size": None,
"chi-squared value": None,
"avg degree": 0.0,
}
assert set(label_stats) == set(expected_analysis)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

11
tests/manual/interactive_planning.cpp
View File

@@ -27,6 +27,7 @@
#include "query/plan/planner.hpp"
#include "query/plan/pretty_print.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/indices.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/string.hpp"
@@ -213,8 +214,12 @@ class InteractiveDbAccessor {
return label_property_index_.at(key);
}
std::optional<memgraph::storage::IndexStats> GetIndexStats(memgraph::storage::LabelId label,
memgraph::storage::PropertyId property) const {
std::optional<memgraph::storage::LabelIndexStats> GetIndexStats(const memgraph::storage::LabelId label) const {
return dba_->GetIndexStats(label);
}
std::optional<memgraph::storage::LabelPropertyIndexStats> GetIndexStats(
const memgraph::storage::LabelId label, const memgraph::storage::PropertyId property) const {
return dba_->GetIndexStats(label, property);
}
@@ -458,7 +463,7 @@ auto MakeLogicalPlans(memgraph::query::CypherQuery *query, memgraph::query::AstS
memgraph::query::AstStorage ast_copy;
auto unoptimized_plan = plan->Clone(&ast_copy);
auto rewritten_plan = post_process.Rewrite(std::move(plan), &ctx);
double cost = post_process.EstimatePlanCost(rewritten_plan, dba);
double cost = post_process.EstimatePlanCost(rewritten_plan, dba, symbol_table);
interactive_plans.push_back(
InteractivePlan{std::move(unoptimized_plan), std::move(ast_copy), std::move(rewritten_plan), cost});
}

4
tests/unit/query_cost_estimator.cpp
View File

@@ -74,7 +74,7 @@ class QueryCostEstimator : public ::testing::Test {
}
auto Cost() {
CostEstimator<memgraph::query::DbAccessor> cost_estimator(&*dba, parameters_);
CostEstimator<memgraph::query::DbAccessor> cost_estimator(&*dba, symbol_table_, parameters_);
last_op_->Accept(cost_estimator);
return cost_estimator.cost();
}
@@ -201,7 +201,7 @@ TEST_F(QueryCostEstimator, SubqueryCartesian) {
std::shared_ptr<LogicalOperator> input = std::make_shared<ScanAll>(std::make_shared<Once>(), NextSymbol());
std::shared_ptr<LogicalOperator> subquery = std::make_shared<ScanAll>(std::make_shared<Once>(), NextSymbol());
MakeOp<memgraph::query::plan::Apply>(input, subquery, true);
EXPECT_COST(CostParam::kSubquery * no_vertices * no_vertices);
EXPECT_COST(CostParam::kSubquery * no_vertices * no_vertices + no_vertices);
}
TEST_F(QueryCostEstimator, UnitSubquery) {

10
tests/unit/query_plan_checker.hpp
View File

@@ -500,9 +500,13 @@ class FakeDbAccessor {
return false;
}
memgraph::storage::IndexStats GetIndexStats(memgraph::storage::LabelId label,
memgraph::storage::PropertyId property) const {
return memgraph::storage::IndexStats{.statistic = 0, .avg_group_size = 1}; // unique id
std::optional<memgraph::storage::LabelPropertyIndexStats> GetIndexStats(
const memgraph::storage::LabelId label, const memgraph::storage::PropertyId property) const {
return memgraph::storage::LabelPropertyIndexStats{.statistic = 0, .avg_group_size = 1}; // unique id
}
std::optional<memgraph::storage::LabelIndexStats> GetIndexStats(const memgraph::storage::LabelId label) const {
return memgraph::storage::LabelIndexStats{.count = 0, .avg_degree = 0}; // unique id
}
void SetIndexCount(memgraph::storage::LabelId label, int64_t count) { label_index_[label] = count; }

7
tests/unit/query_semantic.cpp
View File

@@ -1252,4 +1252,11 @@ TEST_F(TestSymbolGenerator, Subqueries) {
query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), CALL_SUBQUERY(subquery), RETURN("n", "m")));
symbol_table = MakeSymbolTable(query);
ASSERT_EQ(symbol_table.max_position(), 11);
// MATCH (n) CALL { MATCH (s) RETURN s } RETURN n UNION MATCH (n) CALL { MATCH (s) RETURN s } RETURN n
subquery = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("s"))), RETURN("s")));
query = QUERY(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), CALL_SUBQUERY(subquery), RETURN("n")),
UNION(SINGLE_QUERY(MATCH(PATTERN(NODE("n"))), CALL_SUBQUERY(subquery), RETURN("n"))));
symbol_table = MakeSymbolTable(query);
ASSERT_EQ(symbol_table.max_position(), 13);
}