Compare commits

...

22 Commits

Author SHA1 Message Date
Josip Mrden
b507cfea36 Merge branch 'fix-mg-assert-pool-error' into performance-improvements 2023-09-29 10:19:56 +02:00
Josip Mrden
b30a8671b9 Refactor cost estimator 2023-09-27 17:10:12 +02:00
Josip Mrden
c836c5084c Refactor index lookup 2023-09-27 17:09:07 +02:00
Josip Mrden
5a0d0d430c Remove cartesian expansion flag 2023-09-27 17:04:22 +02:00
Josip Mrden
1fe516023b Rename modified symbols for providing context to IndexedJoin 2023-09-27 17:04:09 +02:00
Josip Mrden
5cdd6230d9 Make IndexedJoin printable 2023-09-27 17:03:26 +02:00
Josip Mrden
5163b95913 Make removal of expressions more clear 2023-09-27 14:31:21 +02:00
Josip Mrden
86b453b666 Merge branch 'master' into return-cartesian-product-on-multi-match-clauses 2023-09-27 11:14:02 +02:00
antoniofilipovic
48b835eb24 fix potential bug 2023-09-26 14:51:59 +02:00
Josip Mrden
eef57b1ffb Add IndexedJoin operator to cope with joins that were implicit in Memgraph 2023-09-13 19:33:42 +02:00
Josip Mrden
72c30183ee Merge branch 'master' into return-cartesian-product-on-multi-match-clauses 2023-09-13 16:59:32 +02:00
Josip Mrden
94006328bb Add cartesian expansion flag 2023-09-11 11:32:22 +02:00
Josip Mrden
be48e8dc51 Add cartesian expansion flag 2023-09-11 11:32:05 +02:00
Josip Mrden
609e45e2a2 Adjust cost estimator 2023-09-09 23:32:41 +02:00
Josipmrden
2aab709101 Merge branch 'master' into return-cartesian-product-on-multi-match-clauses 2023-09-09 18:23:27 +02:00
Josip Mrden
7d856e0168 Merge branch 'master' into return-cartesian-product-on-multi-match-clauses 2023-09-05 15:30:02 +02:00
Josip Mrden
0ad11b4ca3 Add correct implementation of the cartesian feature 2023-09-04 08:44:06 +02:00
Josip Mrden
1970d4e8f2 Merge branch 'master' into return-cartesian-product-on-multi-match-clauses 2023-09-04 08:34:46 +02:00
Josip Mrden
a9151b2b42 Corrected isomorphism picking in cartesian products 2023-08-31 19:00:53 +02:00
Josip Mrden
e14444a1ac Make gqlbehave tests pass 2023-08-28 13:08:22 +02:00
Josip Mrden
7fe0d920a9 Add cartesian product logic 2023-08-28 10:33:30 +02:00
Josip Mrden
7448c51e46 Add cartesian product 2023-08-23 18:50:26 +02:00
15 changed files with 633 additions and 100 deletions

View File

@@ -38,6 +38,14 @@ struct Scope {
std::unordered_map<std::string, SymbolStatistics> symbol_stats;
};
struct CostEstimation {
// expense of running the query
double cost;
// expected number of rows
double cardinality;
};
/**
* Query plan execution time cost estimator, for comparing and choosing optimal
* execution plans.
@@ -90,6 +98,7 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
static constexpr double kExpand{3.0};
static constexpr double kExpandVariable{9.0};
static constexpr double kFilter{0.25};
static constexpr double kIndexedJoin{0.25};
static constexpr double kEdgeUniquenessFilter{0.95};
};
@@ -271,12 +280,12 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
}
bool PreVisit(Union &op) override {
double left_cost = EstimateCostOnBranch(&op.left_op_);
double right_cost = EstimateCostOnBranch(&op.right_op_);
auto left_estimation = EstimateCostOnBranch(&op.left_op_);
auto right_estimation = EstimateCostOnBranch(&op.right_op_);
// the number of hits in the previous operator should be the joined number of results of both parts of the union
cardinality_ *= (left_cost + right_cost);
IncrementCost(CostParam::kUnion);
cost_ = left_estimation.cost + right_estimation.cost;
cardinality_ = left_estimation.cardinality + right_estimation.cardinality;
return false;
}
@@ -303,11 +312,41 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
// Estimate cost on the subquery branch independently, use a copy
auto &last_scope = scopes_.back();
double subquery_cost = EstimateCostOnBranch(&op.subquery_, last_scope);
subquery_cost = !utils::ApproxEqualDecimal(subquery_cost, 0.0) ? subquery_cost : 1;
cardinality_ *= subquery_cost;
auto subquery_estimation = EstimateCostOnBranch(&op.subquery_, last_scope);
double subquery_cost = !utils::ApproxEqualDecimal(subquery_estimation.cost, 0.0) ? subquery_estimation.cost : 1;
IncrementCost(subquery_cost);
IncrementCost(CostParam::kSubquery);
double subquery_cardinality =
!utils::ApproxEqualDecimal(subquery_estimation.cardinality, 0.0) ? subquery_estimation.cardinality : 1;
cardinality_ *= subquery_cardinality;
return false;
}
bool PreVisit(Cartesian &op) override {
// Get the cost of the main branch
op.left_op_->Accept(*this);
// add cost from the right branch and multiply cardinalities
auto cost_estimation = EstimateCostOnBranch(&op.right_op_);
cost_ += cost_estimation.cost;
auto right_cardinality =
!utils::ApproxEqualDecimal(cost_estimation.cardinality, 0.0) ? cost_estimation.cardinality : 1;
cardinality_ *= right_cardinality;
return false;
}
bool PreVisit(IndexedJoin &op) override {
// Get the cost of the main branch
op.left_->Accept(*this);
// add cost from the right branch and multiply cardinalities
auto cost_estimation = EstimateCostOnBranch(&op.right_);
cost_ += cost_estimation.cost;
auto right_cardinality =
!utils::ApproxEqualDecimal(cost_estimation.cardinality, 0.0) ? cost_estimation.cardinality : 1;
cardinality_ *= right_cardinality * CardParam::kIndexedJoin;
return false;
}
@@ -339,16 +378,16 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
void IncrementCost(double param) { cost_ += param * cardinality_; }
double EstimateCostOnBranch(std::shared_ptr<LogicalOperator> *branch) {
CostEstimation EstimateCostOnBranch(std::shared_ptr<LogicalOperator> *branch) {
CostEstimator<TDbAccessor> cost_estimator(db_accessor_, table_, parameters);
(*branch)->Accept(cost_estimator);
return cost_estimator.cost();
return CostEstimation{.cost = cost_estimator.cost(), .cardinality = cost_estimator.cardinality()};
}
double EstimateCostOnBranch(std::shared_ptr<LogicalOperator> *branch, Scope scope) {
CostEstimation EstimateCostOnBranch(std::shared_ptr<LogicalOperator> *branch, Scope scope) {
CostEstimator<TDbAccessor> cost_estimator(db_accessor_, table_, parameters, scope);
(*branch)->Accept(cost_estimator);
return cost_estimator.cost();
return CostEstimation{.cost = cost_estimator.cost(), .cardinality = cost_estimator.cardinality()};
}
// converts an optional ScanAll range bound into a property value

View File

@@ -130,6 +130,7 @@ extern const Event ForeachOperator;
extern const Event EmptyResultOperator;
extern const Event EvaluatePatternFilterOperator;
extern const Event ApplyOperator;
extern const Event IndexedJoinOperator;
} // namespace memgraph::metrics
namespace memgraph::query::plan {
@@ -5179,4 +5180,65 @@ void Apply::ApplyCursor::Reset() {
pull_input_ = true;
}
IndexedJoin::IndexedJoin(const std::shared_ptr<LogicalOperator> left, const std::shared_ptr<LogicalOperator> right)
: left_(left ? left : std::make_shared<Once>()), right_(right) {}
WITHOUT_SINGLE_INPUT(IndexedJoin);
bool IndexedJoin::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
if (visitor.PreVisit(*this)) {
left_->Accept(visitor) && right_->Accept(visitor);
}
return visitor.PostVisit(*this);
}
UniqueCursorPtr IndexedJoin::MakeCursor(utils::MemoryResource *mem) const {
memgraph::metrics::IncrementCounter(memgraph::metrics::IndexedJoinOperator);
return MakeUniqueCursorPtr<IndexedJoinCursor>(mem, *this, mem);
}
IndexedJoin::IndexedJoinCursor::IndexedJoinCursor(const IndexedJoin &self, utils::MemoryResource *mem)
: self_(self), left_(self.left_->MakeCursor(mem)), right_(self.right_->MakeCursor(mem)) {}
std::vector<Symbol> IndexedJoin::ModifiedSymbols(const SymbolTable &table) const {
// Since Apply is the Cartesian product, modified symbols are combined from
// both execution branches.
auto symbols = left_->ModifiedSymbols(table);
auto subquery_symbols = right_->ModifiedSymbols(table);
symbols.insert(symbols.end(), subquery_symbols.begin(), subquery_symbols.end());
return symbols;
}
bool IndexedJoin::IndexedJoinCursor::Pull(Frame &frame, ExecutionContext &context) {
SCOPED_PROFILE_OP("IndexedJoin");
while (true) {
if (pull_input_ && !left_->Pull(frame, context)) {
return false;
};
if (right_->Pull(frame, context)) {
// if successful, next Pull from this should not pull_input_
pull_input_ = false;
return true;
}
// failed to pull from subquery cursor
// skip that row
pull_input_ = true;
right_->Reset();
}
}
void IndexedJoin::IndexedJoinCursor::Shutdown() {
left_->Shutdown();
right_->Shutdown();
}
void IndexedJoin::IndexedJoinCursor::Reset() {
left_->Reset();
right_->Reset();
pull_input_ = true;
}
} // namespace memgraph::query::plan

View File

@@ -129,6 +129,7 @@ class Foreach;
class EmptyResult;
class EvaluatePatternFilter;
class Apply;
class IndexedJoin;
using LogicalOperatorCompositeVisitor =
utils::CompositeVisitor<Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel, ScanAllByLabelPropertyRange,
@@ -136,7 +137,7 @@ using LogicalOperatorCompositeVisitor =
ConstructNamedPath, Filter, Produce, Delete, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit,
OrderBy, Merge, Optional, Unwind, Distinct, Union, Cartesian, CallProcedure, LoadCsv,
Foreach, EmptyResult, EvaluatePatternFilter, Apply>;
Foreach, EmptyResult, EvaluatePatternFilter, Apply, IndexedJoin>;
using LogicalOperatorLeafVisitor = utils::LeafVisitor<Once>;
@@ -2477,6 +2478,49 @@ class Apply : public memgraph::query::plan::LogicalOperator {
};
};
/// Applies symbols from both output branches.
class IndexedJoin : public memgraph::query::plan::LogicalOperator {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
IndexedJoin() {}
IndexedJoin(std::shared_ptr<LogicalOperator> left, std::shared_ptr<LogicalOperator> right);
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override;
std::vector<Symbol> ModifiedSymbols(const SymbolTable &) const override;
bool HasSingleInput() const override;
std::shared_ptr<LogicalOperator> input() const override;
void set_input(std::shared_ptr<LogicalOperator>) override;
std::shared_ptr<memgraph::query::plan::LogicalOperator> left_;
std::shared_ptr<memgraph::query::plan::LogicalOperator> right_;
std::unique_ptr<LogicalOperator> Clone(AstStorage *storage) const override {
auto object = std::make_unique<IndexedJoin>();
object->left_ = left_ ? left_->Clone(storage) : nullptr;
object->right_ = right_ ? right_->Clone(storage) : nullptr;
return object;
}
private:
class IndexedJoinCursor : public Cursor {
public:
IndexedJoinCursor(const IndexedJoin &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
private:
const IndexedJoin &self_;
UniqueCursorPtr left_;
UniqueCursorPtr right_;
bool pull_input_{true};
};
};
} // namespace plan
} // namespace query
} // namespace memgraph

View File

@@ -148,4 +148,7 @@ constexpr utils::TypeInfo query::plan::Foreach::kType{utils::TypeId::FOREACH, "F
constexpr utils::TypeInfo query::plan::Apply::kType{utils::TypeId::APPLY, "Apply",
&query::plan::LogicalOperator::kType};
constexpr utils::TypeInfo query::plan::IndexedJoin::kType{utils::TypeId::INDEXED_JOIN, "IndexedJoin",
&query::plan::LogicalOperator::kType};
} // namespace memgraph

View File

@@ -55,29 +55,33 @@ void ForEachPattern(Pattern &pattern, std::function<void(NodeAtom *)> base,
// want to start expanding.
std::vector<Expansion> NormalizePatterns(const SymbolTable &symbol_table, const std::vector<Pattern *> &patterns) {
std::vector<Expansion> expansions;
ExpansionId unknown_expansion_id = ExpansionId::FromInt(-1);
auto ignore_node = [&](auto *) {};
auto collect_expansion = [&](auto *prev_node, auto *edge, auto *current_node) {
UsedSymbolsCollector collector(symbol_table);
if (edge->IsVariable()) {
if (edge->lower_bound_) edge->lower_bound_->Accept(collector);
if (edge->upper_bound_) edge->upper_bound_->Accept(collector);
if (edge->filter_lambda_.expression) edge->filter_lambda_.expression->Accept(collector);
// Remove symbols which are bound by lambda arguments.
collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_edge));
collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_node));
if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH || edge->type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS) {
collector.symbols_.erase(symbol_table.at(*edge->weight_lambda_.inner_edge));
collector.symbols_.erase(symbol_table.at(*edge->weight_lambda_.inner_node));
}
}
expansions.emplace_back(Expansion{prev_node, edge, edge->direction_, false, collector.symbols_, current_node});
};
for (const auto &pattern : patterns) {
for (size_t i = 0, size = patterns.size(); i < size; i++) {
const auto &pattern = patterns[i];
if (pattern->atoms_.size() == 1U) {
auto *node = utils::Downcast<NodeAtom>(pattern->atoms_[0]);
DMG_ASSERT(node, "First pattern atom is not a node");
expansions.emplace_back(Expansion{node});
expansions.emplace_back(Expansion{.node1 = node, .isomorphic_id = unknown_expansion_id});
} else {
auto collect_expansion = [&](auto *prev_node, auto *edge, auto *current_node) {
UsedSymbolsCollector collector(symbol_table);
if (edge->IsVariable()) {
if (edge->lower_bound_) edge->lower_bound_->Accept(collector);
if (edge->upper_bound_) edge->upper_bound_->Accept(collector);
if (edge->filter_lambda_.expression) edge->filter_lambda_.expression->Accept(collector);
// Remove symbols which are bound by lambda arguments.
collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_edge));
collector.symbols_.erase(symbol_table.at(*edge->filter_lambda_.inner_node));
if (edge->type_ == EdgeAtom::Type::WEIGHTED_SHORTEST_PATH ||
edge->type_ == EdgeAtom::Type::ALL_SHORTEST_PATHS) {
collector.symbols_.erase(symbol_table.at(*edge->weight_lambda_.inner_edge));
collector.symbols_.erase(symbol_table.at(*edge->weight_lambda_.inner_node));
}
}
expansions.emplace_back(Expansion{prev_node, edge, edge->direction_, false, collector.symbols_, current_node,
unknown_expansion_id});
};
ForEachPattern(*pattern, ignore_node, collect_expansion);
}
}
@@ -487,9 +491,26 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
// were in a Where clause).
void AddMatching(const std::vector<Pattern *> &patterns, Where *where, SymbolTable &symbol_table, AstStorage &storage,
Matching &matching) {
ExpansionId next_isomorphic_id = ExpansionId::FromUint(matching.number_of_isomorphisms + 1);
auto assign_isomorphic_id = [&matching, &next_isomorphic_id](Symbol symbol, Expansion &expansion) {
auto isomorphic_id_to_assign = next_isomorphic_id;
if (matching.node_symbol_to_isomorphic_id.contains(symbol)) {
isomorphic_id_to_assign = matching.node_symbol_to_isomorphic_id[symbol];
}
if (expansion.isomorphic_id.AsInt() == -1) {
expansion.isomorphic_id = isomorphic_id_to_assign;
} else if (isomorphic_id_to_assign.AsInt() < expansion.isomorphic_id.AsInt()) {
expansion.isomorphic_id = isomorphic_id_to_assign;
}
matching.node_symbol_to_isomorphic_id[symbol] = expansion.isomorphic_id;
};
auto expansions = NormalizePatterns(symbol_table, patterns);
std::unordered_set<Symbol> edge_symbols;
for (const auto &expansion : expansions) {
for (auto &expansion : expansions) {
// Matching may already have some expansions, so offset our index.
const size_t expansion_ix = matching.expansions.size();
// Map node1 symbol to expansion
@@ -497,6 +518,9 @@ void AddMatching(const std::vector<Pattern *> &patterns, Where *where, SymbolTab
matching.node_symbol_to_expansions[node1_sym].insert(expansion_ix);
// Add node1 to all symbols.
matching.expansion_symbols.insert(node1_sym);
assign_isomorphic_id(node1_sym, expansion);
if (expansion.edge) {
const auto &edge_sym = symbol_table.at(*expansion.edge->identifier_);
// Fill edge symbols for Cyphermorphism.
@@ -507,12 +531,22 @@ void AddMatching(const std::vector<Pattern *> &patterns, Where *where, SymbolTab
// Add edge and node2 to all symbols
matching.expansion_symbols.insert(edge_sym);
matching.expansion_symbols.insert(node2_sym);
assign_isomorphic_id(edge_sym, expansion);
assign_isomorphic_id(node2_sym, expansion);
}
matching.expansions.push_back(expansion);
matching.number_of_isomorphisms = matching.number_of_isomorphisms < expansion.isomorphic_id.AsUint()
? expansion.isomorphic_id.AsUint()
: matching.number_of_isomorphisms;
next_isomorphic_id = ExpansionId::FromUint(matching.number_of_isomorphisms + 1);
}
if (!edge_symbols.empty()) {
matching.edge_symbols.emplace_back(edge_symbols);
}
for (auto *const pattern : patterns) {
matching.filters.CollectPatternFilters(*pattern, symbol_table, storage);
if (pattern->identifier_->user_declared_) {
@@ -525,6 +559,10 @@ void AddMatching(const std::vector<Pattern *> &patterns, Where *where, SymbolTab
if (where) {
matching.filters.CollectWhereFilter(*where, symbol_table);
}
for (const auto &expansion : matching.expansions) {
MG_ASSERT(expansion.isomorphic_id.AsInt() != -1, "Expansion isomorphic ID is not assigned to the pattern!");
}
}
void AddMatching(const Match &match, SymbolTable &symbol_table, AstStorage &storage, Matching &matching) {

View File

@@ -103,6 +103,35 @@ class UsedSymbolsCollector : public HierarchicalTreeVisitor {
bool in_exists{false};
};
#define PREPROCESS_DEFINE_ID_TYPE(name) \
class name final { \
private: \
explicit name(uint64_t id) : id_(id) {} \
\
public: \
/* Default constructor to allow serialization or preallocation. */ \
name() = default; \
\
static name FromUint(uint64_t id) { return name{id}; } \
static name FromInt(int64_t id) { return name{utils::MemcpyCast<uint64_t>(id)}; } \
uint64_t AsUint() const { return id_; } \
int64_t AsInt() const { return utils::MemcpyCast<int64_t>(id_); } \
\
private: \
uint64_t id_; \
}; \
static_assert(std::is_trivially_copyable<name>::value, "query::plan::" #name " must be trivially copyable!"); \
inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \
inline bool operator!=(const name &first, const name &second) { return first.AsUint() != second.AsUint(); } \
inline bool operator<(const name &first, const name &second) { return first.AsUint() < second.AsUint(); } \
inline bool operator>(const name &first, const name &second) { return first.AsUint() > second.AsUint(); } \
inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \
inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); }
PREPROCESS_DEFINE_ID_TYPE(ExpansionId);
#undef STORAGE_DEFINE_ID_TYPE
/// Normalized representation of a pattern that needs to be matched.
struct Expansion {
/// The first node in the expansion, it can be a single node.
@@ -119,6 +148,7 @@ struct Expansion {
/// Optional node at the other end of an edge. If the expansion
/// contains an edge, then this node is required.
NodeAtom *node2 = nullptr;
ExpansionId isomorphic_id = ExpansionId();
};
struct FilterMatching;
@@ -394,6 +424,9 @@ struct Matching {
Filters filters;
/// Maps node symbols to expansions which bind them.
std::unordered_map<Symbol, std::set<size_t>> node_symbol_to_expansions{};
size_t number_of_isomorphisms{0};
std::unordered_map<Symbol, ExpansionId> node_symbol_to_isomorphic_id{};
/// Maps named path symbols to a vector of Symbols that define its pattern.
std::unordered_map<Symbol, std::vector<Symbol>> named_paths{};
/// All node and edge symbols across all expansions (from all matches).

View File

@@ -101,11 +101,19 @@ PRE_VISIT(SetProperties);
PRE_VISIT(SetLabels);
PRE_VISIT(RemoveProperty);
PRE_VISIT(RemoveLabels);
PRE_VISIT(EdgeUniquenessFilter);
PRE_VISIT(Accumulate);
PRE_VISIT(EmptyResult);
PRE_VISIT(EvaluatePatternFilter);
bool PlanPrinter::PreVisit(query::plan::EdgeUniquenessFilter &op) {
WithPrintLn([&](auto &out) {
out << "* EdgeUniquenessFilter [";
utils::PrintIterable(out, op.previous_symbols_, ", ", [](auto &out, const auto &sym) { out << sym.name(); });
out << " != " << op.expand_symbol_.name() << "]";
});
return true;
}
bool PlanPrinter::PreVisit(query::plan::Aggregate &op) {
WithPrintLn([&](auto &out) { out << "* " << op.ToString(); });
return true;
@@ -194,6 +202,13 @@ bool PlanPrinter::PreVisit(query::plan::Apply &op) {
op.input_->Accept(*this);
return false;
}
bool PlanPrinter::PreVisit(query::plan::IndexedJoin &op) {
WithPrintLn([](auto &out) { out << "* IndexedJoin"; });
Branch(*op.right_);
op.left_->Accept(*this);
return false;
}
#undef PRE_VISIT
bool PlanPrinter::DefaultPreVisit() {
@@ -921,6 +936,20 @@ bool PlanToJsonVisitor::PreVisit(Apply &op) {
return false;
}
bool PlanToJsonVisitor::PreVisit(IndexedJoin &op) {
json self;
self["name"] = "IndexedJoin";
op.left_->Accept(*this);
self["left"] = PopOutput();
op.right_->Accept(*this);
self["right"] = PopOutput();
output_ = std::move(self);
return false;
}
} // namespace impl
} // namespace memgraph::query::plan

View File

@@ -96,6 +96,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(LoadCsv &) override;
bool PreVisit(Foreach &) override;
bool PreVisit(Apply & /*unused*/) override;
bool PreVisit(IndexedJoin & /*unused*/) override;
bool Visit(Once &) override;
@@ -192,6 +193,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(EdgeUniquenessFilter &) override;
bool PreVisit(Cartesian &) override;
bool PreVisit(Apply & /*unused*/) override;
bool PreVisit(IndexedJoin & /*unused*/) override;
bool PreVisit(ScanAll &) override;
bool PreVisit(ScanAllByLabel &) override;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,30 +22,50 @@ DEFINE_VALIDATED_int64(query_vertex_count_to_expand_existing, 10,
namespace memgraph::query::plan::impl {
Expression *RemoveAndExpressions(Expression *expr, const std::unordered_set<Expression *> &exprs_to_remove) {
ExpressionRemovalResult RemoveAndExpressions(Expression *expr,
const std::unordered_set<Expression *> &exprs_to_remove) {
auto *and_op = utils::Downcast<AndOperator>(expr);
if (!and_op) return expr;
// currently we are processing expressions by dividing them into and disjoint expressions
// no work needed if there is no multiple and expressions
if (!and_op) return ExpressionRemovalResult{.trimmed_expression = expr};
// and operation is fully contained inside the expressions to remove
if (utils::Contains(exprs_to_remove, and_op)) {
return nullptr;
return ExpressionRemovalResult{.trimmed_expression = nullptr, .did_remove = true};
}
bool did_remove = false;
if (utils::Contains(exprs_to_remove, and_op->expression1_)) {
and_op->expression1_ = nullptr;
did_remove = true;
}
if (utils::Contains(exprs_to_remove, and_op->expression2_)) {
and_op->expression2_ = nullptr;
did_remove = true;
}
and_op->expression1_ = RemoveAndExpressions(and_op->expression1_, exprs_to_remove);
and_op->expression2_ = RemoveAndExpressions(and_op->expression2_, exprs_to_remove);
auto removal1 = RemoveAndExpressions(and_op->expression1_, exprs_to_remove);
and_op->expression1_ = removal1.trimmed_expression;
did_remove = did_remove || removal1.did_remove;
auto removal2 = RemoveAndExpressions(and_op->expression2_, exprs_to_remove);
and_op->expression2_ = removal2.trimmed_expression;
did_remove = did_remove || removal2.did_remove;
if (!and_op->expression1_ && !and_op->expression2_) {
return nullptr;
return ExpressionRemovalResult{.trimmed_expression = nullptr, .did_remove = did_remove};
}
if (and_op->expression1_ && !and_op->expression2_) {
return and_op->expression1_;
return ExpressionRemovalResult{.trimmed_expression = and_op->expression1_, .did_remove = did_remove};
}
if (and_op->expression2_ && !and_op->expression1_) {
return and_op->expression2_;
return ExpressionRemovalResult{.trimmed_expression = and_op->expression2_, .did_remove = did_remove};
}
return and_op;
return ExpressionRemovalResult{.trimmed_expression = and_op, .did_remove = did_remove};
}
} // namespace memgraph::query::plan::impl

View File

@@ -34,9 +34,14 @@ namespace memgraph::query::plan {
namespace impl {
struct ExpressionRemovalResult {
Expression *trimmed_expression;
bool did_remove{false};
};
// Return the new root expression after removing the given expressions from the
// given expression tree.
Expression *RemoveAndExpressions(Expression *expr, const std::unordered_set<Expression *> &exprs_to_remove);
ExpressionRemovalResult RemoveAndExpressions(Expression *expr, const std::unordered_set<Expression *> &exprs_to_remove);
template <class TDbAccessor>
class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
@@ -61,10 +66,31 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
// free the memory.
bool PostVisit(Filter &op) override {
prev_ops_.pop_back();
op.expression_ = RemoveAndExpressions(op.expression_, filter_exprs_for_removal_);
if (!op.expression_ || utils::Contains(filter_exprs_for_removal_, op.expression_)) {
SetOnParent(op.input());
ExpressionRemovalResult removal = RemoveAndExpressions(op.expression_, filter_exprs_for_removal_);
op.expression_ = removal.trimmed_expression;
bool is_child_cartesian = op.input()->GetTypeInfo() == Cartesian::kType;
if (!removal.did_remove) {
// nothing to be replaced, filter will stay
return true;
}
if (is_child_cartesian) {
// if we removed something from filter in front of a Cartesian, then we are doing a join from
// 2 different branches
auto cartesian = std::dynamic_pointer_cast<Cartesian>(op.input());
auto indexed_join = std::make_shared<IndexedJoin>(cartesian->left_op_, cartesian->right_op_);
SetOnParent(indexed_join);
return true;
}
if (!op.expression_) {
// if we emptied all the expressions from the filter, then we don't need this operator anymore
SetOnParent(op.input());
return true;
}
// still left something in the filter
return true;
}
@@ -154,41 +180,37 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
return true;
}
// Rewriting Cartesian assumes that the input plan will have Filter operations
// as soon as they are possible. Therefore we do not track filters above
// Cartesian because they should be irrelevant.
//
// For example, the following plan is not expected to be an input to
// IndexLookupRewriter.
//
// Filter n.prop = 16
// |
// Cartesian
// |
// |\
// | ScanAll (n)
// |
// ScanAll (m)
//
// Instead, the equivalent set of operations should be done this way:
//
// Cartesian
// |
// |\
// | Filter n.prop = 16
// | |
// | ScanAll (n)
// |
// ScanAll (m)
bool PreVisit(Cartesian &op) override {
prev_ops_.push_back(&op);
RewriteBranch(&op.left_op_);
RewriteBranch(&op.right_op_);
// we add the symbols that we encountered in the left part of the cartesian
// the reason for that is that in right part of the cartesian, we could be
// possibly using an indexed operation instead of a scan all
additional_bound_symbols_.insert(op.left_symbols_.begin(), op.left_symbols_.end());
op.right_op_->Accept(*this);
return false;
}
bool PostVisit(Cartesian &) override {
prev_ops_.pop_back();
// clear cartesian symbols as we exited the cartesian operator
additional_bound_symbols_.clear();
return true;
}
bool PreVisit(IndexedJoin &op) override {
prev_ops_.push_back(&op);
RewriteBranch(&op.left_);
RewriteBranch(&op.right_);
return false;
}
bool PostVisit(IndexedJoin &) override {
prev_ops_.pop_back();
return true;
}
@@ -488,6 +510,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
std::unordered_set<Expression *> filter_exprs_for_removal_;
std::vector<LogicalOperator *> prev_ops_;
// additional symbols that are present from other non-main branches but have influence on indexing
std::unordered_set<Symbol> additional_bound_symbols_;
struct LabelPropertyIndex {
LabelIx label;
// FilterInfo with PropertyFilter.
@@ -505,7 +530,22 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
new_root_ = input;
return;
}
prev_ops_.back()->set_input(input);
auto *parent = prev_ops_.back();
if (parent->HasSingleInput()) {
parent->set_input(input);
return;
}
if (parent->GetTypeInfo() == Cartesian::kType) {
auto *parent_cartesian = dynamic_cast<Cartesian *>(parent);
parent_cartesian->right_op_ = input;
parent_cartesian->right_symbols_ = input->ModifiedSymbols(*symbol_table_);
return;
}
// if we're sure that we want to set on parent, this should never happen
LOG_FATAL("Error during index rewriting of the query!");
}
void RewriteBranch(std::shared_ptr<LogicalOperator> *branch) {
@@ -535,10 +575,10 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
}
// Finds the label-property combination. The first criteria based on number of vertices indexed -> if one index has
// 10x less than the other one, always choose the smaller one. Otherwise, choose the index with smallest average group
// size based on key distribution. If average group size is equal, choose the index that has distribution closer to
// uniform distribution. Conditions based on average group size and key distribution can be only taken into account if
// the user has run `ANALYZE GRAPH` query before If the index cannot be found, nullopt is returned.
// 10x less than the other one, always choose the smaller one. Otherwise, choose the index with smallest average
// group size based on key distribution. If average group size is equal, choose the index that has distribution
// closer to uniform distribution. Conditions based on average group size and key distribution can be only taken
// into account if the user has run `ANALYZE GRAPH` query before If the index cannot be found, nullopt is returned.
std::optional<LabelPropertyIndex> FindBestLabelPropertyIndex(const Symbol &symbol,
const std::unordered_set<Symbol> &bound_symbols) {
auto are_bound = [&bound_symbols](const auto &used_symbols) {
@@ -551,10 +591,10 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
};
/*
* Comparator function between two indices. If new index has >= 10x vertices than the existing, it cannot be better.
* If it is <= 10x in number of vertices, check average group size of property values. The index with smaller
* average group size is better. If the average group size is the same, choose the one closer to the uniform
* distribution
* Comparator function between two indices. If new index has >= 10x vertices than the existing, it cannot be
* better. If it is <= 10x in number of vertices, check average group size of property values. The index with
* smaller average group size is better. If the average group size is the same, choose the one closer to the
* uniform distribution
* @param found: Current best label-property index.
* @param new_stats: Label-property index candidate.
* @param vertex_count: New index's number of vertices.
@@ -633,8 +673,12 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
const auto &input = scan.input();
const auto &node_symbol = scan.output_symbol_;
const auto &view = scan.view_;
const auto &modified_symbols = scan.ModifiedSymbols(*symbol_table_);
auto modified_symbols = scan.ModifiedSymbols(*symbol_table_);
std::unordered_set<Symbol> bound_symbols(modified_symbols.begin(), modified_symbols.end());
bound_symbols.insert(additional_bound_symbols_.begin(), additional_bound_symbols_.end());
auto are_bound = [&bound_symbols](const auto &used_symbols) {
for (const auto &used_symbol : used_symbols) {
if (!utils::Contains(bound_symbols, used_symbol)) {

View File

@@ -438,7 +438,6 @@ class RuleBasedPlanner {
last_op = HandleExpansion(std::move(last_op), matching, symbol_table, storage, bound_symbols,
match_context.new_symbols, named_paths, filters, match_context.view);
MG_ASSERT(named_paths.empty(), "Expected to generate all named paths");
// We bound all named path symbols, so just add them to new_symbols.
for (const auto &named_path : matching.named_paths) {
@@ -475,28 +474,156 @@ class RuleBasedPlanner {
return std::make_unique<plan::Merge>(std::move(input_op), std::move(on_match), std::move(on_create));
}
std::unique_ptr<LogicalOperator> GenerateExpansion(std::unique_ptr<LogicalOperator> last_op, const Matching &matching,
const Expansion &expansion, const SymbolTable &symbol_table,
AstStorage &storage, std::unordered_set<Symbol> &bound_symbols,
std::vector<Symbol> &new_symbols,
std::unordered_map<Symbol, std::vector<Symbol>> &named_paths,
Filters &filters, storage::View view) {
const auto &node1_symbol = symbol_table.at(*expansion.node1->identifier_);
if (bound_symbols.insert(node1_symbol).second) {
// We have just bound this symbol, so generate ScanAll which fills it.
last_op = std::make_unique<ScanAll>(std::move(last_op), node1_symbol, view);
new_symbols.emplace_back(node1_symbol);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
}
if (expansion.edge) {
last_op = GenExpand(std::move(last_op), expansion, symbol_table, bound_symbols, matching, storage, filters,
named_paths, new_symbols, view);
}
return last_op;
}
std::unique_ptr<LogicalOperator> GenerateIsomorphicExpansion(
std::unique_ptr<LogicalOperator> last_op, const Matching &matching, const SymbolTable &symbol_table,
AstStorage &storage, std::unordered_set<Symbol> &bound_symbols, std::vector<Symbol> &new_symbols,
std::unordered_map<Symbol, std::vector<Symbol>> &named_paths, Filters &filters, storage::View view,
ExpansionId isomorphic_id) {
for (size_t i = 0, size = matching.expansions.size(); i < size; i++) {
const auto &expansion = matching.expansions[i];
if (expansion.isomorphic_id != isomorphic_id) {
continue;
}
// When we picked a pattern to expand, we expand it through the end
last_op = GenerateExpansion(std::move(last_op), matching, expansion, symbol_table, storage, bound_symbols,
new_symbols, named_paths, filters, view);
}
return last_op;
}
std::unique_ptr<LogicalOperator> GenerateCartesian(std::unique_ptr<LogicalOperator> left,
std::unique_ptr<LogicalOperator> right,
const SymbolTable &symbol_table) {
auto left_symbols = left->ModifiedSymbols(symbol_table);
auto right_symbols = right->ModifiedSymbols(symbol_table);
return std::make_unique<Cartesian>(std::move(left), left_symbols, std::move(right), right_symbols);
}
std::unique_ptr<LogicalOperator> HandleExpansion(std::unique_ptr<LogicalOperator> last_op, const Matching &matching,
const SymbolTable &symbol_table, AstStorage &storage,
std::unordered_set<Symbol> &bound_symbols,
std::vector<Symbol> &new_symbols,
std::unordered_map<Symbol, std::vector<Symbol>> &named_paths,
Filters &filters, storage::View view) {
if (matching.expansions.empty()) {
return last_op;
}
std::set<ExpansionId> all_isomorphic_expansions;
for (const auto &expansion : matching.expansions) {
const auto &node1_symbol = symbol_table.at(*expansion.node1->identifier_);
if (bound_symbols.insert(node1_symbol).second) {
// We have just bound this symbol, so generate ScanAll which fills it.
last_op = std::make_unique<ScanAll>(std::move(last_op), node1_symbol, view);
new_symbols.emplace_back(node1_symbol);
all_isomorphic_expansions.insert(expansion.isomorphic_id);
}
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
if (!last_op) {
if (matching.expansions.size() == 1) {
return GenerateExpansion(std::move(last_op), matching, matching.expansions[0], symbol_table, storage,
bound_symbols, new_symbols, named_paths, filters, view);
}
if (all_isomorphic_expansions.size() == 1) {
return GenerateIsomorphicExpansion(std::move(last_op), matching, symbol_table, storage, bound_symbols,
new_symbols, named_paths, filters, view,
matching.expansions[0].isomorphic_id);
}
}
std::set<ExpansionId> visited_isomorphic_expansions;
bool added_new_expansions = true;
while (added_new_expansions) {
added_new_expansions = false;
for (size_t i = 0, size = matching.expansions.size(); i < size; i++) {
const auto &expansion = matching.expansions[i];
// We want to create separate matching branch operators for each isomorphic group of patterns
if (visited_isomorphic_expansions.contains(expansion.isomorphic_id)) {
continue;
}
const auto &node1_symbol = symbol_table.at(*expansion.node1->identifier_);
if (bound_symbols.contains(node1_symbol)) {
last_op = GenerateIsomorphicExpansion(std::move(last_op), matching, symbol_table, storage, bound_symbols,
new_symbols, named_paths, filters, view, expansion.isomorphic_id);
visited_isomorphic_expansions.insert(expansion.isomorphic_id);
added_new_expansions = true;
break;
}
if (expansion.edge) {
const auto &node2_symbol = symbol_table.at(*expansion.node2->identifier_);
const auto &edge_symbol = symbol_table.at(*expansion.edge->identifier_);
if (bound_symbols.contains(node2_symbol) || bound_symbols.contains(edge_symbol)) {
last_op = GenerateIsomorphicExpansion(std::move(last_op), matching, symbol_table, storage, bound_symbols,
new_symbols, named_paths, filters, view, expansion.isomorphic_id);
visited_isomorphic_expansions.insert(expansion.isomorphic_id);
added_new_expansions = true;
break;
}
}
}
}
std::vector<Symbol> cross_new_symbols;
std::unordered_set<Symbol> initial_bound_symbols = bound_symbols;
for (size_t i = 0, size = matching.expansions.size(); i < size; i++) {
const auto &expansion = matching.expansions[i];
// We want to create separate matching branch operators for each isomorphic group of patterns
if (visited_isomorphic_expansions.contains(expansion.isomorphic_id)) {
continue;
}
if (expansion.edge) {
last_op = GenExpand(std::move(last_op), expansion, symbol_table, bound_symbols, matching, storage, filters,
named_paths, new_symbols, view);
std::vector<Symbol> new_isomorphic_symbols{};
std::unordered_set<Symbol> new_bound_symbols{};
std::unique_ptr<LogicalOperator> isomorphic_expansion =
GenerateIsomorphicExpansion(std::make_unique<Once>(), matching, symbol_table, storage, new_bound_symbols,
new_isomorphic_symbols, named_paths, filters, view, expansion.isomorphic_id);
visited_isomorphic_expansions.insert(expansion.isomorphic_id);
new_symbols.insert(new_symbols.end(), new_isomorphic_symbols.begin(), new_isomorphic_symbols.end());
bound_symbols.insert(new_bound_symbols.begin(), new_bound_symbols.end());
if (!last_op) {
last_op = std::move(isomorphic_expansion);
cross_new_symbols = new_isomorphic_symbols;
continue;
}
last_op = GenerateCartesian(std::move(last_op), std::move(isomorphic_expansion), symbol_table);
for (const auto &new_symbol : cross_new_symbols) {
if (new_symbol.type_ == Symbol::Type::EDGE) {
last_op = EnsureCyphermorphism(std::move(last_op), new_symbol, matching, new_bound_symbols);
}
}
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
cross_new_symbols = new_isomorphic_symbols;
}
return last_op;
@@ -587,9 +714,24 @@ class RuleBasedPlanner {
new_symbols.emplace_back(node_symbol);
}
last_op = EnsureCyphermorphism(std::move(last_op), edge_symbol, matching, bound_symbols);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
return last_op;
}
std::unique_ptr<LogicalOperator> EnsureCyphermorphism(std::unique_ptr<LogicalOperator> last_op,
const Symbol &edge_symbol, const Matching &matching,
const std::unordered_set<Symbol> &bound_symbols) {
// Ensure Cyphermorphism (different edge symbols always map to
// different edges).
for (const auto &edge_symbols : matching.edge_symbols) {
if (edge_symbols.size() <= 1) {
continue;
}
if (edge_symbols.find(edge_symbol) == edge_symbols.end()) {
continue;
}
@@ -605,10 +747,6 @@ class RuleBasedPlanner {
}
}
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
return last_op;
}

View File

@@ -54,6 +54,7 @@
M(ForeachOperator, Operator, "Number of times Foreach operator was used.") \
M(EvaluatePatternFilterOperator, Operator, "Number of times EvaluatePatternFilter operator was used.") \
M(ApplyOperator, Operator, "Number of times ApplyOperator operator was used.") \
M(IndexedJoinOperator, Operator, "Number of times IndexedJoin operator was used.") \
\
M(ActiveLabelIndices, Index, "Number of active label indices in the system.") \
M(ActiveLabelPropertyIndices, Index, "Number of active label property indices in the system<.") \

View File

@@ -303,6 +303,7 @@ void *PoolResource::DoAllocate(size_t bytes, size_t alignment) {
[](const auto &a, const auto &b) { return a.GetBlockSize() < b.GetBlockSize(); });
if (it != pools_.end() && it->GetBlockSize() == block_size) {
last_alloc_pool_ = &*it;
last_dealloc_pool_ = &*it;
return it->Allocate();
}
// We don't have a pool for this block_size, so insert it in the sorted

View File

@@ -65,6 +65,7 @@ enum class TypeId : uint64_t {
LOAD_CSV,
FOREACH,
APPLY,
INDEXED_JOIN,
// Replication
REP_APPEND_DELTAS_REQ,

View File

@@ -171,3 +171,81 @@ Feature: Cartesian
MATCH (a)-[]->() MATCH (a:B) MATCH (a:C) RETURN a
"""
Then the result should be empty
Scenario: Multiple match + with 01
Given an empty graph
And having executed
"""
CREATE (:A {id: 1}), (:A {id: 2}), (:B {id: 1})
"""
When executing query:
"""
MATCH (a:A) WITH a MATCH (b:B) WHERE a.id = b.id RETURN a, b
"""
Then the result should be:
| a | b |
| (:A {id: 1}) | (:B {id: 1}) |
Scenario: Multiple match + with 02
Given an empty graph
And having executed
"""
CREATE (:A {id: 1}), (:A {id: 2}), (:B {id: 1})
"""
When executing query:
"""
MATCH (a:A) WITH a.id as id MATCH (a:A) return a;
"""
Then the result should be:
| a |
| (:A {id: 1}) |
| (:A {id: 2}) |
| (:A {id: 1}) |
| (:A {id: 2}) |
Scenario: Multiple match + with 03
Given an empty graph
And having executed
"""
CREATE (:A {id: 1})-[:TYPE]->(:B {id: 1}), (:A {id: 2})-[:TYPE]->(:B {id: 2})
"""
When executing query:
"""
MATCH (a:A) WITH a.id as id MATCH (a)-[:TYPE]->(b) return a, b;
"""
Then the result should be:
| a | b |
| (:A {id: 1}) | (:B {id: 1}) |
| (:A {id: 2}) | (:B {id: 2}) |
| (:A {id: 1}) | (:B {id: 1}) |
| (:A {id: 2}) | (:B {id: 2}) |
Scenario: Multiple match + with 04
Given an empty graph
And having executed
"""
CREATE (:A {id: 1})-[:TYPE]->(:B {id: 1}), (:A {id: 2})-[:TYPE]->(:B {id: 2})
"""
When executing query:
"""
MATCH (a:A) WITH a MATCH (a)-[:TYPE]->(b) return a, b;
"""
Then the result should be:
| a | b |
| (:A {id: 1}) | (:B {id: 1}) |
| (:A {id: 2}) | (:B {id: 2}) |
Scenario: Multiple match + with 05
Given an empty graph
And having executed
"""
CREATE (:A {id: 1})-[:TYPE]->(:B {id: 1}), (:A {id: 2})-[:TYPE]->(:B {id: 2})
"""
When executing query:
"""
MATCH (a:A) WITH a MATCH (c:A {id: 1}), (a)-[:TYPE]->(b) return a, b;
"""
Then the result should be:
| a | b |
| (:A {id: 1}) | (:B {id: 1}) |
| (:A {id: 2}) | (:B {id: 2}) |