memgraph/src/query/plan/variable_start_planner.cpp

#include "query/plan/variable_start_planner.hpp"

#include <limits>
#include <queue>

#include "utils/flag_validation.hpp"

DEFINE_VALIDATED_uint64(
    query_max_plans, 1000U, "Maximum number of generated plans for a query",
    FLAG_IN_RANGE(1, std::numeric_limits<std::uint64_t>::max()));

namespace query::plan::impl {

namespace {

class NodeSymbolHash {
 public:
  explicit NodeSymbolHash(const SymbolTable &symbol_table)
      : symbol_table_(symbol_table) {}

  size_t operator()(const NodeAtom *node_atom) const {
    return std::hash<Symbol>{}(symbol_table_.at(*node_atom->identifier_));
  }

 private:
  const SymbolTable &symbol_table_;
};

class NodeSymbolEqual {
 public:
  explicit NodeSymbolEqual(const SymbolTable &symbol_table)
      : symbol_table_(symbol_table) {}

  bool operator()(const NodeAtom *node_atom1,
                  const NodeAtom *node_atom2) const {
    return symbol_table_.at(*node_atom1->identifier_) ==
           symbol_table_.at(*node_atom2->identifier_);
  }

 private:
  const SymbolTable &symbol_table_;
};

// Add applicable expansions for `node_symbol` to `next_expansions`. These
// expansions are removed from `node_symbol_to_expansions`, while
// `seen_expansions` and `expanded_symbols` are populated with new data.
void AddNextExpansions(
    const Symbol &node_symbol, const Matching &matching,
    const SymbolTable &symbol_table,
    std::unordered_set<Symbol> &expanded_symbols,
    std::unordered_map<Symbol, std::set<int>> &node_symbol_to_expansions,
    std::unordered_set<int> &seen_expansions,
    std::queue<Expansion> &next_expansions) {
  auto node_to_expansions_it = node_symbol_to_expansions.find(node_symbol);
  if (node_to_expansions_it == node_symbol_to_expansions.end()) {
    return;
  }
  // Returns true if the expansion is a regular expand or if it is a variable
  // path expand, but with bound symbols used inside the range expression.
  auto can_expand = [&](auto &expansion) {
    for (const auto &range_symbol : expansion.symbols_in_range) {
      // If the symbols used in range need to be bound during this whole
      // expansion, we must check whether they have already been expanded and
      // therefore bound. If the symbols are not found in the whole expansion,
      // then the semantic analysis should guarantee that the symbols have been
      // bound long before we expand.
      if (matching.expansion_symbols.find(range_symbol) !=
              matching.expansion_symbols.end() &&
          expanded_symbols.find(range_symbol) == expanded_symbols.end()) {
        return false;
      }
    }
    return true;
  };
  auto &node_expansions = node_to_expansions_it->second;
  auto node_expansions_it = node_expansions.begin();
  while (node_expansions_it != node_to_expansions_it->second.end()) {
    auto expansion_id = *node_expansions_it;
    if (seen_expansions.find(expansion_id) != seen_expansions.end()) {
      // Skip and erase seen (already expanded) expansions.
      node_expansions_it = node_expansions.erase(node_expansions_it);
      continue;
    }
    auto expansion = matching.expansions[expansion_id];
    if (!can_expand(expansion)) {
      // Skip but save expansions which need other symbols for later.
      ++node_expansions_it;
      continue;
    }
    if (symbol_table.at(*expansion.node1->identifier_) != node_symbol) {
      // We are not expanding from node1, so flip the expansion.
      debug_assert(
          expansion.node2 &&
              symbol_table.at(*expansion.node2->identifier_) == node_symbol,
          "Expected node_symbol to be bound in node2");
      if (expansion.edge->type_ != EdgeAtom::Type::BREADTH_FIRST) {
        // BFS must *not* be flipped. Doing that changes the BFS results.
        std::swap(expansion.node1, expansion.node2);
        expansion.is_flipped = true;
        if (expansion.direction != EdgeAtom::Direction::BOTH) {
          expansion.direction = expansion.direction == EdgeAtom::Direction::IN
                                    ? EdgeAtom::Direction::OUT
                                    : EdgeAtom::Direction::IN;
        }
      }
    }
    seen_expansions.insert(expansion_id);
    expanded_symbols.insert(symbol_table.at(*expansion.node1->identifier_));
    if (expansion.edge) {
      expanded_symbols.insert(symbol_table.at(*expansion.edge->identifier_));
      expanded_symbols.insert(symbol_table.at(*expansion.node2->identifier_));
    }
    next_expansions.emplace(std::move(expansion));
    node_expansions_it = node_expansions.erase(node_expansions_it);
  }
  if (node_expansions.empty()) {
    node_symbol_to_expansions.erase(node_to_expansions_it);
  }
}

// Generates expansions emanating from the start_node by forming a chain. When
// the chain can no longer be continued, a different starting node is picked
// among remaining expansions and the process continues. This is done until all
// matching.expansions are used.
std::vector<Expansion> ExpansionsFrom(const NodeAtom *start_node,
                                      const Matching &matching,
                                      const SymbolTable &symbol_table) {
  // Make a copy of node_symbol_to_expansions, because we will modify it as
  // expansions are chained.
  auto node_symbol_to_expansions = matching.node_symbol_to_expansions;
  std::unordered_set<int> seen_expansions;
  std::queue<Expansion> next_expansions;
  std::unordered_set<Symbol> expanded_symbols(
      {symbol_table.at(*start_node->identifier_)});
  auto add_next_expansions = [&](const auto *node) {
    AddNextExpansions(symbol_table.at(*node->identifier_), matching,
                      symbol_table, expanded_symbols, node_symbol_to_expansions,
                      seen_expansions, next_expansions);
  };
  add_next_expansions(start_node);
  // Potential optimization: expansions and next_expansions could be merge into
  // a single vector and an index could be used to determine from which should
  // additional expansions be added.
  std::vector<Expansion> expansions;
  while (!next_expansions.empty()) {
    auto expansion = next_expansions.front();
    next_expansions.pop();
    expansions.emplace_back(expansion);
    add_next_expansions(expansion.node1);
    if (expansion.node2) {
      add_next_expansions(expansion.node2);
    }
  }
  if (!node_symbol_to_expansions.empty()) {
    // We could pick a new starting expansion, but to avoid runtime
    // complexity, simply append the remaining expansions. They should have the
    // correct order, since the original expansions were verified during
    // semantic analysis.
    for (int i = 0; i < matching.expansions.size(); ++i) {
      if (seen_expansions.find(i) != seen_expansions.end()) {
        continue;
      }
      expansions.emplace_back(matching.expansions[i]);
    }
  }
  return expansions;
}

// Collect all unique nodes from expansions. Uniqueness is determined by
// symbol uniqueness.
auto ExpansionNodes(const std::vector<Expansion> &expansions,
                    const SymbolTable &symbol_table) {
  std::unordered_set<NodeAtom *, NodeSymbolHash, NodeSymbolEqual> nodes(
      expansions.size(), NodeSymbolHash(symbol_table),
      NodeSymbolEqual(symbol_table));
  for (const auto &expansion : expansions) {
    // TODO: Handle labels and properties from different node atoms.
    nodes.insert(expansion.node1);
    if (expansion.node2) {
      nodes.insert(expansion.node2);
    }
  }
  return nodes;
}

// Generates n matchings, where n is the number of nodes to match. Each Matching
// will have a different node as a starting node for expansion.
class VaryMatchingStart {
 public:
  VaryMatchingStart(const Matching &matching, const SymbolTable &symbol_table)
      : matching_(matching),
        symbol_table_(symbol_table),
        nodes_(ExpansionNodes(matching.expansions, symbol_table)) {}

  class iterator {
   public:
    typedef std::input_iterator_tag iterator_category;
    typedef Matching value_type;
    typedef long difference_type;
    typedef const Matching &reference;
    typedef const Matching *pointer;

    iterator(VaryMatchingStart &self, bool is_done)
        : self_(self),
          // Use the original matching as the first matching. We are only
          // interested in changing the expansions part, so the remaining fields
          // should stay the same. This also produces a matching for the case
          // when there are no nodes.
          current_matching_(self.matching_) {
      if (!self_.nodes_.empty()) {
        // Overwrite the original matching expansions with the new ones by
        // generating it from the first start node.
        start_nodes_it_ = self_.nodes_.begin();
        current_matching_.expansions = ExpansionsFrom(
            **start_nodes_it_, self_.matching_, self_.symbol_table_);
      }
      debug_assert(
          start_nodes_it_ || self_.nodes_.empty(),
          "start_nodes_it_ should only be nullopt when self_.nodes_ is empty");
      if (is_done) {
        start_nodes_it_ = self.nodes_.end();
      }
    }

    iterator &operator++() {
      if (!start_nodes_it_) {
        debug_assert(self_.nodes_.empty(),
                     "start_nodes_it_ should only be nullopt when self_.nodes_ "
                     "is empty");
        start_nodes_it_ = self_.nodes_.end();
      }
      if (*start_nodes_it_ == self_.nodes_.end()) {
        return *this;
      }
      ++*start_nodes_it_;
      // start_nodes_it_ can become equal to `end` and we shouldn't dereference
      // iterator in that case.
      if (*start_nodes_it_ == self_.nodes_.end()) {
        return *this;
      }
      const auto &start_node = **start_nodes_it_;
      current_matching_.expansions =
          ExpansionsFrom(start_node, self_.matching_, self_.symbol_table_);
      return *this;
    }

    bool operator==(const iterator &other) const {
      return &self_ == &other.self_ && start_nodes_it_ == other.start_nodes_it_;
    }

    bool operator!=(const iterator &other) const { return !(*this == other); }

    reference operator*() const { return current_matching_; }
    pointer operator->() const { return &current_matching_; }

   private:
    VaryMatchingStart &self_;
    Matching current_matching_;
    // Iterator over start nodes. Optional is used for differentiating the case
    // when there are no start nodes vs. VaryMatchingStart::iterator itself
    // being at the end. When there are no nodes, this iterator needs to produce
    // a single result, which is the original matching passed in. Setting
    // start_nodes_it_ to end signifies the end of our iteration.
    std::experimental::optional<std::unordered_set<NodeAtom *, NodeSymbolHash,
                                                   NodeSymbolEqual>::iterator>
        start_nodes_it_;
  };

  auto begin() { return iterator(*this, false); }
  auto end() { return iterator(*this, true); }

 private:
  friend class iterator;
  const Matching &matching_;
  const SymbolTable &symbol_table_;
  std::unordered_set<NodeAtom *, NodeSymbolHash, NodeSymbolEqual> nodes_;
};

// Similar to VaryMatchingStart, but varies the starting nodes for all given
// matchings. After all matchings produce multiple alternative starts, the
// Cartesian product of all of them is returned.
auto VaryMultiMatchingStarts(const std::vector<Matching> &matchings,
                             const SymbolTable &symbol_table) {
  std::vector<std::vector<Matching>> variants;
  for (const auto &matching : matchings) {
    auto variant = iter::slice(VaryMatchingStart(matching, symbol_table), 0UL,
                               FLAGS_query_max_plans);
    variants.emplace_back(
        std::vector<Matching>(variant.begin(), variant.end()));
  }
  return iter::slice(MakeCartesianProduct(std::move(variants)), 0UL,
                     FLAGS_query_max_plans);
}

}  // namespace

// Produces alternative query parts out of a single part by varying how each
// graph matching is done.
std::vector<QueryPart> VaryQueryPartMatching(const QueryPart &query_part,
                                             const SymbolTable &symbol_table) {
  std::vector<QueryPart> variants;
  // Get multiple regular matchings, each starting from different node.
  auto matchings = VaryMatchingStart(query_part.matching, symbol_table);
  // Get multiple optional matchings, where each combination has different
  // starting nodes.
  auto optional_matchings =
      VaryMultiMatchingStarts(query_part.optional_matching, symbol_table);
  // Like optional matching, but for merge matchings.
  auto merge_matchings =
      VaryMultiMatchingStarts(query_part.merge_matching, symbol_table);
  // After we have all valid combinations of each matching, we need to produce
  // combinations of them. This is similar to Cartesian product, but some
  // matchings can be empty (optional and merge) and `matchings` is of different
  // type (vector) than `optional_matchings` and `merge_matchings` (which are
  // vectors of vectors).
  for (const auto &matching : matchings) {
    // matchings will always have at least a single element, so we can use a for
    // loop. On the other hand, optional and merge matchings can be empty so we
    // need an iterator and do...while loop.
    auto optional_it = optional_matchings.begin();
    auto optional_end = optional_matchings.end();
    do {
      auto merge_it = merge_matchings.begin();
      auto merge_end = merge_matchings.end();
      do {
        // Produce parts for each possible combination. E.g. if we have:
        //    * matchings (m1) and (m2)
        //    * optional matchings (o1) and (o2)
        //    * merge matching (g1)
        // We want to produce parts for:
        //    * (m1), (o1), (g1)
        //    * (m1), (o2), (g1)
        //    * (m2), (o1), (g1)
        //    * (m2), (o2), (g1)
        variants.emplace_back(QueryPart{matching});
        variants.back().remaining_clauses = query_part.remaining_clauses;
        if (optional_it != optional_matchings.end()) {
          // In case we started with empty optional matchings.
          variants.back().optional_matching = *optional_it;
        }
        if (merge_it != merge_matchings.end()) {
          // In case we started with empty merge matchings.
          variants.back().merge_matching = *merge_it;
        }
        // Since we can start with the iterator at the end, we have to first
        // compare it and then increment it. After we increment, we need to
        // check again to avoid generating with empty matching.
      } while (merge_it != merge_end && ++merge_it != merge_end);
    } while (optional_it != optional_end && ++optional_it != optional_end);
  }
  return variants;
}

}  // namespace query::plan::impl